Light, fluffy, and always free - The AWS Local Emulator alternative
fix(iam): make entity stores thread-safe under concurrent mutation (#1509)
Steve976f819e5c2c
diff --git a/src/main/java/io/github/hectorvent/floci/services/iam/IamService.java b/src/main/java/io/github/hectorvent/floci/services/iam/IamService.javaindex 10194aa792..03a6fb0a41 100644--- a/src/main/java/io/github/hectorvent/floci/services/iam/IamService.java+++ b/src/main/java/io/github/hectorvent/floci/services/iam/IamService.java@@ -447,18 +447,22 @@ public class IamService {public PolicyVersion createPolicyVersion(String policyArn, String document, boolean setAsDefault) {rejectIfAwsManaged(policyArn);IamPolicy policy = getPolicy(policyArn);- int nextVersionNum = policy.getVersions().size() + 1;- if (nextVersionNum > 5) {- throw new AwsException("LimitExceeded",- "A managed policy can have up to 5 versions.", 409);- }- String versionId = "v" + nextVersionNum;- PolicyVersion version = new PolicyVersion(versionId, document, setAsDefault);- if (setAsDefault) {- policy.getVersions().values().forEach(v -> v.setDefaultVersion(false));- policy.setDefaultVersionId(versionId);+ Map<String, PolicyVersion> versions = policy.getVersions();+ PolicyVersion version;+ synchronized (versions) {+ int nextVersionNum = versions.size() + 1;+ if (nextVersionNum > 5) {+ throw new AwsException("LimitExceeded",+ "A managed policy can have up to 5 versions.", 409);+ }+ String versionId = "v" + nextVersionNum;+ version = new PolicyVersion(versionId, document, setAsDefault);+ if (setAsDefault) {+ versions.values().forEach(v -> v.setDefaultVersion(false));+ policy.setDefaultVersionId(versionId);+ }+ versions.put(versionId, version);}- policy.getVersions().put(versionId, version);policy.setUpdateDate(Instant.now());policies.put(policyArn, policy);return version;@@ -481,27 +485,36 @@ public class IamService {throw new AwsException("DeleteConflict","Cannot delete the default version of a policy.", 409);}- if (!policy.getVersions().containsKey(versionId)) {- throw new AwsException("NoSuchEntity",- "Policy version " + versionId + " does not exist.", 404);+ Map<String, PolicyVersion> versions = policy.getVersions();+ synchronized (versions) {+ if (!versions.containsKey(versionId)) {+ throw new AwsException("NoSuchEntity",+ "Policy version " + versionId + " does not exist.", 404);+ }+ versions.remove(versionId);}- policy.getVersions().remove(versionId);policies.put(policyArn, policy);}public List<PolicyVersion> listPolicyVersions(String policyArn) {- return new ArrayList<>(getPolicy(policyArn).getVersions().values());+ Map<String, PolicyVersion> versions = getPolicy(policyArn).getVersions();+ synchronized (versions) {+ return new ArrayList<>(versions.values());+ }}public void setDefaultPolicyVersion(String policyArn, String versionId) {rejectIfAwsManaged(policyArn);IamPolicy policy = getPolicy(policyArn);- if (!policy.getVersions().containsKey(versionId)) {- throw new AwsException("NoSuchEntity",- "Policy version " + versionId + " does not exist.", 404);+ Map<String, PolicyVersion> versions = policy.getVersions();+ synchronized (versions) {+ if (!versions.containsKey(versionId)) {+ throw new AwsException("NoSuchEntity",+ "Policy version " + versionId + " does not exist.", 404);+ }+ versions.values().forEach(v -> v.setDefaultVersion(false));+ versions.get(versionId).setDefaultVersion(true);}- policy.getVersions().values().forEach(v -> v.setDefaultVersion(false));- policy.getVersions().get(versionId).setDefaultVersion(true);policy.setDefaultVersionId(versionId);policies.put(policyArn, policy);}@@ -826,13 +839,16 @@ public class IamService {public void addRoleToInstanceProfile(String instanceProfileName, String roleName) {InstanceProfile profile = getInstanceProfile(instanceProfileName);getRole(roleName); // validates existence- if (!profile.getRoleNames().contains(roleName)) {- if (!profile.getRoleNames().isEmpty()) {- throw new AwsException("LimitExceeded",- "An instance profile can contain at most 1 role.", 409);+ List<String> roleNames = profile.getRoleNames();+ synchronized (roleNames) {+ if (!roleNames.contains(roleName)) {+ if (!roleNames.isEmpty()) {+ throw new AwsException("LimitExceeded",+ "An instance profile can contain at most 1 role.", 409);+ }+ roleNames.add(roleName);+ instanceProfiles.put(instanceProfileName, profile);}- profile.getRoleNames().add(roleName);- instanceProfiles.put(instanceProfileName, profile);}}diff --git a/src/main/java/io/github/hectorvent/floci/services/iam/model/IamGroup.java b/src/main/java/io/github/hectorvent/floci/services/iam/model/IamGroup.javaindex 0b2f120d47..40618b4a20 100644--- a/src/main/java/io/github/hectorvent/floci/services/iam/model/IamGroup.java+++ b/src/main/java/io/github/hectorvent/floci/services/iam/model/IamGroup.java@@ -4,10 +4,10 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties;import io.quarkus.runtime.annotations.RegisterForReflection;import java.time.Instant;-import java.util.ArrayList;-import java.util.HashMap;import java.util.List;import java.util.Map;+import java.util.concurrent.ConcurrentHashMap;+import java.util.concurrent.CopyOnWriteArrayList;@RegisterForReflection@JsonIgnoreProperties(ignoreUnknown = true)@@ -18,9 +18,9 @@ public class IamGroup {private String path;private String arn;private Instant createDate;- private List<String> userNames = new ArrayList<>();- private List<String> attachedPolicyArns = new ArrayList<>();- private Map<String, String> inlinePolicies = new HashMap<>();+ private List<String> userNames = new CopyOnWriteArrayList<>();+ private List<String> attachedPolicyArns = new CopyOnWriteArrayList<>();+ private Map<String, String> inlinePolicies = new ConcurrentHashMap<>();public IamGroup() {}@@ -48,11 +48,17 @@ public class IamGroup {public void setCreateDate(Instant createDate) { this.createDate = createDate; }public List<String> getUserNames() { return userNames; }- public void setUserNames(List<String> userNames) { this.userNames = userNames; }+ public void setUserNames(List<String> userNames) {+ this.userNames = new CopyOnWriteArrayList<>(userNames);+ }public List<String> getAttachedPolicyArns() { return attachedPolicyArns; }- public void setAttachedPolicyArns(List<String> attachedPolicyArns) { this.attachedPolicyArns = attachedPolicyArns; }+ public void setAttachedPolicyArns(List<String> attachedPolicyArns) {+ this.attachedPolicyArns = new CopyOnWriteArrayList<>(attachedPolicyArns);+ }public Map<String, String> getInlinePolicies() { return inlinePolicies; }- public void setInlinePolicies(Map<String, String> inlinePolicies) { this.inlinePolicies = inlinePolicies; }+ public void setInlinePolicies(Map<String, String> inlinePolicies) {+ this.inlinePolicies = new ConcurrentHashMap<>(inlinePolicies);+ }}diff --git a/src/main/java/io/github/hectorvent/floci/services/iam/model/IamPolicy.java b/src/main/java/io/github/hectorvent/floci/services/iam/model/IamPolicy.javaindex 21fe34fa9a..b7a947367b 100644--- a/src/main/java/io/github/hectorvent/floci/services/iam/model/IamPolicy.java+++ b/src/main/java/io/github/hectorvent/floci/services/iam/model/IamPolicy.java@@ -4,9 +4,10 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties;import io.quarkus.runtime.annotations.RegisterForReflection;import java.time.Instant;-import java.util.HashMap;+import java.util.Collections;import java.util.LinkedHashMap;import java.util.Map;+import java.util.concurrent.ConcurrentHashMap;@RegisterForReflection@JsonIgnoreProperties(ignoreUnknown = true)@@ -21,9 +22,9 @@ public class IamPolicy {private int attachmentCount = 0;private Instant createDate;private Instant updateDate;- private Map<String, String> tags = new HashMap<>();+ private Map<String, String> tags = new ConcurrentHashMap<>();// versionId -> PolicyVersion (ordered for consistent listing)- private Map<String, PolicyVersion> versions = new LinkedHashMap<>();+ private Map<String, PolicyVersion> versions = Collections.synchronizedMap(new LinkedHashMap<>());public IamPolicy() {}@@ -73,8 +74,12 @@ public class IamPolicy {public void setUpdateDate(Instant updateDate) { this.updateDate = updateDate; }public Map<String, String> getTags() { return tags; }- public void setTags(Map<String, String> tags) { this.tags = tags; }+ public void setTags(Map<String, String> tags) {+ this.tags = new ConcurrentHashMap<>(tags);+ }public Map<String, PolicyVersion> getVersions() { return versions; }- public void setVersions(Map<String, PolicyVersion> versions) { this.versions = versions; }+ public void setVersions(Map<String, PolicyVersion> versions) {+ this.versions = Collections.synchronizedMap(new LinkedHashMap<>(versions));+ }}diff --git a/src/main/java/io/github/hectorvent/floci/services/iam/model/IamRole.java b/src/main/java/io/github/hectorvent/floci/services/iam/model/IamRole.javaindex 1c445a808e..e964a83b9d 100644--- a/src/main/java/io/github/hectorvent/floci/services/iam/model/IamRole.java+++ b/src/main/java/io/github/hectorvent/floci/services/iam/model/IamRole.java@@ -4,10 +4,10 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties;import io.quarkus.runtime.annotations.RegisterForReflection;import java.time.Instant;-import java.util.ArrayList;-import java.util.HashMap;import java.util.List;import java.util.Map;+import java.util.concurrent.ConcurrentHashMap;+import java.util.concurrent.CopyOnWriteArrayList;@RegisterForReflection@JsonIgnoreProperties(ignoreUnknown = true)@@ -21,9 +21,9 @@ public class IamRole {private String description;private int maxSessionDuration = 3600;private Instant createDate;- private Map<String, String> tags = new HashMap<>();- private List<String> attachedPolicyArns = new ArrayList<>();- private Map<String, String> inlinePolicies = new HashMap<>();+ private Map<String, String> tags = new ConcurrentHashMap<>();+ private List<String> attachedPolicyArns = new CopyOnWriteArrayList<>();+ private Map<String, String> inlinePolicies = new ConcurrentHashMap<>();private String permissionsBoundaryArn;public IamRole() {}@@ -63,13 +63,19 @@ public class IamRole {public void setCreateDate(Instant createDate) { this.createDate = createDate; }public Map<String, String> getTags() { return tags; }- public void setTags(Map<String, String> tags) { this.tags = tags; }+ public void setTags(Map<String, String> tags) {+ this.tags = new ConcurrentHashMap<>(tags);+ }public List<String> getAttachedPolicyArns() { return attachedPolicyArns; }- public void setAttachedPolicyArns(List<String> attachedPolicyArns) { this.attachedPolicyArns = attachedPolicyArns; }+ public void setAttachedPolicyArns(List<String> attachedPolicyArns) {+ this.attachedPolicyArns = new CopyOnWriteArrayList<>(attachedPolicyArns);+ }public Map<String, String> getInlinePolicies() { return inlinePolicies; }- public void setInlinePolicies(Map<String, String> inlinePolicies) { this.inlinePolicies = inlinePolicies; }+ public void setInlinePolicies(Map<String, String> inlinePolicies) {+ this.inlinePolicies = new ConcurrentHashMap<>(inlinePolicies);+ }public String getPermissionsBoundaryArn() { return permissionsBoundaryArn; }public void setPermissionsBoundaryArn(String permissionsBoundaryArn) { this.permissionsBoundaryArn = permissionsBoundaryArn; }diff --git a/src/main/java/io/github/hectorvent/floci/services/iam/model/IamUser.java b/src/main/java/io/github/hectorvent/floci/services/iam/model/IamUser.javaindex 07705952a0..d6776873da 100644--- a/src/main/java/io/github/hectorvent/floci/services/iam/model/IamUser.java+++ b/src/main/java/io/github/hectorvent/floci/services/iam/model/IamUser.java@@ -4,10 +4,10 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties;import io.quarkus.runtime.annotations.RegisterForReflection;import java.time.Instant;-import java.util.ArrayList;-import java.util.HashMap;import java.util.List;import java.util.Map;+import java.util.concurrent.ConcurrentHashMap;+import java.util.concurrent.CopyOnWriteArrayList;@RegisterForReflection@JsonIgnoreProperties(ignoreUnknown = true)@@ -19,10 +19,10 @@ public class IamUser {private String arn;private Instant createDate;private Instant passwordLastUsed;- private Map<String, String> tags = new HashMap<>();- private List<String> groupNames = new ArrayList<>();- private List<String> attachedPolicyArns = new ArrayList<>();- private Map<String, String> inlinePolicies = new HashMap<>();+ private Map<String, String> tags = new ConcurrentHashMap<>();+ private List<String> groupNames = new CopyOnWriteArrayList<>();+ private List<String> attachedPolicyArns = new CopyOnWriteArrayList<>();+ private Map<String, String> inlinePolicies = new ConcurrentHashMap<>();private String permissionsBoundaryArn;public IamUser() {}@@ -54,16 +54,24 @@ public class IamUser {public void setPasswordLastUsed(Instant passwordLastUsed) { this.passwordLastUsed = passwordLastUsed; }public Map<String, String> getTags() { return tags; }- public void setTags(Map<String, String> tags) { this.tags = tags; }+ public void setTags(Map<String, String> tags) {+ this.tags = new ConcurrentHashMap<>(tags);+ }public List<String> getGroupNames() { return groupNames; }- public void setGroupNames(List<String> groupNames) { this.groupNames = groupNames; }+ public void setGroupNames(List<String> groupNames) {+ this.groupNames = new CopyOnWriteArrayList<>(groupNames);+ }public List<String> getAttachedPolicyArns() { return attachedPolicyArns; }- public void setAttachedPolicyArns(List<String> attachedPolicyArns) { this.attachedPolicyArns = attachedPolicyArns; }+ public void setAttachedPolicyArns(List<String> attachedPolicyArns) {+ this.attachedPolicyArns = new CopyOnWriteArrayList<>(attachedPolicyArns);+ }public Map<String, String> getInlinePolicies() { return inlinePolicies; }- public void setInlinePolicies(Map<String, String> inlinePolicies) { this.inlinePolicies = inlinePolicies; }+ public void setInlinePolicies(Map<String, String> inlinePolicies) {+ this.inlinePolicies = new ConcurrentHashMap<>(inlinePolicies);+ }public String getPermissionsBoundaryArn() { return permissionsBoundaryArn; }public void setPermissionsBoundaryArn(String permissionsBoundaryArn) { this.permissionsBoundaryArn = permissionsBoundaryArn; }diff --git a/src/main/java/io/github/hectorvent/floci/services/iam/model/InstanceProfile.java b/src/main/java/io/github/hectorvent/floci/services/iam/model/InstanceProfile.javaindex 5f04569aec..aa54acee55 100644--- a/src/main/java/io/github/hectorvent/floci/services/iam/model/InstanceProfile.java+++ b/src/main/java/io/github/hectorvent/floci/services/iam/model/InstanceProfile.java@@ -4,8 +4,8 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties;import io.quarkus.runtime.annotations.RegisterForReflection;import java.time.Instant;-import java.util.ArrayList;import java.util.List;+import java.util.concurrent.CopyOnWriteArrayList;@RegisterForReflection@JsonIgnoreProperties(ignoreUnknown = true)@@ -16,7 +16,7 @@ public class InstanceProfile {private String path;private String arn;private Instant createDate;- private List<String> roleNames = new ArrayList<>();+ private List<String> roleNames = new CopyOnWriteArrayList<>();public InstanceProfile() {}@@ -45,5 +45,7 @@ public class InstanceProfile {public void setCreateDate(Instant createDate) { this.createDate = createDate; }public List<String> getRoleNames() { return roleNames; }- public void setRoleNames(List<String> roleNames) { this.roleNames = roleNames; }+ public void setRoleNames(List<String> roleNames) {+ this.roleNames = new CopyOnWriteArrayList<>(roleNames);+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/iam/IamConcurrencyTest.java b/src/test/java/io/github/hectorvent/floci/services/iam/IamConcurrencyTest.javanew file mode 100644index 0000000000..f4cf13b20e--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/iam/IamConcurrencyTest.java@@ -0,0 +1,260 @@+package io.github.hectorvent.floci.services.iam;++import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.storage.InMemoryStorage;+import io.github.hectorvent.floci.services.iam.model.IamPolicy;+import org.junit.jupiter.api.DynamicTest;+import org.junit.jupiter.api.TestFactory;+import org.junit.jupiter.api.Test;++import java.util.HashSet;+import java.util.List;+import java.util.Set;+import java.util.concurrent.CopyOnWriteArrayList;+import java.util.concurrent.CountDownLatch;+import java.util.concurrent.ExecutorService;+import java.util.concurrent.Executors;+import java.util.concurrent.TimeUnit;+import java.util.function.IntConsumer;+import java.util.function.ToIntFunction;+import java.util.stream.Stream;++import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.junit.jupiter.api.Assertions.fail;++/**+ * Concurrent mutation of every shared IAM entity collection (tags, inline policies,+ * attachments, group membership, policy versions, instance-profile roles) must not lose+ * or corrupt entries. Each scenario runs over many trials to surface the race reliably.+ */+class IamConcurrencyTest {++ private static final int TRIALS = 200;+ private static final int N = 24;+ private static final int THREADS = 8;++ private static IamService newIamService() {+ return new IamService(+ new InMemoryStorage<>(), new InMemoryStorage<>(), new InMemoryStorage<>(),+ new InMemoryStorage<>(), new InMemoryStorage<>(), new InMemoryStorage<>(),+ new InMemoryStorage<>(), new RegionResolver("eu-central-1", "000000000000"), false);+ }++ /** Sets up one entity on the given service and returns the i-th concurrent mutation. */+ private interface RaceSetup {+ IntConsumer prepare(IamService iam);+ }++ @TestFactory+ Stream<DynamicTest> everyIamCollectionStaysRaceFree() {+ record Scenario(String name, RaceSetup setup, ToIntFunction<IamService> finalCount) {}++ List<Scenario> scenarios = List.of(+ new Scenario("role.tags", iam -> {+ iam.createRole("r", "/", "{}", null, 3600, null);+ return i -> iam.tagRole("r", java.util.Map.of("k" + i, "v" + i));+ }, iam -> iam.getRole("r").getTags().size()),++ new Scenario("role.inlinePolicies", iam -> {+ iam.createRole("r", "/", "{}", null, 3600, null);+ return i -> iam.putRolePolicy("r", "p" + i, "{\"S\":" + i + "}");+ }, iam -> iam.listRolePolicies("r").size()),++ new Scenario("role.attachedPolicyArns", iam -> {+ iam.createRole("r", "/", "{}", null, 3600, null);+ String[] arns = createPolicies(iam, "rp");+ return i -> iam.attachRolePolicy("r", arns[i]);+ }, iam -> iam.listAttachedRolePolicies("r", null).size()),++ new Scenario("user.tags", iam -> {+ iam.createUser("u", "/");+ return i -> iam.tagUser("u", java.util.Map.of("k" + i, "v" + i));+ }, iam -> iam.getUser("u").getTags().size()),++ new Scenario("user.inlinePolicies", iam -> {+ iam.createUser("u", "/");+ return i -> iam.putUserPolicy("u", "p" + i, "{\"S\":" + i + "}");+ }, iam -> iam.listUserPolicies("u").size()),++ new Scenario("user.attachedPolicyArns", iam -> {+ iam.createUser("u", "/");+ String[] arns = createPolicies(iam, "up");+ return i -> iam.attachUserPolicy("u", arns[i]);+ }, iam -> iam.listAttachedUserPolicies("u", null).size()),++ new Scenario("user.groupNames", iam -> {+ iam.createUser("u", "/");+ for (int i = 0; i < N; i++) iam.createGroup("g" + i, "/");+ return i -> iam.addUserToGroup("g" + i, "u");+ }, iam -> iam.getUser("u").getGroupNames().size()),++ new Scenario("group.userNames", iam -> {+ iam.createGroup("g", "/");+ for (int i = 0; i < N; i++) iam.createUser("u" + i, "/");+ return i -> iam.addUserToGroup("g", "u" + i);+ }, iam -> iam.getGroup("g").getUserNames().size()),++ new Scenario("group.inlinePolicies", iam -> {+ iam.createGroup("g", "/");+ return i -> iam.putGroupPolicy("g", "p" + i, "{\"S\":" + i + "}");+ }, iam -> iam.listGroupPolicies("g").size()),++ new Scenario("group.attachedPolicyArns", iam -> {+ iam.createGroup("g", "/");+ String[] arns = createPolicies(iam, "gp");+ return i -> iam.attachGroupPolicy("g", arns[i]);+ }, iam -> iam.listAttachedGroupPolicies("g", null).size()),++ new Scenario("policy.tags", iam -> {+ IamPolicy p = iam.createPolicy("pol", "/", null, "{}", null);+ return i -> iam.tagPolicy(p.getArn(), java.util.Map.of("k" + i, "v" + i));+ }, iam -> iam.getPolicy(onlyPolicyArn(iam)).getTags().size())+ );++ return scenarios.stream().map(s -> DynamicTest.dynamicTest(s.name(), () -> runRace(s.name(), s.setup(), s.finalCount())));+ }++ private void runRace(String name, RaceSetup setup, ToIntFunction<IamService> finalCount) throws Exception {+ ExecutorService pool = Executors.newFixedThreadPool(THREADS);+ try {+ for (int trial = 0; trial < TRIALS; trial++) {+ IamService iam = newIamService();+ IntConsumer op = setup.prepare(iam);++ CountDownLatch start = new CountDownLatch(1);+ CountDownLatch done = new CountDownLatch(N);+ List<Throwable> errors = new CopyOnWriteArrayList<>();++ for (int i = 0; i < N; i++) {+ int idx = i;+ pool.submit(() -> {+ try {+ start.await();+ op.accept(idx);+ } catch (Throwable t) {+ errors.add(t);+ } finally {+ done.countDown();+ }+ });+ }+ start.countDown();+ assertTrue(done.await(30, TimeUnit.SECONDS), name + ": workers stalled (HashMap CPU spin)");++ int actual;+ try {+ actual = finalCount.applyAsInt(iam);+ } catch (Throwable t) {+ fail(name + " trial " + trial + ": reading final state threw (corrupted collection): " + t);+ return;+ }+ if (actual != N || !errors.isEmpty()) {+ fail(name + " trial " + trial + ": expected " + N + " entries, got " + actual+ + "; mutation errors=" + errors.size()+ + (errors.isEmpty() ? "" : " e.g. " + errors.get(0)));+ }+ }+ } finally {+ pool.shutdownNow();+ }+ }++ @Test+ void policyVersionsStayRaceFree() throws Exception {+ int trials = 400;+ int extraVersions = 4; // v1 is seeded by createPolicy; cap is 5+ ExecutorService pool = Executors.newFixedThreadPool(extraVersions);+ try {+ for (int trial = 0; trial < trials; trial++) {+ IamService iam = newIamService();+ IamPolicy p = iam.createPolicy("pol", "/", null, "{}", null);+ String arn = p.getArn();++ CountDownLatch start = new CountDownLatch(1);+ CountDownLatch done = new CountDownLatch(extraVersions);+ List<Throwable> errors = new CopyOnWriteArrayList<>();+ for (int i = 0; i < extraVersions; i++) {+ pool.submit(() -> {+ try {+ start.await();+ iam.createPolicyVersion(arn, "{\"v\":true}", false);+ } catch (Throwable t) {+ errors.add(t);+ } finally {+ done.countDown();+ }+ });+ }+ start.countDown();+ assertTrue(done.await(30, TimeUnit.SECONDS), "version workers stalled");++ List<?> versions = iam.listPolicyVersions(arn);+ Set<String> ids = new HashSet<>();+ iam.getPolicy(arn).getVersions().keySet().forEach(ids::add);++ if (!errors.isEmpty() || versions.size() != 1 + extraVersions || ids.size() != 1 + extraVersions) {+ fail("policy.versions trial " + trial + ": expected " + (1 + extraVersions)+ + " distinct versions, got list=" + versions.size() + " distinctIds=" + ids.size()+ + "; errors=" + errors.size() + (errors.isEmpty() ? "" : " e.g. " + errors.get(0)));+ }+ }+ } finally {+ pool.shutdownNow();+ }+ }++ @Test+ void instanceProfileRoleNamesStayRaceFree() throws Exception {+ int trials = 400;+ int racers = 8;+ ExecutorService pool = Executors.newFixedThreadPool(racers);+ try {+ for (int trial = 0; trial < trials; trial++) {+ IamService iam = newIamService();+ iam.createInstanceProfile("prof", "/");+ for (int i = 0; i < racers; i++) iam.createRole("role" + i, "/", "{}", null, 3600, null);++ CountDownLatch start = new CountDownLatch(1);+ CountDownLatch done = new CountDownLatch(racers);+ List<Throwable> unexpected = new CopyOnWriteArrayList<>();+ for (int i = 0; i < racers; i++) {+ int idx = i;+ pool.submit(() -> {+ try {+ start.await();+ iam.addRoleToInstanceProfile("prof", "role" + idx);+ } catch (io.github.hectorvent.floci.core.common.AwsException expected) {+ // LimitExceeded once one role is in+ } catch (Throwable t) {+ unexpected.add(t);+ } finally {+ done.countDown();+ }+ });+ }+ start.countDown();+ assertTrue(done.await(30, TimeUnit.SECONDS), "profile workers stalled");++ int roles = iam.getInstanceProfile("prof").getRoleNames().size();+ if (roles != 1 || !unexpected.isEmpty()) {+ fail("instanceProfile.roleNames trial " + trial + ": cap violated/corrupted, roleNames="+ + roles + " unexpectedErrors=" + unexpected.size());+ }+ }+ } finally {+ pool.shutdownNow();+ }+ }++ private static String[] createPolicies(IamService iam, String prefix) {+ String[] arns = new String[N];+ for (int i = 0; i < N; i++) {+ arns[i] = iam.createPolicy(prefix + "-" + i, "/", null, "{}", null).getArn();+ }+ return arns;+ }++ private static String onlyPolicyArn(IamService iam) {+ return iam.listPolicies(null, null).get(0).getArn();+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/iam/IamRolePolicyConcurrencyTest.java b/src/test/java/io/github/hectorvent/floci/services/iam/IamRolePolicyConcurrencyTest.javanew file mode 100644index 0000000000..b1a1143972--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/iam/IamRolePolicyConcurrencyTest.java@@ -0,0 +1,114 @@+package io.github.hectorvent.floci.services.iam;++import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.storage.InMemoryStorage;+import org.junit.jupiter.api.Test;++import java.util.Collections;+import java.util.List;+import java.util.concurrent.CopyOnWriteArrayList;+import java.util.concurrent.CountDownLatch;+import java.util.concurrent.ExecutorService;+import java.util.concurrent.Executors;+import java.util.concurrent.TimeUnit;++import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.junit.jupiter.api.Assertions.fail;++/**+ * Concurrent PutRolePolicy on the same role must not lose or corrupt inline policies:+ * a policy that PutRolePolicy acked has to stay visible to a later GetRolePolicy.+ */+class IamRolePolicyConcurrencyTest {++ private static IamService newIamService() {+ return new IamService(+ new InMemoryStorage<>(),+ new InMemoryStorage<>(),+ new InMemoryStorage<>(),+ new InMemoryStorage<>(),+ new InMemoryStorage<>(),+ new InMemoryStorage<>(),+ new InMemoryStorage<>(),+ new RegionResolver("eu-central-1", "000000000000"),+ false+ );+ }++ @Test+ void concurrentPutRolePolicyOnSameRoleStaysReadable() throws Exception {+ int trials = 300;+ int policiesPerTrial = 32;+ int threads = 8;+ ExecutorService pool = Executors.newFixedThreadPool(threads);++ try {+ for (int trial = 0; trial < trials; trial++) {+ IamService iam = newIamService();+ String roleName = "repro-role-" + trial;+ iam.createRole(roleName, "/", "{}", null, 3600, null);++ CountDownLatch start = new CountDownLatch(1);+ CountDownLatch done = new CountDownLatch(policiesPerTrial);+ List<String> lostOnReadBack = new CopyOnWriteArrayList<>();+ List<Throwable> corruption = new CopyOnWriteArrayList<>();++ for (int i = 0; i < policiesPerTrial; i++) {+ String policyName = "inline-" + i;+ String doc = "{\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"secretsmanager:GetSecretValue\","+ + "\"Resource\":\"*\",\"Sid\":\"" + policyName + "\"}]}";+ pool.submit(() -> {+ try {+ start.await();+ iam.putRolePolicy(roleName, policyName, doc);+ try {+ iam.getRolePolicy(roleName, policyName);+ } catch (AwsException e) {+ lostOnReadBack.add(policyName);+ }+ } catch (Throwable t) {+ corruption.add(t);+ } finally {+ done.countDown();+ }+ });+ }++ start.countDown();+ assertTrue(done.await(30, TimeUnit.SECONDS),+ "trial " + trial + ": workers stalled (possible HashMap CPU spin under concurrent resize)");++ int finalCount;+ try {+ finalCount = iam.getRole(roleName).getInlinePolicies().size();+ } catch (Throwable t) {+ finalCount = -1;+ corruption.add(t);+ }++ if (!lostOnReadBack.isEmpty() || !corruption.isEmpty() || finalCount != policiesPerTrial) {+ System.out.println("[repro] FAILED on trial " + trial);+ System.out.println("[repro] NoSuchEntity on read-back : " + lostOnReadBack.size()+ + " " + sorted(lostOnReadBack));+ System.out.println("[repro] corruption exceptions : " + corruption.size()+ + (corruption.isEmpty() ? "" : " e.g. " + corruption.get(0)));+ System.out.println("[repro] inline policies retained : " + finalCount + " / " + policiesPerTrial);+ fail("Concurrent PutRolePolicy on one role lost/corrupted inline policies "+ + "(unsynchronized read-modify-write in IamService.putRolePolicy). "+ + "NoSuchEntity reads=" + sorted(lostOnReadBack)+ + ", retained=" + finalCount + "/" + policiesPerTrial+ + ", corruption=" + corruption.size());+ }+ }+ } finally {+ pool.shutdownNow();+ }+ }++ private static List<String> sorted(List<String> in) {+ List<String> copy = new java.util.ArrayList<>(in);+ Collections.sort(copy);+ return copy;+ }+}