Light, fluffy, and always free - The AWS Local Emulator alternative
feat(glue): implement partition indexes (#3551)
Ovidiu (Ovi)38d1d0a8e4a5
diff --git a/docs/services/glue.md b/docs/services/glue.mdindex 2f5743d119..e11bb7f0a0 100644--- a/docs/services/glue.md+++ b/docs/services/glue.md@@ -33,6 +33,18 @@ Floci emulates the AWS Glue Data Catalog and Glue Schema Registry, allowing you|--------|-------------|| CreatePartition | Creates a partition for a Data Catalog table. || GetPartitions | Lists partitions stored for a Data Catalog table. |+| CreatePartitionIndex | Registers a partition index on a table. Keys must name partition columns, and a table holds at most 3 indexes. The index reports `CREATING` before `ACTIVE`. |+| GetPartitionIndexes | Lists a table's partition indexes, each with its keys resolved to name and type. |+| DeletePartitionIndex | Removes a partition index from a table. The index reports `DELETING` before it disappears. |++Partition indexes carry the AWS lifecycle. A new index reports `CREATING` and then `ACTIVE`; a+deleted one reports `DELETING` and then disappears. As on AWS, only one index per table may be+created or deleted at a time, and an index that is still `CREATING` cannot be deleted yet.++The transitions are driven by reads rather than by a timer, so they are deterministic: each+`GetPartitionIndexes` reports the current state and settles it, and a client that polls (as the+Terraform provider does) converges on its next call. The `FAILED` state is not emulated, since it+only arises from a backfill failure.#### User-defined Functionsdiff --git a/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.javaindex f7e46efec3..273c97ddb8 100644--- a/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java+++ b/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java@@ -84,15 +84,29 @@ public class GlueJsonHandler {Table table = glueService.getTable(dbName, tableName);yield Response.ok(Map.of("Table", table)).build();}+ case "CreatePartitionIndex" -> {+ String dbName = request.get("DatabaseName").asText();+ String tableName = request.get("TableName").asText();+ PartitionIndex index = mapper.treeToValue(request.get("PartitionIndex"), PartitionIndex.class);+ glueService.createPartitionIndex(dbName, tableName, index);+ // AWS answers with an empty body, as it does for CreateTable above.+ yield Response.ok().build();+ }+ case "DeletePartitionIndex" -> {+ String dbName = request.get("DatabaseName").asText();+ String tableName = request.get("TableName").asText();+ String indexName = request.get("IndexName").asText();+ glueService.deletePartitionIndex(dbName, tableName, indexName);+ yield Response.ok().build();+ }case "GetPartitionIndexes" -> {String dbName = request.get("DatabaseName").asText();String tableName = request.get("TableName").asText();- // Resolve the table so a missing one is reported as such rather than as an empty- // index list. No index can exist until CreatePartitionIndex is supported, so the- // list is empty for every table that does resolve - which is the answer a client- // reading a table's indexes needs, rather than an unsupported-action failure.- glueService.getTable(dbName, tableName);- yield Response.ok(Map.of("PartitionIndexDescriptorList", List.of())).build();+ // The service resolves the table first, so a missing one is reported as such+ // rather than as an empty index list.+ yield Response.ok(Map.of(+ "PartitionIndexDescriptorList",+ glueService.getPartitionIndexes(dbName, tableName))).build();}case "GetTables" -> {String dbName = request.get("DatabaseName").asText();diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java b/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.javaindex ae48d67ae9..45d706e334 100644--- a/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java+++ b/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java@@ -12,7 +12,10 @@ import io.github.hectorvent.floci.services.glue.model.CrawlerTargets;import io.github.hectorvent.floci.services.glue.model.Database;import io.github.hectorvent.floci.services.glue.model.Job;import io.github.hectorvent.floci.services.glue.model.JobUpdate;+import io.github.hectorvent.floci.services.glue.model.KeySchemaElement;import io.github.hectorvent.floci.services.glue.model.Partition;+import io.github.hectorvent.floci.services.glue.model.PartitionIndex;+import io.github.hectorvent.floci.services.glue.model.PartitionIndexDescriptor;import io.github.hectorvent.floci.services.glue.model.SchemaReference;import io.github.hectorvent.floci.services.glue.model.StorageDescriptor;import io.github.hectorvent.floci.services.glue.model.Table;@@ -59,11 +62,21 @@ public class GlueService {"\\s*([A-Za-z_][A-Za-z0-9_]*)\\s+in\\s*\\((.*)\\)\\s*",Pattern.CASE_INSENSITIVE);+ /** Measured against real Glue (us-west-2): a fourth index reports the limit. */+ private static final int MAX_PARTITION_INDEXES_PER_TABLE = 3;++ // Glue's partition index states. FAILED also exists but is only reachable through a backfill+ // failure, which is not emulated.+ private static final String INDEX_STATUS_CREATING = "CREATING";+ private static final String INDEX_STATUS_ACTIVE = "ACTIVE";+ private static final String INDEX_STATUS_DELETING = "DELETING";+private final StorageBackend<String, Database> databaseStore;private final StorageBackend<String, Table> tableStore;private final StorageBackend<String, Table> tableVersionStore;private final StorageBackend<String, Map<String, Object>> columnStatisticsStore;private final StorageBackend<String, Partition> partitionStore;+ private final StorageBackend<String, PartitionIndexDescriptor> partitionIndexStore;private final StorageBackend<String, Map<String, Object>> partitionColumnStatisticsStore;private final StorageBackend<String, UserDefinedFunction> functionStore;private final StorageBackend<String, Job> jobStore;@@ -82,6 +95,8 @@ public class GlueService {this.tableVersionStore = storageFactory.create("glue", "table_versions.json", new TypeReference<>() {});this.columnStatisticsStore = storageFactory.create("glue", "column_statistics.json", new TypeReference<>() {});this.partitionStore = storageFactory.create("glue", "partitions.json", new TypeReference<>() {});+ this.partitionIndexStore = storageFactory.create(+ "glue", "partition_indexes.json", new TypeReference<>() {});this.partitionColumnStatisticsStore = storageFactory.create("glue", "partition_column_statistics.json", new TypeReference<>() {});this.functionStore = storageFactory.create("glue", "functions.json", new TypeReference<>() {});@@ -97,6 +112,7 @@ public class GlueService {StorageBackend<String, Table> tableVersionStore,StorageBackend<String, Map<String, Object>> columnStatisticsStore,StorageBackend<String, Partition> partitionStore,+ StorageBackend<String, PartitionIndexDescriptor> partitionIndexStore,StorageBackend<String, Map<String, Object>> partitionColumnStatisticsStore,StorageBackend<String, UserDefinedFunction> functionStore,StorageBackend<String, Job> jobStore,@@ -109,6 +125,7 @@ public class GlueService {this.tableVersionStore = tableVersionStore;this.columnStatisticsStore = columnStatisticsStore;this.partitionStore = partitionStore;+ this.partitionIndexStore = partitionIndexStore;this.partitionColumnStatisticsStore = partitionColumnStatisticsStore;this.functionStore = functionStore;this.jobStore = jobStore;@@ -281,6 +298,9 @@ public class GlueService {partitionColumnStatisticsStore.keys().stream().filter(statisticsKey -> statisticsKey.startsWith(key + ":")).forEach(partitionColumnStatisticsStore::delete);+ partitionIndexStore.keys().stream()+ .filter(indexKey -> indexKey.startsWith(key + ":"))+ .forEach(partitionIndexStore::delete);LOG.infov("Deleted Glue Table: {0}.{1}", databaseName, tableName);}@@ -425,6 +445,158 @@ public class GlueService {.toList();}+ /**+ * Registers a partition index on a table.+ *+ * <p>The index is stored {@code CREATING}, as real Glue reports it while the backfill runs. It+ * settles to {@code ACTIVE} on the next {@code GetPartitionIndexes}, so a client that polls+ * sees the transition without the emulator depending on elapsed time.+ *+ * <p>Synchronized with the other partition-index methods, and with {@code updateTable}: the cap+ * and duplicate checks read the stored indexes before writing, so concurrent creates would+ * otherwise both pass a check that only one of them should.+ */+ public synchronized void createPartitionIndex(String databaseName, String tableName, PartitionIndex index) {+ Table table = getTable(databaseName, tableName);++ if (index == null || index.getIndexName() == null || index.getIndexName().isBlank()) {+ throw new AwsException("InvalidInputException", "IndexName is required", 400);+ }+ List<String> keyNames = index.getKeys();+ if (keyNames == null || keyNames.isEmpty()) {+ throw new AwsException("InvalidInputException", "Keys is required", 400);+ }++ // Every key must name one of the table's partition keys: an index exists to narrow a+ // partition scan, so a key outside that set could never be used.+ List<KeySchemaElement> resolvedKeys = new ArrayList<>();+ for (String keyName : keyNames) {+ int position = partitionKeyIndex(table, keyName);+ if (position < 0) {+ throw new AwsException("InvalidInputException",+ "IndexKeys not a part of PartitionColumns. Verify the indexKeys : [" + keyName + "]", 400);+ }+ Column partitionKey = table.getPartitionKeys().get(position);+ resolvedKeys.add(new KeySchemaElement(partitionKey.getName(), partitionKey.getType()));+ }++ // Measured against real Glue (us-west-2): the cap is evaluated before the duplicate+ // checks, so a create that is both over the limit and a duplicate reports the limit.+ List<PartitionIndexDescriptor> existing = readPartitionIndexes(databaseName, tableName);+ requireNoIndexInProgress(existing);+ if (existing.size() >= MAX_PARTITION_INDEXES_PER_TABLE) {+ throw new AwsException("ResourceNumberLimitExceededException",+ "Partition index limit exceeded. Maximum: " + MAX_PARTITION_INDEXES_PER_TABLE, 400);+ }++ String key = partitionIndexKey(databaseName, tableName, index.getIndexName());+ if (partitionIndexStore.get(key).isPresent()) {+ throw new AwsException("AlreadyExistsException",+ "Partition Index " + index.getIndexName() + " already exists with the same name.", 400);+ }++ // Glue also refuses a second index over the same keys under a different name. The+ // comparison is order sensitive: [year, month] and [month, year] are distinct indexes.+ List<String> resolvedKeyNames = resolvedKeys.stream().map(KeySchemaElement::getName).toList();+ for (PartitionIndexDescriptor other : existing) {+ List<String> otherKeyNames = other.getKeys() == null+ ? List.of()+ : other.getKeys().stream().map(KeySchemaElement::getName).toList();+ if (otherKeyNames.equals(resolvedKeyNames)) {+ throw new AwsException("AlreadyExistsException",+ "Partition Index " + other.getIndexName() + " already exists with the same keys.", 400);+ }+ }++ PartitionIndexDescriptor descriptor = new PartitionIndexDescriptor();+ descriptor.setIndexName(index.getIndexName());+ descriptor.setIndexStatus(INDEX_STATUS_CREATING);+ descriptor.setKeys(resolvedKeys);+ partitionIndexStore.put(key, descriptor);+ LOG.infov("Created Glue partition index: {0}.{1} {2}", databaseName, tableName, index.getIndexName());+ }++ public synchronized void deletePartitionIndex(String databaseName, String tableName, String indexName) {+ getTable(databaseName, tableName);+ if (indexName == null || indexName.isBlank()) {+ throw new AwsException("InvalidInputException", "IndexName is required", 400);+ }+ String key = partitionIndexKey(databaseName, tableName, indexName);+ PartitionIndexDescriptor target = partitionIndexStore.get(key).orElse(null);+ // An index still being created is not addressable by name yet: real Glue reports it as+ // absent even while GetPartitionIndexes lists it as CREATING (measured in isolation).+ if (target == null || INDEX_STATUS_CREATING.equals(target.getIndexStatus())) {+ throw new AwsException("EntityNotFoundException",+ "Index with the given indexName : " + indexName + " does not exist.", 400);+ }+ if (INDEX_STATUS_DELETING.equals(target.getIndexStatus())) {+ throw new AwsException("EntityNotFoundException",+ "Index with the given indexName : " + indexName + " does not exist.", 400);+ }+ requireNoIndexInProgress(readPartitionIndexes(databaseName, tableName));++ // Deletion is observable: the index reports DELETING before it disappears.+ target.setIndexStatus(INDEX_STATUS_DELETING);+ partitionIndexStore.put(key, target);+ LOG.infov("Deleting Glue partition index: {0}.{1} {2}", databaseName, tableName, indexName);+ }++ /**+ * Returns a table's partition indexes and then advances any that are mid-lifecycle.+ *+ * <p>Real Glue creates and deletes an index asynchronously, so a caller sees {@code CREATING}+ * before {@code ACTIVE} and {@code DELETING} before the index disappears. Rather than tie those+ * transitions to wall-clock time, which would make tests racy, each read reports the current+ * state and settles it: the next read sees the outcome. A client that polls for {@code ACTIVE}+ * or for the index to vanish, as the Terraform provider does, converges on its second read.+ */+ public synchronized List<PartitionIndexDescriptor> getPartitionIndexes(String databaseName, String tableName) {+ List<PartitionIndexDescriptor> current = readPartitionIndexes(databaseName, tableName);+ for (PartitionIndexDescriptor descriptor : current) {+ String key = partitionIndexKey(databaseName, tableName, descriptor.getIndexName());+ if (INDEX_STATUS_CREATING.equals(descriptor.getIndexStatus())) {+ PartitionIndexDescriptor settled = copyWithStatus(descriptor, INDEX_STATUS_ACTIVE);+ partitionIndexStore.put(key, settled);+ } else if (INDEX_STATUS_DELETING.equals(descriptor.getIndexStatus())) {+ partitionIndexStore.delete(key);+ }+ }+ return current;+ }++ /** Reads the stored indexes without advancing the lifecycle. */+ private List<PartitionIndexDescriptor> readPartitionIndexes(String databaseName, String tableName) {+ getTable(databaseName, tableName);+ String prefix = tableKey(databaseName, tableName) + ":";+ return partitionIndexStore.scan(k -> k.startsWith(prefix)).stream()+ .sorted(Comparator.comparing(PartitionIndexDescriptor::getIndexName))+ .toList();+ }++ private static PartitionIndexDescriptor copyWithStatus(PartitionIndexDescriptor source, String status) {+ PartitionIndexDescriptor copy = new PartitionIndexDescriptor();+ copy.setIndexName(source.getIndexName());+ copy.setIndexStatus(status);+ copy.setKeys(source.getKeys());+ copy.setBackfillErrors(source.getBackfillErrors());+ return copy;+ }++ /**+ * Glue allows one index per table to be created or deleted at a time. Measured message, with+ * the offending index and its state named.+ */+ private static void requireNoIndexInProgress(List<PartitionIndexDescriptor> existing) {+ for (PartitionIndexDescriptor descriptor : existing) {+ String status = descriptor.getIndexStatus();+ if (INDEX_STATUS_CREATING.equals(status) || INDEX_STATUS_DELETING.equals(status)) {+ throw new AwsException("ResourceNumberLimitExceededException",+ "Index " + descriptor.getIndexName() + " is in " + status+ + " state. Only 1 index can be created or deleted simultaneously per table.", 400);+ }+ }+ }+// The partition store scan returns values in storage iteration order, which is not stable.// Return partitions ordered by their values so GetPartitions is deterministic across calls.private static int comparePartitionValues(Partition a, Partition b) {@@ -624,6 +796,10 @@ public class GlueService {return tableKey(databaseName, tableName) + ":" + normalizeName(columnName);}+ private static String partitionIndexKey(String databaseName, String tableName, String indexName) {+ return tableKey(databaseName, tableName) + ":" + normalizeName(indexName);+ }+private static String partitionKey(String databaseName, String tableName, List<String> partitionValues) {return tableKey(databaseName, tableName) + ":" + String.join(":", partitionValues.stream().map(GlueService::encodePartitionValue)diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/KeySchemaElement.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/KeySchemaElement.javanew file mode 100644index 0000000000..6923baaf1b--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/KeySchemaElement.java@@ -0,0 +1,24 @@+package io.github.hectorvent.floci.services.glue.model;++import com.fasterxml.jackson.annotation.JsonProperty;+import io.quarkus.runtime.annotations.RegisterForReflection;++/** One key of a partition index, as returned by {@code GetPartitionIndexes}. */+@RegisterForReflection+public class KeySchemaElement {+ @JsonProperty("Name")+ private String name;+ @JsonProperty("Type")+ private String type;++ public KeySchemaElement() {}+ public KeySchemaElement(String name, String type) {+ this.name = name;+ this.type = type;+ }++ public String getName() { return name; }+ public void setName(String name) { this.name = name; }+ public String getType() { return type; }+ public void setType(String type) { this.type = type; }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/PartitionIndex.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/PartitionIndex.javanew file mode 100644index 0000000000..7764c0090b--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/PartitionIndex.java@@ -0,0 +1,28 @@+package io.github.hectorvent.floci.services.glue.model;++import com.fasterxml.jackson.annotation.JsonProperty;+import io.quarkus.runtime.annotations.RegisterForReflection;++import java.util.ArrayList;+import java.util.List;++/**+ * The partition index as supplied to {@code CreatePartitionIndex}.+ *+ * <p>Note the asymmetry with {@link PartitionIndexDescriptor}: a request carries key <em>names</em>+ * only, while a read returns each key's name and type, resolved from the table's partition keys.+ */+@RegisterForReflection+public class PartitionIndex {+ @JsonProperty("IndexName")+ private String indexName;+ @JsonProperty("Keys")+ private List<String> keys;++ public PartitionIndex() {}++ public String getIndexName() { return indexName; }+ public void setIndexName(String indexName) { this.indexName = indexName; }+ public List<String> getKeys() { return keys == null ? null : new ArrayList<>(keys); }+ public void setKeys(List<String> keys) { this.keys = keys == null ? null : new ArrayList<>(keys); }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/PartitionIndexDescriptor.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/PartitionIndexDescriptor.javanew file mode 100644index 0000000000..4e6caa257c--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/PartitionIndexDescriptor.java@@ -0,0 +1,43 @@+package io.github.hectorvent.floci.services.glue.model;++import com.fasterxml.jackson.annotation.JsonInclude;+import com.fasterxml.jackson.annotation.JsonProperty;+import io.quarkus.runtime.annotations.RegisterForReflection;++import java.util.ArrayList;+import java.util.List;++/**+ * A stored partition index, as returned by {@code GetPartitionIndexes}.+ *+ * <p>{@code IndexStatus} follows the real lifecycle, advanced by reads rather than by time: an+ * index is stored {@code CREATING} and becomes {@code ACTIVE} on the next+ * {@code GetPartitionIndexes}; a deleted one is stored {@code DELETING} and disappears on that+ * same read. Real Glue backfills asynchronously, so a client that polls for {@code ACTIVE} (the+ * Terraform provider does) sees the transition it expects, without the emulator needing a clock.+ */+@RegisterForReflection+@JsonInclude(JsonInclude.Include.NON_NULL)+public class PartitionIndexDescriptor {+ @JsonProperty("IndexName")+ private String indexName;+ @JsonProperty("IndexStatus")+ private String indexStatus;+ @JsonProperty("Keys")+ private List<KeySchemaElement> keys;+ @JsonProperty("BackfillErrors")+ private List<Object> backfillErrors;++ public PartitionIndexDescriptor() {}++ public String getIndexName() { return indexName; }+ public void setIndexName(String indexName) { this.indexName = indexName; }+ public String getIndexStatus() { return indexStatus; }+ public void setIndexStatus(String indexStatus) { this.indexStatus = indexStatus; }+ public List<KeySchemaElement> getKeys() { return keys == null ? null : new ArrayList<>(keys); }+ public void setKeys(List<KeySchemaElement> keys) { this.keys = keys == null ? null : new ArrayList<>(keys); }+ public List<Object> getBackfillErrors() { return backfillErrors == null ? null : new ArrayList<>(backfillErrors); }+ public void setBackfillErrors(List<Object> backfillErrors) {+ this.backfillErrors = backfillErrors == null ? null : new ArrayList<>(backfillErrors);+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerTest.javaindex cdc0bf0d5e..8e33bd28cc 100644--- a/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerTest.java@@ -2,7 +2,9 @@ package io.github.hectorvent.floci.services.glue;import com.fasterxml.jackson.databind.JsonNode;import com.fasterxml.jackson.databind.ObjectMapper;+import com.fasterxml.jackson.databind.node.ArrayNode;import com.fasterxml.jackson.databind.node.ObjectNode;+import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.RegionResolver;import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;import io.github.hectorvent.floci.core.storage.StorageFactory;@@ -13,6 +15,7 @@ import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;import com.fasterxml.jackson.core.type.TypeReference;+import java.util.List;import java.util.Map;import static org.junit.jupiter.api.Assertions.assertEquals;@@ -71,7 +74,7 @@ class GlueJsonHandlerTest {@SuppressWarnings("unchecked")Map<String, Object> body = (Map<String, Object>) response.getEntity();assertTrue(body.containsKey("PartitionIndexDescriptorList"));- assertTrue(((java.util.List<?>) body.get("PartitionIndexDescriptorList")).isEmpty());+ assertTrue(((List<?>) body.get("PartitionIndexDescriptorList")).isEmpty());}/** A missing table is reported as missing, not as a table that happens to have no indexes. */@@ -86,6 +89,241 @@ class GlueJsonHandlerTest {assertThrows(Exception.class, () -> handler.handle("GetPartitionIndexes", request, REGION));}+ /** Creates a table whose partition keys are the given names, all typed {@code string}. */+ private void createPartitionedTable(String dbName, String tableName, String... partitionKeys) throws Exception {+ ObjectNode createDb = mapper.createObjectNode();+ createDb.putObject("DatabaseInput").put("Name", dbName);+ try {+ handler.handle("CreateDatabase", createDb, REGION);+ } catch (AwsException alreadyExists) {+ // The helper is called twice in the drop-and-recreate case; the database survives.+ }++ ObjectNode createTable = mapper.createObjectNode();+ createTable.put("DatabaseName", dbName);+ ObjectNode tableInput = createTable.putObject("TableInput");+ tableInput.put("Name", tableName);+ ArrayNode keys = tableInput.putArray("PartitionKeys");+ for (String partitionKey : partitionKeys) {+ keys.addObject().put("Name", partitionKey).put("Type", "string");+ }+ assertEquals(200, handler.handle("CreateTable", createTable, REGION).getStatus());+ }++ private Response createIndex(String dbName, String tableName, String indexName, String... keys) throws Exception {+ ObjectNode request = mapper.createObjectNode();+ request.put("DatabaseName", dbName);+ request.put("TableName", tableName);+ ObjectNode index = request.putObject("PartitionIndex");+ index.put("IndexName", indexName);+ ArrayNode keyArray = index.putArray("Keys");+ for (String key : keys) {+ keyArray.add(key);+ }+ return handler.handle("CreatePartitionIndex", request, REGION);+ }++ private Response deleteIndex(String dbName, String tableName, String indexName) throws Exception {+ ObjectNode request = mapper.createObjectNode();+ request.put("DatabaseName", dbName);+ request.put("TableName", tableName);+ request.put("IndexName", indexName);+ return handler.handle("DeletePartitionIndex", request, REGION);+ }++ @SuppressWarnings("unchecked")+ private List<Map<String, Object>> listIndexes(String dbName, String tableName) throws Exception {+ ObjectNode request = mapper.createObjectNode();+ request.put("DatabaseName", dbName);+ request.put("TableName", tableName);+ Response response = handler.handle("GetPartitionIndexes", request, REGION);+ assertEquals(200, response.getStatus());+ Map<String, Object> body = (Map<String, Object>) response.getEntity();+ return mapper.convertValue(+ body.get("PartitionIndexDescriptorList"), new TypeReference<List<Map<String, Object>>>() {});+ }++ /** Reads once to settle any in-progress index, mirroring a client that polls to ACTIVE. */+ private void settleIndexes(String dbName, String tableName) throws Exception {+ listIndexes(dbName, tableName);+ }++ private static String statusOf(List<Map<String, Object>> indexes, String indexName) {+ return indexes.stream()+ .filter(index -> indexName.equals(index.get("IndexName")))+ .map(index -> (String) index.get("IndexStatus"))+ .findFirst()+ .orElse(null);+ }++ // ── Partition indexes ──────────────────────────────────────────+ //+ // Behaviour below was captured against real Glue in us-west-2 on a table partitioned by+ // tenant/year/month/day, so the codes, messages and lifecycle states are AWS's own.++ @Test+ void aNewIndexReportsCreatingAndThenBecomesActive() throws Exception {+ createPartitionedTable("idx_db", "events", "tenant", "year", "month", "day");+ assertEquals(200, createIndex("idx_db", "events", "by_tenant", "tenant").getStatus());++ // Real Glue reports CREATING until the backfill finishes, then ACTIVE.+ List<Map<String, Object>> whileCreating = listIndexes("idx_db", "events");+ assertEquals("CREATING", statusOf(whileCreating, "by_tenant"));++ List<Map<String, Object>> settled = listIndexes("idx_db", "events");+ assertEquals("ACTIVE", statusOf(settled, "by_tenant"));+ }++ @Test+ void aReadResolvesEachKeyToItsNameAndType() throws Exception {+ createPartitionedTable("idx_db_keys_shape", "events", "tenant", "year");+ createIndex("idx_db_keys_shape", "events", "by_tenant", "tenant");++ List<Map<String, Object>> indexes = listIndexes("idx_db_keys_shape", "events");+ assertEquals(1, indexes.size());+ assertEquals("by_tenant", indexes.get(0).get("IndexName"));+ @SuppressWarnings("unchecked")+ List<Map<String, Object>> keys = (List<Map<String, Object>>) indexes.get(0).get("Keys");+ // A request carries key names only; the read resolves each one's type from the table.+ assertEquals("tenant", keys.get(0).get("Name"));+ assertEquals("string", keys.get(0).get("Type"));+ }++ @Test+ void aDeletedIndexReportsDeletingAndThenDisappears() throws Exception {+ createPartitionedTable("idx_db_del", "events", "tenant", "year");+ createIndex("idx_db_del", "events", "by_tenant", "tenant");+ settleIndexes("idx_db_del", "events");++ assertEquals(200, deleteIndex("idx_db_del", "events", "by_tenant").getStatus());++ List<Map<String, Object>> whileDeleting = listIndexes("idx_db_del", "events");+ assertEquals("DELETING", statusOf(whileDeleting, "by_tenant"));++ assertTrue(listIndexes("idx_db_del", "events").isEmpty());+ }++ @Test+ void aSecondIndexCannotBeCreatedWhileOneIsCreating() throws Exception {+ createPartitionedTable("idx_db_busy", "events", "tenant", "year");+ createIndex("idx_db_busy", "events", "first", "tenant");++ AwsException ex = assertThrows(AwsException.class,+ () -> createIndex("idx_db_busy", "events", "second", "year"));+ assertEquals("ResourceNumberLimitExceededException", ex.getErrorCode());+ assertTrue(ex.getMessage().contains("is in CREATING state"));+ }++ @Test+ void anIndexCannotBeCreatedWhileAnotherIsDeleting() throws Exception {+ createPartitionedTable("idx_db_busy2", "events", "tenant", "year");+ createIndex("idx_db_busy2", "events", "first", "tenant");+ settleIndexes("idx_db_busy2", "events");+ deleteIndex("idx_db_busy2", "events", "first");++ AwsException ex = assertThrows(AwsException.class,+ () -> createIndex("idx_db_busy2", "events", "second", "year"));+ assertEquals("ResourceNumberLimitExceededException", ex.getErrorCode());+ assertTrue(ex.getMessage().contains("is in DELETING state"));+ }++ @Test+ void anIndexStillCreatingCannotBeDeleted() throws Exception {+ createPartitionedTable("idx_db_earlydel", "events", "tenant");+ createIndex("idx_db_earlydel", "events", "by_tenant", "tenant");++ // Measured in isolation: Glue reports it absent even while a read lists it as CREATING.+ AwsException ex = assertThrows(AwsException.class,+ () -> deleteIndex("idx_db_earlydel", "events", "by_tenant"));+ assertEquals("EntityNotFoundException", ex.getErrorCode());+ }++ @Test+ void anIndexKeyMustBeOneOfTheTablesPartitionKeys() throws Exception {+ createPartitionedTable("idx_db_key", "events", "tenant", "year");++ AwsException ex = assertThrows(AwsException.class,+ () -> createIndex("idx_db_key", "events", "bad", "nosuchcolumn"));+ assertEquals("InvalidInputException", ex.getErrorCode());+ }++ @Test+ void anIndexNameCannotBeReused() throws Exception {+ createPartitionedTable("idx_db_dup", "events", "tenant", "year");+ createIndex("idx_db_dup", "events", "dup", "tenant");+ settleIndexes("idx_db_dup", "events");++ AwsException ex = assertThrows(AwsException.class,+ () -> createIndex("idx_db_dup", "events", "dup", "year"));+ assertEquals("AlreadyExistsException", ex.getErrorCode());+ assertTrue(ex.getMessage().contains("same name"));+ }++ @Test+ void aSecondIndexOverTheSameKeysIsRefusedEvenUnderANewName() throws Exception {+ createPartitionedTable("idx_db_keys", "events", "tenant", "year");+ createIndex("idx_db_keys", "events", "first", "tenant");+ settleIndexes("idx_db_keys", "events");++ AwsException ex = assertThrows(AwsException.class,+ () -> createIndex("idx_db_keys", "events", "second", "tenant"));+ assertEquals("AlreadyExistsException", ex.getErrorCode());+ assertTrue(ex.getMessage().contains("same keys"));+ }++ @Test+ void keyOrderDistinguishesTwoIndexes() throws Exception {+ createPartitionedTable("idx_db_order", "events", "tenant", "year", "month");++ // [year, month] and [month, year] are different indexes on real Glue, not duplicates.+ assertEquals(200, createIndex("idx_db_order", "events", "ord1", "year", "month").getStatus());+ settleIndexes("idx_db_order", "events");+ assertEquals(200, createIndex("idx_db_order", "events", "ord2", "month", "year").getStatus());+ settleIndexes("idx_db_order", "events");+ assertEquals(2, listIndexes("idx_db_order", "events").size());+ }++ @Test+ void aFourthIndexExceedsTheLimit() throws Exception {+ createPartitionedTable("idx_db_cap", "events", "tenant", "year", "month", "day");+ createIndex("idx_db_cap", "events", "i1", "tenant");+ settleIndexes("idx_db_cap", "events");+ createIndex("idx_db_cap", "events", "i2", "year");+ settleIndexes("idx_db_cap", "events");+ createIndex("idx_db_cap", "events", "i3", "month");+ settleIndexes("idx_db_cap", "events");++ AwsException ex = assertThrows(AwsException.class,+ () -> createIndex("idx_db_cap", "events", "i4", "day"));+ assertEquals("ResourceNumberLimitExceededException", ex.getErrorCode());+ assertTrue(ex.getMessage().contains("Maximum: 3"));+ }++ @Test+ void deletingAnUnknownIndexFails() throws Exception {+ createPartitionedTable("idx_db_missing", "events", "tenant");++ AwsException ex = assertThrows(AwsException.class,+ () -> deleteIndex("idx_db_missing", "events", "absent"));+ assertEquals("EntityNotFoundException", ex.getErrorCode());+ }++ @Test+ void droppingATableDropsItsIndexes() throws Exception {+ createPartitionedTable("idx_db_cascade", "events", "tenant");+ createIndex("idx_db_cascade", "events", "by_tenant", "tenant");+ settleIndexes("idx_db_cascade", "events");++ ObjectNode dropTable = mapper.createObjectNode();+ dropTable.put("DatabaseName", "idx_db_cascade");+ dropTable.put("Name", "events");+ assertEquals(200, handler.handle("DeleteTable", dropTable, REGION).getStatus());++ // Recreating the table must not resurrect the old index.+ createPartitionedTable("idx_db_cascade", "events", "tenant");+ assertTrue(listIndexes("idx_db_cascade", "events").isEmpty());+ }+@Testvoid createCrawlerWithScheduleSucceeds() throws Exception {ObjectNode request = mapper.createObjectNode();diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.javaindex 0f856c2f43..9bf4a9722e 100644--- a/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java@@ -15,6 +15,8 @@ import io.github.hectorvent.floci.services.glue.model.Job;import io.github.hectorvent.floci.services.glue.model.JobCommand;import io.github.hectorvent.floci.services.glue.model.JobUpdate;import io.github.hectorvent.floci.services.glue.model.Partition;+import io.github.hectorvent.floci.services.glue.model.PartitionIndex;+import io.github.hectorvent.floci.services.glue.model.PartitionIndexDescriptor;import io.github.hectorvent.floci.services.glue.model.S3Target;import io.github.hectorvent.floci.services.glue.model.SchemaReference;import io.github.hectorvent.floci.services.glue.model.StorageDescriptor;@@ -31,6 +33,8 @@ import java.time.Instant;import java.util.LinkedHashMap;import java.util.List;import java.util.Map;+import java.util.concurrent.TimeUnit;+import java.util.concurrent.CountDownLatch;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertNotNull;@@ -76,6 +80,7 @@ class GlueServiceTest {tableVersionStore,columnStatisticsStore,partitionStore,+ new InMemoryStorage<String, PartitionIndexDescriptor>(),new InMemoryStorage<String, Map<String, Object>>(),new InMemoryStorage<String, UserDefinedFunction>(),new InMemoryStorage<String, Job>(),@@ -1258,4 +1263,52 @@ class GlueServiceTest {AwsException udfTagEx = assertThrows(AwsException.class, () -> glueService.tagResource(fakeUdfArn, Map.of("k", "v"), REGION));assertEquals("EntityNotFoundException", udfTagEx.getErrorCode());}++ @Test+ void concurrentPartitionIndexCreatesStopAtTheCap() throws Exception {+ Table table = new Table();+ table.setName("indexed");+ StorageDescriptor sd = new StorageDescriptor();+ sd.setColumns(java.util.List.of(new Column("a", "string")));+ table.setStorageDescriptor(sd);+ table.setPartitionKeys(java.util.List.of(new Column("a", "string"), new Column("b", "string")));+ glueService.createTable("db1", table);++ // Two settled indexes leave exactly one slot under the cap of three.+ createSettledIndex("idx0", "a");+ createSettledIndex("idx1", "b");++ int attempts = 4;+ CountDownLatch start = new CountDownLatch(1);+ CountDownLatch done = new CountDownLatch(attempts);+ for (int i = 0; i < attempts; i++) {+ String indexName = "race" + i;+ Thread.ofVirtual().start(() -> {+ try {+ start.await();+ PartitionIndex index = new PartitionIndex();+ index.setIndexName(indexName);+ index.setKeys(java.util.List.of("a", "b"));+ glueService.createPartitionIndex("db1", "indexed", index);+ } catch (AwsException | InterruptedException expected) {+ // Only the create that takes the last slot succeeds; the rest are rejected.+ } finally {+ done.countDown();+ }+ });+ }+ start.countDown();+ assertTrue(done.await(10, TimeUnit.SECONDS), "partition index creates did not finish");++ assertEquals(3, glueService.getPartitionIndexes("db1", "indexed").size());+ }++ private void createSettledIndex(String indexName, String key) {+ PartitionIndex index = new PartitionIndex();+ index.setIndexName(indexName);+ index.setKeys(java.util.List.of(key));+ glueService.createPartitionIndex("db1", "indexed", index);+ glueService.getPartitionIndexes("db1", "indexed");+ }+}