Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(glue): implement partition indexes (#3551)

Ovidiu (Ovi)38d1d0a8e4a5
diff --git a/docs/services/glue.md b/docs/services/glue.md
index 2f5743d119..e11bb7f0a0 100644
--- a/docs/services/glue.md
+++ b/docs/services/glue.md
@@ -33,6 +33,18 @@ Floci emulates the AWS Glue Data Catalog and Glue Schema Registry, allowing you
|--------|-------------|
| CreatePartition | Creates a partition for a Data Catalog table. |
| GetPartitions | Lists partitions stored for a Data Catalog table. |
+| CreatePartitionIndex | Registers a partition index on a table. Keys must name partition columns, and a table holds at most 3 indexes. The index reports `CREATING` before `ACTIVE`. |
+| GetPartitionIndexes | Lists a table's partition indexes, each with its keys resolved to name and type. |
+| DeletePartitionIndex | Removes a partition index from a table. The index reports `DELETING` before it disappears. |
+
+Partition indexes carry the AWS lifecycle. A new index reports `CREATING` and then `ACTIVE`; a
+deleted one reports `DELETING` and then disappears. As on AWS, only one index per table may be
+created or deleted at a time, and an index that is still `CREATING` cannot be deleted yet.
+
+The transitions are driven by reads rather than by a timer, so they are deterministic: each
+`GetPartitionIndexes` reports the current state and settles it, and a client that polls (as the
+Terraform provider does) converges on its next call. The `FAILED` state is not emulated, since it
+only arises from a backfill failure.
#### User-defined Functions
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java
index f7e46efec3..273c97ddb8 100644
--- a/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java
@@ -84,15 +84,29 @@ public class GlueJsonHandler {
Table table = glueService.getTable(dbName, tableName);
yield Response.ok(Map.of("Table", table)).build();
}
+ case "CreatePartitionIndex" -> {
+ String dbName = request.get("DatabaseName").asText();
+ String tableName = request.get("TableName").asText();
+ PartitionIndex index = mapper.treeToValue(request.get("PartitionIndex"), PartitionIndex.class);
+ glueService.createPartitionIndex(dbName, tableName, index);
+ // AWS answers with an empty body, as it does for CreateTable above.
+ yield Response.ok().build();
+ }
+ case "DeletePartitionIndex" -> {
+ String dbName = request.get("DatabaseName").asText();
+ String tableName = request.get("TableName").asText();
+ String indexName = request.get("IndexName").asText();
+ glueService.deletePartitionIndex(dbName, tableName, indexName);
+ yield Response.ok().build();
+ }
case "GetPartitionIndexes" -> {
String dbName = request.get("DatabaseName").asText();
String tableName = request.get("TableName").asText();
- // Resolve the table so a missing one is reported as such rather than as an empty
- // index list. No index can exist until CreatePartitionIndex is supported, so the
- // list is empty for every table that does resolve - which is the answer a client
- // reading a table's indexes needs, rather than an unsupported-action failure.
- glueService.getTable(dbName, tableName);
- yield Response.ok(Map.of("PartitionIndexDescriptorList", List.of())).build();
+ // The service resolves the table first, so a missing one is reported as such
+ // rather than as an empty index list.
+ yield Response.ok(Map.of(
+ "PartitionIndexDescriptorList",
+ glueService.getPartitionIndexes(dbName, tableName))).build();
}
case "GetTables" -> {
String dbName = request.get("DatabaseName").asText();
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java b/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java
index ae48d67ae9..45d706e334 100644
--- a/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java
@@ -12,7 +12,10 @@ import io.github.hectorvent.floci.services.glue.model.CrawlerTargets;
import io.github.hectorvent.floci.services.glue.model.Database;
import io.github.hectorvent.floci.services.glue.model.Job;
import io.github.hectorvent.floci.services.glue.model.JobUpdate;
+import io.github.hectorvent.floci.services.glue.model.KeySchemaElement;
import io.github.hectorvent.floci.services.glue.model.Partition;
+import io.github.hectorvent.floci.services.glue.model.PartitionIndex;
+import io.github.hectorvent.floci.services.glue.model.PartitionIndexDescriptor;
import io.github.hectorvent.floci.services.glue.model.SchemaReference;
import io.github.hectorvent.floci.services.glue.model.StorageDescriptor;
import io.github.hectorvent.floci.services.glue.model.Table;
@@ -59,11 +62,21 @@ public class GlueService {
"\\s*([A-Za-z_][A-Za-z0-9_]*)\\s+in\\s*\\((.*)\\)\\s*",
Pattern.CASE_INSENSITIVE);
+ /** Measured against real Glue (us-west-2): a fourth index reports the limit. */
+ private static final int MAX_PARTITION_INDEXES_PER_TABLE = 3;
+
+ // Glue's partition index states. FAILED also exists but is only reachable through a backfill
+ // failure, which is not emulated.
+ private static final String INDEX_STATUS_CREATING = "CREATING";
+ private static final String INDEX_STATUS_ACTIVE = "ACTIVE";
+ private static final String INDEX_STATUS_DELETING = "DELETING";
+
private final StorageBackend<String, Database> databaseStore;
private final StorageBackend<String, Table> tableStore;
private final StorageBackend<String, Table> tableVersionStore;
private final StorageBackend<String, Map<String, Object>> columnStatisticsStore;
private final StorageBackend<String, Partition> partitionStore;
+ private final StorageBackend<String, PartitionIndexDescriptor> partitionIndexStore;
private final StorageBackend<String, Map<String, Object>> partitionColumnStatisticsStore;
private final StorageBackend<String, UserDefinedFunction> functionStore;
private final StorageBackend<String, Job> jobStore;
@@ -82,6 +95,8 @@ public class GlueService {
this.tableVersionStore = storageFactory.create("glue", "table_versions.json", new TypeReference<>() {});
this.columnStatisticsStore = storageFactory.create("glue", "column_statistics.json", new TypeReference<>() {});
this.partitionStore = storageFactory.create("glue", "partitions.json", new TypeReference<>() {});
+ this.partitionIndexStore = storageFactory.create(
+ "glue", "partition_indexes.json", new TypeReference<>() {});
this.partitionColumnStatisticsStore = storageFactory.create(
"glue", "partition_column_statistics.json", new TypeReference<>() {});
this.functionStore = storageFactory.create("glue", "functions.json", new TypeReference<>() {});
@@ -97,6 +112,7 @@ public class GlueService {
StorageBackend<String, Table> tableVersionStore,
StorageBackend<String, Map<String, Object>> columnStatisticsStore,
StorageBackend<String, Partition> partitionStore,
+ StorageBackend<String, PartitionIndexDescriptor> partitionIndexStore,
StorageBackend<String, Map<String, Object>> partitionColumnStatisticsStore,
StorageBackend<String, UserDefinedFunction> functionStore,
StorageBackend<String, Job> jobStore,
@@ -109,6 +125,7 @@ public class GlueService {
this.tableVersionStore = tableVersionStore;
this.columnStatisticsStore = columnStatisticsStore;
this.partitionStore = partitionStore;
+ this.partitionIndexStore = partitionIndexStore;
this.partitionColumnStatisticsStore = partitionColumnStatisticsStore;
this.functionStore = functionStore;
this.jobStore = jobStore;
@@ -281,6 +298,9 @@ public class GlueService {
partitionColumnStatisticsStore.keys().stream()
.filter(statisticsKey -> statisticsKey.startsWith(key + ":"))
.forEach(partitionColumnStatisticsStore::delete);
+ partitionIndexStore.keys().stream()
+ .filter(indexKey -> indexKey.startsWith(key + ":"))
+ .forEach(partitionIndexStore::delete);
LOG.infov("Deleted Glue Table: {0}.{1}", databaseName, tableName);
}
@@ -425,6 +445,158 @@ public class GlueService {
.toList();
}
+ /**
+ * Registers a partition index on a table.
+ *
+ * <p>The index is stored {@code CREATING}, as real Glue reports it while the backfill runs. It
+ * settles to {@code ACTIVE} on the next {@code GetPartitionIndexes}, so a client that polls
+ * sees the transition without the emulator depending on elapsed time.
+ *
+ * <p>Synchronized with the other partition-index methods, and with {@code updateTable}: the cap
+ * and duplicate checks read the stored indexes before writing, so concurrent creates would
+ * otherwise both pass a check that only one of them should.
+ */
+ public synchronized void createPartitionIndex(String databaseName, String tableName, PartitionIndex index) {
+ Table table = getTable(databaseName, tableName);
+
+ if (index == null || index.getIndexName() == null || index.getIndexName().isBlank()) {
+ throw new AwsException("InvalidInputException", "IndexName is required", 400);
+ }
+ List<String> keyNames = index.getKeys();
+ if (keyNames == null || keyNames.isEmpty()) {
+ throw new AwsException("InvalidInputException", "Keys is required", 400);
+ }
+
+ // Every key must name one of the table's partition keys: an index exists to narrow a
+ // partition scan, so a key outside that set could never be used.
+ List<KeySchemaElement> resolvedKeys = new ArrayList<>();
+ for (String keyName : keyNames) {
+ int position = partitionKeyIndex(table, keyName);
+ if (position < 0) {
+ throw new AwsException("InvalidInputException",
+ "IndexKeys not a part of PartitionColumns. Verify the indexKeys : [" + keyName + "]", 400);
+ }
+ Column partitionKey = table.getPartitionKeys().get(position);
+ resolvedKeys.add(new KeySchemaElement(partitionKey.getName(), partitionKey.getType()));
+ }
+
+ // Measured against real Glue (us-west-2): the cap is evaluated before the duplicate
+ // checks, so a create that is both over the limit and a duplicate reports the limit.
+ List<PartitionIndexDescriptor> existing = readPartitionIndexes(databaseName, tableName);
+ requireNoIndexInProgress(existing);
+ if (existing.size() >= MAX_PARTITION_INDEXES_PER_TABLE) {
+ throw new AwsException("ResourceNumberLimitExceededException",
+ "Partition index limit exceeded. Maximum: " + MAX_PARTITION_INDEXES_PER_TABLE, 400);
+ }
+
+ String key = partitionIndexKey(databaseName, tableName, index.getIndexName());
+ if (partitionIndexStore.get(key).isPresent()) {
+ throw new AwsException("AlreadyExistsException",
+ "Partition Index " + index.getIndexName() + " already exists with the same name.", 400);
+ }
+
+ // Glue also refuses a second index over the same keys under a different name. The
+ // comparison is order sensitive: [year, month] and [month, year] are distinct indexes.
+ List<String> resolvedKeyNames = resolvedKeys.stream().map(KeySchemaElement::getName).toList();
+ for (PartitionIndexDescriptor other : existing) {
+ List<String> otherKeyNames = other.getKeys() == null
+ ? List.of()
+ : other.getKeys().stream().map(KeySchemaElement::getName).toList();
+ if (otherKeyNames.equals(resolvedKeyNames)) {
+ throw new AwsException("AlreadyExistsException",
+ "Partition Index " + other.getIndexName() + " already exists with the same keys.", 400);
+ }
+ }
+
+ PartitionIndexDescriptor descriptor = new PartitionIndexDescriptor();
+ descriptor.setIndexName(index.getIndexName());
+ descriptor.setIndexStatus(INDEX_STATUS_CREATING);
+ descriptor.setKeys(resolvedKeys);
+ partitionIndexStore.put(key, descriptor);
+ LOG.infov("Created Glue partition index: {0}.{1} {2}", databaseName, tableName, index.getIndexName());
+ }
+
+ public synchronized void deletePartitionIndex(String databaseName, String tableName, String indexName) {
+ getTable(databaseName, tableName);
+ if (indexName == null || indexName.isBlank()) {
+ throw new AwsException("InvalidInputException", "IndexName is required", 400);
+ }
+ String key = partitionIndexKey(databaseName, tableName, indexName);
+ PartitionIndexDescriptor target = partitionIndexStore.get(key).orElse(null);
+ // An index still being created is not addressable by name yet: real Glue reports it as
+ // absent even while GetPartitionIndexes lists it as CREATING (measured in isolation).
+ if (target == null || INDEX_STATUS_CREATING.equals(target.getIndexStatus())) {
+ throw new AwsException("EntityNotFoundException",
+ "Index with the given indexName : " + indexName + " does not exist.", 400);
+ }
+ if (INDEX_STATUS_DELETING.equals(target.getIndexStatus())) {
+ throw new AwsException("EntityNotFoundException",
+ "Index with the given indexName : " + indexName + " does not exist.", 400);
+ }
+ requireNoIndexInProgress(readPartitionIndexes(databaseName, tableName));
+
+ // Deletion is observable: the index reports DELETING before it disappears.
+ target.setIndexStatus(INDEX_STATUS_DELETING);
+ partitionIndexStore.put(key, target);
+ LOG.infov("Deleting Glue partition index: {0}.{1} {2}", databaseName, tableName, indexName);
+ }
+
+ /**
+ * Returns a table's partition indexes and then advances any that are mid-lifecycle.
+ *
+ * <p>Real Glue creates and deletes an index asynchronously, so a caller sees {@code CREATING}
+ * before {@code ACTIVE} and {@code DELETING} before the index disappears. Rather than tie those
+ * transitions to wall-clock time, which would make tests racy, each read reports the current
+ * state and settles it: the next read sees the outcome. A client that polls for {@code ACTIVE}
+ * or for the index to vanish, as the Terraform provider does, converges on its second read.
+ */
+ public synchronized List<PartitionIndexDescriptor> getPartitionIndexes(String databaseName, String tableName) {
+ List<PartitionIndexDescriptor> current = readPartitionIndexes(databaseName, tableName);
+ for (PartitionIndexDescriptor descriptor : current) {
+ String key = partitionIndexKey(databaseName, tableName, descriptor.getIndexName());
+ if (INDEX_STATUS_CREATING.equals(descriptor.getIndexStatus())) {
+ PartitionIndexDescriptor settled = copyWithStatus(descriptor, INDEX_STATUS_ACTIVE);
+ partitionIndexStore.put(key, settled);
+ } else if (INDEX_STATUS_DELETING.equals(descriptor.getIndexStatus())) {
+ partitionIndexStore.delete(key);
+ }
+ }
+ return current;
+ }
+
+ /** Reads the stored indexes without advancing the lifecycle. */
+ private List<PartitionIndexDescriptor> readPartitionIndexes(String databaseName, String tableName) {
+ getTable(databaseName, tableName);
+ String prefix = tableKey(databaseName, tableName) + ":";
+ return partitionIndexStore.scan(k -> k.startsWith(prefix)).stream()
+ .sorted(Comparator.comparing(PartitionIndexDescriptor::getIndexName))
+ .toList();
+ }
+
+ private static PartitionIndexDescriptor copyWithStatus(PartitionIndexDescriptor source, String status) {
+ PartitionIndexDescriptor copy = new PartitionIndexDescriptor();
+ copy.setIndexName(source.getIndexName());
+ copy.setIndexStatus(status);
+ copy.setKeys(source.getKeys());
+ copy.setBackfillErrors(source.getBackfillErrors());
+ return copy;
+ }
+
+ /**
+ * Glue allows one index per table to be created or deleted at a time. Measured message, with
+ * the offending index and its state named.
+ */
+ private static void requireNoIndexInProgress(List<PartitionIndexDescriptor> existing) {
+ for (PartitionIndexDescriptor descriptor : existing) {
+ String status = descriptor.getIndexStatus();
+ if (INDEX_STATUS_CREATING.equals(status) || INDEX_STATUS_DELETING.equals(status)) {
+ throw new AwsException("ResourceNumberLimitExceededException",
+ "Index " + descriptor.getIndexName() + " is in " + status
+ + " state. Only 1 index can be created or deleted simultaneously per table.", 400);
+ }
+ }
+ }
+
// The partition store scan returns values in storage iteration order, which is not stable.
// Return partitions ordered by their values so GetPartitions is deterministic across calls.
private static int comparePartitionValues(Partition a, Partition b) {
@@ -624,6 +796,10 @@ public class GlueService {
return tableKey(databaseName, tableName) + ":" + normalizeName(columnName);
}
+ private static String partitionIndexKey(String databaseName, String tableName, String indexName) {
+ return tableKey(databaseName, tableName) + ":" + normalizeName(indexName);
+ }
+
private static String partitionKey(String databaseName, String tableName, List<String> partitionValues) {
return tableKey(databaseName, tableName) + ":" + String.join(":", partitionValues.stream()
.map(GlueService::encodePartitionValue)
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/KeySchemaElement.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/KeySchemaElement.java
new file mode 100644
index 0000000000..6923baaf1b
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/KeySchemaElement.java
@@ -0,0 +1,24 @@
+package io.github.hectorvent.floci.services.glue.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+/** One key of a partition index, as returned by {@code GetPartitionIndexes}. */
+@RegisterForReflection
+public class KeySchemaElement {
+ @JsonProperty("Name")
+ private String name;
+ @JsonProperty("Type")
+ private String type;
+
+ public KeySchemaElement() {}
+ public KeySchemaElement(String name, String type) {
+ this.name = name;
+ this.type = type;
+ }
+
+ public String getName() { return name; }
+ public void setName(String name) { this.name = name; }
+ public String getType() { return type; }
+ public void setType(String type) { this.type = type; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/PartitionIndex.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/PartitionIndex.java
new file mode 100644
index 0000000000..7764c0090b
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/PartitionIndex.java
@@ -0,0 +1,28 @@
+package io.github.hectorvent.floci.services.glue.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * The partition index as supplied to {@code CreatePartitionIndex}.
+ *
+ * <p>Note the asymmetry with {@link PartitionIndexDescriptor}: a request carries key <em>names</em>
+ * only, while a read returns each key's name and type, resolved from the table's partition keys.
+ */
+@RegisterForReflection
+public class PartitionIndex {
+ @JsonProperty("IndexName")
+ private String indexName;
+ @JsonProperty("Keys")
+ private List<String> keys;
+
+ public PartitionIndex() {}
+
+ public String getIndexName() { return indexName; }
+ public void setIndexName(String indexName) { this.indexName = indexName; }
+ public List<String> getKeys() { return keys == null ? null : new ArrayList<>(keys); }
+ public void setKeys(List<String> keys) { this.keys = keys == null ? null : new ArrayList<>(keys); }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/PartitionIndexDescriptor.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/PartitionIndexDescriptor.java
new file mode 100644
index 0000000000..4e6caa257c
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/PartitionIndexDescriptor.java
@@ -0,0 +1,43 @@
+package io.github.hectorvent.floci.services.glue.model;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * A stored partition index, as returned by {@code GetPartitionIndexes}.
+ *
+ * <p>{@code IndexStatus} follows the real lifecycle, advanced by reads rather than by time: an
+ * index is stored {@code CREATING} and becomes {@code ACTIVE} on the next
+ * {@code GetPartitionIndexes}; a deleted one is stored {@code DELETING} and disappears on that
+ * same read. Real Glue backfills asynchronously, so a client that polls for {@code ACTIVE} (the
+ * Terraform provider does) sees the transition it expects, without the emulator needing a clock.
+ */
+@RegisterForReflection
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class PartitionIndexDescriptor {
+ @JsonProperty("IndexName")
+ private String indexName;
+ @JsonProperty("IndexStatus")
+ private String indexStatus;
+ @JsonProperty("Keys")
+ private List<KeySchemaElement> keys;
+ @JsonProperty("BackfillErrors")
+ private List<Object> backfillErrors;
+
+ public PartitionIndexDescriptor() {}
+
+ public String getIndexName() { return indexName; }
+ public void setIndexName(String indexName) { this.indexName = indexName; }
+ public String getIndexStatus() { return indexStatus; }
+ public void setIndexStatus(String indexStatus) { this.indexStatus = indexStatus; }
+ public List<KeySchemaElement> getKeys() { return keys == null ? null : new ArrayList<>(keys); }
+ public void setKeys(List<KeySchemaElement> keys) { this.keys = keys == null ? null : new ArrayList<>(keys); }
+ public List<Object> getBackfillErrors() { return backfillErrors == null ? null : new ArrayList<>(backfillErrors); }
+ public void setBackfillErrors(List<Object> backfillErrors) {
+ this.backfillErrors = backfillErrors == null ? null : new ArrayList<>(backfillErrors);
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerTest.java
index cdc0bf0d5e..8e33bd28cc 100644
--- a/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerTest.java
@@ -2,7 +2,9 @@ package io.github.hectorvent.floci.services.glue;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
+import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.RegionResolver;
import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
import io.github.hectorvent.floci.core.storage.StorageFactory;
@@ -13,6 +15,7 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import com.fasterxml.jackson.core.type.TypeReference;
+import java.util.List;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -71,7 +74,7 @@ class GlueJsonHandlerTest {
@SuppressWarnings("unchecked")
Map<String, Object> body = (Map<String, Object>) response.getEntity();
assertTrue(body.containsKey("PartitionIndexDescriptorList"));
- assertTrue(((java.util.List<?>) body.get("PartitionIndexDescriptorList")).isEmpty());
+ assertTrue(((List<?>) body.get("PartitionIndexDescriptorList")).isEmpty());
}
/** A missing table is reported as missing, not as a table that happens to have no indexes. */
@@ -86,6 +89,241 @@ class GlueJsonHandlerTest {
assertThrows(Exception.class, () -> handler.handle("GetPartitionIndexes", request, REGION));
}
+ /** Creates a table whose partition keys are the given names, all typed {@code string}. */
+ private void createPartitionedTable(String dbName, String tableName, String... partitionKeys) throws Exception {
+ ObjectNode createDb = mapper.createObjectNode();
+ createDb.putObject("DatabaseInput").put("Name", dbName);
+ try {
+ handler.handle("CreateDatabase", createDb, REGION);
+ } catch (AwsException alreadyExists) {
+ // The helper is called twice in the drop-and-recreate case; the database survives.
+ }
+
+ ObjectNode createTable = mapper.createObjectNode();
+ createTable.put("DatabaseName", dbName);
+ ObjectNode tableInput = createTable.putObject("TableInput");
+ tableInput.put("Name", tableName);
+ ArrayNode keys = tableInput.putArray("PartitionKeys");
+ for (String partitionKey : partitionKeys) {
+ keys.addObject().put("Name", partitionKey).put("Type", "string");
+ }
+ assertEquals(200, handler.handle("CreateTable", createTable, REGION).getStatus());
+ }
+
+ private Response createIndex(String dbName, String tableName, String indexName, String... keys) throws Exception {
+ ObjectNode request = mapper.createObjectNode();
+ request.put("DatabaseName", dbName);
+ request.put("TableName", tableName);
+ ObjectNode index = request.putObject("PartitionIndex");
+ index.put("IndexName", indexName);
+ ArrayNode keyArray = index.putArray("Keys");
+ for (String key : keys) {
+ keyArray.add(key);
+ }
+ return handler.handle("CreatePartitionIndex", request, REGION);
+ }
+
+ private Response deleteIndex(String dbName, String tableName, String indexName) throws Exception {
+ ObjectNode request = mapper.createObjectNode();
+ request.put("DatabaseName", dbName);
+ request.put("TableName", tableName);
+ request.put("IndexName", indexName);
+ return handler.handle("DeletePartitionIndex", request, REGION);
+ }
+
+ @SuppressWarnings("unchecked")
+ private List<Map<String, Object>> listIndexes(String dbName, String tableName) throws Exception {
+ ObjectNode request = mapper.createObjectNode();
+ request.put("DatabaseName", dbName);
+ request.put("TableName", tableName);
+ Response response = handler.handle("GetPartitionIndexes", request, REGION);
+ assertEquals(200, response.getStatus());
+ Map<String, Object> body = (Map<String, Object>) response.getEntity();
+ return mapper.convertValue(
+ body.get("PartitionIndexDescriptorList"), new TypeReference<List<Map<String, Object>>>() {});
+ }
+
+ /** Reads once to settle any in-progress index, mirroring a client that polls to ACTIVE. */
+ private void settleIndexes(String dbName, String tableName) throws Exception {
+ listIndexes(dbName, tableName);
+ }
+
+ private static String statusOf(List<Map<String, Object>> indexes, String indexName) {
+ return indexes.stream()
+ .filter(index -> indexName.equals(index.get("IndexName")))
+ .map(index -> (String) index.get("IndexStatus"))
+ .findFirst()
+ .orElse(null);
+ }
+
+ // ── Partition indexes ──────────────────────────────────────────
+ //
+ // Behaviour below was captured against real Glue in us-west-2 on a table partitioned by
+ // tenant/year/month/day, so the codes, messages and lifecycle states are AWS's own.
+
+ @Test
+ void aNewIndexReportsCreatingAndThenBecomesActive() throws Exception {
+ createPartitionedTable("idx_db", "events", "tenant", "year", "month", "day");
+ assertEquals(200, createIndex("idx_db", "events", "by_tenant", "tenant").getStatus());
+
+ // Real Glue reports CREATING until the backfill finishes, then ACTIVE.
+ List<Map<String, Object>> whileCreating = listIndexes("idx_db", "events");
+ assertEquals("CREATING", statusOf(whileCreating, "by_tenant"));
+
+ List<Map<String, Object>> settled = listIndexes("idx_db", "events");
+ assertEquals("ACTIVE", statusOf(settled, "by_tenant"));
+ }
+
+ @Test
+ void aReadResolvesEachKeyToItsNameAndType() throws Exception {
+ createPartitionedTable("idx_db_keys_shape", "events", "tenant", "year");
+ createIndex("idx_db_keys_shape", "events", "by_tenant", "tenant");
+
+ List<Map<String, Object>> indexes = listIndexes("idx_db_keys_shape", "events");
+ assertEquals(1, indexes.size());
+ assertEquals("by_tenant", indexes.get(0).get("IndexName"));
+ @SuppressWarnings("unchecked")
+ List<Map<String, Object>> keys = (List<Map<String, Object>>) indexes.get(0).get("Keys");
+ // A request carries key names only; the read resolves each one's type from the table.
+ assertEquals("tenant", keys.get(0).get("Name"));
+ assertEquals("string", keys.get(0).get("Type"));
+ }
+
+ @Test
+ void aDeletedIndexReportsDeletingAndThenDisappears() throws Exception {
+ createPartitionedTable("idx_db_del", "events", "tenant", "year");
+ createIndex("idx_db_del", "events", "by_tenant", "tenant");
+ settleIndexes("idx_db_del", "events");
+
+ assertEquals(200, deleteIndex("idx_db_del", "events", "by_tenant").getStatus());
+
+ List<Map<String, Object>> whileDeleting = listIndexes("idx_db_del", "events");
+ assertEquals("DELETING", statusOf(whileDeleting, "by_tenant"));
+
+ assertTrue(listIndexes("idx_db_del", "events").isEmpty());
+ }
+
+ @Test
+ void aSecondIndexCannotBeCreatedWhileOneIsCreating() throws Exception {
+ createPartitionedTable("idx_db_busy", "events", "tenant", "year");
+ createIndex("idx_db_busy", "events", "first", "tenant");
+
+ AwsException ex = assertThrows(AwsException.class,
+ () -> createIndex("idx_db_busy", "events", "second", "year"));
+ assertEquals("ResourceNumberLimitExceededException", ex.getErrorCode());
+ assertTrue(ex.getMessage().contains("is in CREATING state"));
+ }
+
+ @Test
+ void anIndexCannotBeCreatedWhileAnotherIsDeleting() throws Exception {
+ createPartitionedTable("idx_db_busy2", "events", "tenant", "year");
+ createIndex("idx_db_busy2", "events", "first", "tenant");
+ settleIndexes("idx_db_busy2", "events");
+ deleteIndex("idx_db_busy2", "events", "first");
+
+ AwsException ex = assertThrows(AwsException.class,
+ () -> createIndex("idx_db_busy2", "events", "second", "year"));
+ assertEquals("ResourceNumberLimitExceededException", ex.getErrorCode());
+ assertTrue(ex.getMessage().contains("is in DELETING state"));
+ }
+
+ @Test
+ void anIndexStillCreatingCannotBeDeleted() throws Exception {
+ createPartitionedTable("idx_db_earlydel", "events", "tenant");
+ createIndex("idx_db_earlydel", "events", "by_tenant", "tenant");
+
+ // Measured in isolation: Glue reports it absent even while a read lists it as CREATING.
+ AwsException ex = assertThrows(AwsException.class,
+ () -> deleteIndex("idx_db_earlydel", "events", "by_tenant"));
+ assertEquals("EntityNotFoundException", ex.getErrorCode());
+ }
+
+ @Test
+ void anIndexKeyMustBeOneOfTheTablesPartitionKeys() throws Exception {
+ createPartitionedTable("idx_db_key", "events", "tenant", "year");
+
+ AwsException ex = assertThrows(AwsException.class,
+ () -> createIndex("idx_db_key", "events", "bad", "nosuchcolumn"));
+ assertEquals("InvalidInputException", ex.getErrorCode());
+ }
+
+ @Test
+ void anIndexNameCannotBeReused() throws Exception {
+ createPartitionedTable("idx_db_dup", "events", "tenant", "year");
+ createIndex("idx_db_dup", "events", "dup", "tenant");
+ settleIndexes("idx_db_dup", "events");
+
+ AwsException ex = assertThrows(AwsException.class,
+ () -> createIndex("idx_db_dup", "events", "dup", "year"));
+ assertEquals("AlreadyExistsException", ex.getErrorCode());
+ assertTrue(ex.getMessage().contains("same name"));
+ }
+
+ @Test
+ void aSecondIndexOverTheSameKeysIsRefusedEvenUnderANewName() throws Exception {
+ createPartitionedTable("idx_db_keys", "events", "tenant", "year");
+ createIndex("idx_db_keys", "events", "first", "tenant");
+ settleIndexes("idx_db_keys", "events");
+
+ AwsException ex = assertThrows(AwsException.class,
+ () -> createIndex("idx_db_keys", "events", "second", "tenant"));
+ assertEquals("AlreadyExistsException", ex.getErrorCode());
+ assertTrue(ex.getMessage().contains("same keys"));
+ }
+
+ @Test
+ void keyOrderDistinguishesTwoIndexes() throws Exception {
+ createPartitionedTable("idx_db_order", "events", "tenant", "year", "month");
+
+ // [year, month] and [month, year] are different indexes on real Glue, not duplicates.
+ assertEquals(200, createIndex("idx_db_order", "events", "ord1", "year", "month").getStatus());
+ settleIndexes("idx_db_order", "events");
+ assertEquals(200, createIndex("idx_db_order", "events", "ord2", "month", "year").getStatus());
+ settleIndexes("idx_db_order", "events");
+ assertEquals(2, listIndexes("idx_db_order", "events").size());
+ }
+
+ @Test
+ void aFourthIndexExceedsTheLimit() throws Exception {
+ createPartitionedTable("idx_db_cap", "events", "tenant", "year", "month", "day");
+ createIndex("idx_db_cap", "events", "i1", "tenant");
+ settleIndexes("idx_db_cap", "events");
+ createIndex("idx_db_cap", "events", "i2", "year");
+ settleIndexes("idx_db_cap", "events");
+ createIndex("idx_db_cap", "events", "i3", "month");
+ settleIndexes("idx_db_cap", "events");
+
+ AwsException ex = assertThrows(AwsException.class,
+ () -> createIndex("idx_db_cap", "events", "i4", "day"));
+ assertEquals("ResourceNumberLimitExceededException", ex.getErrorCode());
+ assertTrue(ex.getMessage().contains("Maximum: 3"));
+ }
+
+ @Test
+ void deletingAnUnknownIndexFails() throws Exception {
+ createPartitionedTable("idx_db_missing", "events", "tenant");
+
+ AwsException ex = assertThrows(AwsException.class,
+ () -> deleteIndex("idx_db_missing", "events", "absent"));
+ assertEquals("EntityNotFoundException", ex.getErrorCode());
+ }
+
+ @Test
+ void droppingATableDropsItsIndexes() throws Exception {
+ createPartitionedTable("idx_db_cascade", "events", "tenant");
+ createIndex("idx_db_cascade", "events", "by_tenant", "tenant");
+ settleIndexes("idx_db_cascade", "events");
+
+ ObjectNode dropTable = mapper.createObjectNode();
+ dropTable.put("DatabaseName", "idx_db_cascade");
+ dropTable.put("Name", "events");
+ assertEquals(200, handler.handle("DeleteTable", dropTable, REGION).getStatus());
+
+ // Recreating the table must not resurrect the old index.
+ createPartitionedTable("idx_db_cascade", "events", "tenant");
+ assertTrue(listIndexes("idx_db_cascade", "events").isEmpty());
+ }
+
@Test
void createCrawlerWithScheduleSucceeds() throws Exception {
ObjectNode request = mapper.createObjectNode();
diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java
index 0f856c2f43..9bf4a9722e 100644
--- a/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java
@@ -15,6 +15,8 @@ import io.github.hectorvent.floci.services.glue.model.Job;
import io.github.hectorvent.floci.services.glue.model.JobCommand;
import io.github.hectorvent.floci.services.glue.model.JobUpdate;
import io.github.hectorvent.floci.services.glue.model.Partition;
+import io.github.hectorvent.floci.services.glue.model.PartitionIndex;
+import io.github.hectorvent.floci.services.glue.model.PartitionIndexDescriptor;
import io.github.hectorvent.floci.services.glue.model.S3Target;
import io.github.hectorvent.floci.services.glue.model.SchemaReference;
import io.github.hectorvent.floci.services.glue.model.StorageDescriptor;
@@ -31,6 +33,8 @@ import java.time.Instant;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.CountDownLatch;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
@@ -76,6 +80,7 @@ class GlueServiceTest {
tableVersionStore,
columnStatisticsStore,
partitionStore,
+ new InMemoryStorage<String, PartitionIndexDescriptor>(),
new InMemoryStorage<String, Map<String, Object>>(),
new InMemoryStorage<String, UserDefinedFunction>(),
new InMemoryStorage<String, Job>(),
@@ -1258,4 +1263,52 @@ class GlueServiceTest {
AwsException udfTagEx = assertThrows(AwsException.class, () -> glueService.tagResource(fakeUdfArn, Map.of("k", "v"), REGION));
assertEquals("EntityNotFoundException", udfTagEx.getErrorCode());
}
+
+ @Test
+ void concurrentPartitionIndexCreatesStopAtTheCap() throws Exception {
+ Table table = new Table();
+ table.setName("indexed");
+ StorageDescriptor sd = new StorageDescriptor();
+ sd.setColumns(java.util.List.of(new Column("a", "string")));
+ table.setStorageDescriptor(sd);
+ table.setPartitionKeys(java.util.List.of(new Column("a", "string"), new Column("b", "string")));
+ glueService.createTable("db1", table);
+
+ // Two settled indexes leave exactly one slot under the cap of three.
+ createSettledIndex("idx0", "a");
+ createSettledIndex("idx1", "b");
+
+ int attempts = 4;
+ CountDownLatch start = new CountDownLatch(1);
+ CountDownLatch done = new CountDownLatch(attempts);
+ for (int i = 0; i < attempts; i++) {
+ String indexName = "race" + i;
+ Thread.ofVirtual().start(() -> {
+ try {
+ start.await();
+ PartitionIndex index = new PartitionIndex();
+ index.setIndexName(indexName);
+ index.setKeys(java.util.List.of("a", "b"));
+ glueService.createPartitionIndex("db1", "indexed", index);
+ } catch (AwsException | InterruptedException expected) {
+ // Only the create that takes the last slot succeeds; the rest are rejected.
+ } finally {
+ done.countDown();
+ }
+ });
+ }
+ start.countDown();
+ assertTrue(done.await(10, TimeUnit.SECONDS), "partition index creates did not finish");
+
+ assertEquals(3, glueService.getPartitionIndexes("db1", "indexed").size());
+ }
+
+ private void createSettledIndex(String indexName, String key) {
+ PartitionIndex index = new PartitionIndex();
+ index.setIndexName(indexName);
+ index.setKeys(java.util.List.of(key));
+ glueService.createPartitionIndex("db1", "indexed", index);
+ glueService.getPartitionIndexes("db1", "indexed");
+ }
+
}