Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(glue): populate CatalogId on GetDatabase and GetDatabases (#2814)

Tirth Patel342f58b2b79a
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java b/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java
index 0c1b9796ae..ae48d67ae9 100644
--- a/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java
@@ -128,6 +128,7 @@ public class GlueService {
throw new AwsException("AlreadyExistsException", "Database already exists: " + database.getName(), 400);
}
database.setName(databaseName);
+ database.setCatalogId(regionResolver.getAccountId());
databaseStore.put(databaseName, database);
if (tags != null && !tags.isEmpty()) {
resourceGroupsTaggingService.tagResources(List.of(databaseArn(region, databaseName)), tags, region);
@@ -136,12 +137,27 @@ public class GlueService {
}
public Database getDatabase(String name) {
- return databaseStore.get(normalizeName(name))
+ Database database = databaseStore.get(normalizeName(name))
.orElseThrow(() -> new AwsException("EntityNotFoundException", "Database not found: " + name, 400));
+ backfillCatalogId(database);
+ return database;
}
public List<Database> getDatabases() {
- return databaseStore.scan(k -> true);
+ List<Database> databases = databaseStore.scan(k -> true);
+ databases.forEach(this::backfillCatalogId);
+ return databases;
+ }
+
+ /**
+ * Heals a database persisted before CatalogId was modelled. The store is namespaced by
+ * account, so the reader is always the owning account and the healed value is stable —
+ * the same migrate-on-read the account-aware backend does for un-prefixed keys.
+ */
+ private void backfillCatalogId(Database database) {
+ if (database.getCatalogId() == null) {
+ database.setCatalogId(regionResolver.getAccountId());
+ }
}
public void updateDatabase(String name, Database database) {
@@ -156,6 +172,7 @@ public class GlueService {
updated.setLocationUri(database.getLocationUri());
updated.setParameters(database.getParameters() == null ? null : new LinkedHashMap<>(database.getParameters()));
updated.setCreateTime(existing.getCreateTime());
+ updated.setCatalogId(regionResolver.getAccountId());
databaseStore.put(databaseName, updated);
LOG.infov("Updated Glue Database: {0}", databaseName);
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/Database.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/Database.java
index 3d85f7745c..0eb7efbeea 100644
--- a/src/main/java/io/github/hectorvent/floci/services/glue/model/Database.java
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/Database.java
@@ -19,6 +19,8 @@ public class Database {
@JsonProperty("CreateTime")
@JsonFormat(shape = JsonFormat.Shape.NUMBER)
private Instant createTime;
+ @JsonProperty("CatalogId")
+ private String catalogId;
public Database() {}
public Database(String name) {
@@ -36,4 +38,6 @@ public class Database {
public void setParameters(Map<String, String> parameters) { this.parameters = parameters; }
public Instant getCreateTime() { return createTime; }
public void setCreateTime(Instant createTime) { this.createTime = createTime; }
+ public String getCatalogId() { return catalogId; }
+ public void setCatalogId(String catalogId) { this.catalogId = catalogId; }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueDatabaseCatalogIdIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueDatabaseCatalogIdIntegrationTest.java
new file mode 100644
index 0000000000..93ff7aa8f4
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueDatabaseCatalogIdIntegrationTest.java
@@ -0,0 +1,65 @@
+package io.github.hectorvent.floci.services.glue;
+
+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
+import io.quarkus.test.junit.QuarkusTest;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import java.util.UUID;
+
+import static io.restassured.RestAssured.given;
+import static org.hamcrest.Matchers.equalTo;
+
+/**
+ * terraform-provider-aws builds the aws_glue_catalog_database resource id as
+ * {@code catalog-id:database-name}, so an absent CatalogId yields a malformed id.
+ */
+@QuarkusTest
+class GlueDatabaseCatalogIdIntegrationTest {
+
+ private static final String CONTENT_TYPE = "application/x-amz-json-1.1";
+ private static final String ACCOUNT_ID = "000000000000";
+ private static final String DATABASE_NAME = "catalog-id-db-" + UUID.randomUUID().toString().substring(0, 8);
+
+ @BeforeAll
+ static void configureRestAssured() {
+ RestAssuredJsonUtils.configureAwsContentTypes();
+ }
+
+ @Test
+ void getDatabaseAndGetDatabasesReturnCatalogId() {
+ given().contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.CreateDatabase")
+ .body("""
+ {
+ "DatabaseInput": {
+ "Name": "%s"
+ }
+ }
+ """.formatted(DATABASE_NAME))
+ .when().post("/")
+ .then().statusCode(200);
+
+ given().contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.GetDatabase")
+ .body("""
+ {
+ "Name": "%s"
+ }
+ """.formatted(DATABASE_NAME))
+ .when().post("/")
+ .then()
+ .statusCode(200)
+ .body("Database.Name", equalTo(DATABASE_NAME))
+ .body("Database.CatalogId", equalTo(ACCOUNT_ID));
+
+ given().contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.GetDatabases")
+ .body("{}")
+ .when().post("/")
+ .then()
+ .statusCode(200)
+ .body("DatabaseList.find { it.Name == '%s' }.CatalogId".formatted(DATABASE_NAME),
+ equalTo(ACCOUNT_ID));
+ }
+}
\ No newline at end of file
diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java
index d104c414e5..0f856c2f43 100644
--- a/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java
@@ -54,6 +54,7 @@ class GlueServiceTest {
private GlueService glueService;
private GlueSchemaRegistryService schemaRegistryService;
+ private StorageBackend<String, Database> databaseStore;
private StorageBackend<String, Table> tableStore;
private StorageBackend<String, Table> tableVersionStore;
private StorageBackend<String, Map<String, Object>> columnStatisticsStore;
@@ -64,12 +65,13 @@ class GlueServiceTest {
RegionResolver regionResolver = new RegionResolver(REGION, ACCOUNT_ID);
StorageFactory storageFactory = new InMemoryStorageFactory();
schemaRegistryService = new GlueSchemaRegistryService(storageFactory, regionResolver);
+ databaseStore = new InMemoryStorage<>();
tableStore = new InMemoryStorage<>();
tableVersionStore = new InMemoryStorage<>();
columnStatisticsStore = new InMemoryStorage<>();
partitionStore = new InMemoryStorage<>();
glueService = new GlueService(
- new InMemoryStorage<String, Database>(),
+ databaseStore,
tableStore,
tableVersionStore,
columnStatisticsStore,
@@ -82,6 +84,24 @@ class GlueServiceTest {
glueService.createDatabase(new Database("db1"));
}
+ @Test
+ void createDatabasePersistsCatalogIdSoReadsDoNotWriteIt() {
+ assertEquals(ACCOUNT_ID, databaseStore.get("db1").orElseThrow().getCatalogId());
+ assertEquals(ACCOUNT_ID, glueService.getDatabase("db1").getCatalogId());
+ }
+
+ @Test
+ void databasePersistedBeforeCatalogIdWasModelledIsBackfilledOnRead() {
+ Database legacy = new Database("legacy-db");
+ legacy.setCatalogId(null);
+ databaseStore.put("legacy-db", legacy);
+
+ assertEquals(ACCOUNT_ID, glueService.getDatabase("legacy-db").getCatalogId());
+ assertEquals(ACCOUNT_ID, glueService.getDatabases().stream()
+ .filter(database -> "legacy-db".equals(database.getName()))
+ .findFirst().orElseThrow().getCatalogId());
+ }
+
@Test
void getTableWithoutSchemaReferenceReturnsColumnsUnchanged() {
Table table = new Table();