Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(athena): project the column names the Glue table declares (#3414)

Ovidiuebe9cf2264c6
diff --git a/src/main/java/io/github/hectorvent/floci/services/athena/GlueViewDdlBuilder.java b/src/main/java/io/github/hectorvent/floci/services/athena/GlueViewDdlBuilder.java
index 31fc24fdf3..0ac8693598 100644
--- a/src/main/java/io/github/hectorvent/floci/services/athena/GlueViewDdlBuilder.java
+++ b/src/main/java/io/github/hectorvent/floci/services/athena/GlueViewDdlBuilder.java
@@ -1,14 +1,18 @@
package io.github.hectorvent.floci.services.athena;
import io.github.hectorvent.floci.services.glue.GlueService;
+import io.github.hectorvent.floci.services.glue.model.Column;
import io.github.hectorvent.floci.services.glue.model.Database;
import io.github.hectorvent.floci.services.glue.model.Table;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import org.jboss.logging.Logger;
+import java.util.ArrayList;
+import java.util.HashSet;
import java.util.List;
import java.util.Locale;
+import java.util.Set;
@ApplicationScoped
public class GlueViewDdlBuilder {
@@ -88,7 +92,9 @@ public class GlueViewDdlBuilder {
: quote(t.getName());
sb.append("CREATE OR REPLACE VIEW ")
.append(target)
- .append(" AS SELECT * FROM ")
+ .append(" AS SELECT ")
+ .append(buildProjection(t))
+ .append(" FROM ")
.append(readExpression(readFn, normalizedLocation))
.append(";\n");
} catch (Exception e) {
@@ -97,6 +103,58 @@ public class GlueViewDdlBuilder {
}
}
+ /**
+ * Projects the columns the catalog declares, rather than whatever the underlying files spell.
+ *
+ * <p>Athena reports a table's declared column names: a JSON body written with {@code tenantId}
+ * surfaces as {@code tenantid} when the table declares it that way, because the catalog owns the
+ * schema and the SerDe maps onto it. Inference sees only the data, so without this the view
+ * exposes the file's spelling and a client reading the declared name finds nothing. Identifier
+ * resolution is case-insensitive here, so a declared name still binds to an inferred one that
+ * differs from it only by case.
+ *
+ * <p>Partition keys are part of the table's schema and are what partition predicates filter on,
+ * so they are projected too. A table that declares no columns keeps {@code *}, leaving inference
+ * in charge.
+ */
+ static String buildProjection(Table table) {
+ List<Column> declared = new ArrayList<>();
+ if (table != null) {
+ if (table.getStorageDescriptor() != null && table.getStorageDescriptor().getColumns() != null) {
+ declared.addAll(table.getStorageDescriptor().getColumns());
+ }
+ if (table.getPartitionKeys() != null) {
+ declared.addAll(table.getPartitionKeys());
+ }
+ }
+
+ List<String> names = new ArrayList<>();
+ Set<String> seen = new HashSet<>();
+ for (Column c : declared) {
+ if (c == null || c.getName() == null || c.getName().isBlank()) {
+ continue;
+ }
+ String name = c.getName();
+ // A partition key repeating a data column would bind twice and make the view ambiguous.
+ if (seen.add(name.toLowerCase(Locale.ROOT))) {
+ names.add(name);
+ }
+ }
+
+ if (names.isEmpty()) {
+ return "*";
+ }
+
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < names.size(); i++) {
+ if (i > 0) {
+ sb.append(", ");
+ }
+ sb.append(quote(names.get(i))).append(" AS ").append(quote(names.get(i)));
+ }
+ return sb.toString();
+ }
+
static String inferReadFunction(Table table) {
if (table == null || table.getStorageDescriptor() == null) {
return "read_csv_auto";
diff --git a/src/test/java/io/github/hectorvent/floci/services/athena/GlueViewDdlBuilderTest.java b/src/test/java/io/github/hectorvent/floci/services/athena/GlueViewDdlBuilderTest.java
index 4e65ad153e..e9978de508 100644
--- a/src/test/java/io/github/hectorvent/floci/services/athena/GlueViewDdlBuilderTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/athena/GlueViewDdlBuilderTest.java
@@ -2,6 +2,7 @@ package io.github.hectorvent.floci.services.athena;
import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.services.glue.GlueService;
+import io.github.hectorvent.floci.services.glue.model.Column;
import io.github.hectorvent.floci.services.glue.model.Database;
import io.github.hectorvent.floci.services.glue.model.StorageDescriptor;
import io.github.hectorvent.floci.services.glue.model.Table;
@@ -50,6 +51,82 @@ class GlueViewDdlBuilderTest {
return table;
}
+ private static Column column(String name) {
+ Column c = new Column();
+ c.setName(name);
+ c.setType("string");
+ return c;
+ }
+
+ private Table tableWithColumns(String name, String location, List<Column> columns, List<Column> partitionKeys) {
+ Table table = createTable(name, location, "org.openx.data.jsonserde.JsonSerDe", null);
+ table.getStorageDescriptor().setColumns(columns);
+ table.setPartitionKeys(partitionKeys);
+ return table;
+ }
+
+ private String ddlFor(Table table) {
+ Database db = createDatabase("audit");
+ when(glueService.getDatabases()).thenReturn(List.of(db));
+ when(glueService.getTables("audit")).thenReturn(List.of(table));
+ return builder.build(null);
+ }
+
+ /**
+ * Athena reports the column names the catalog declares. The files here spell them camelCase, so a
+ * view built by inference alone would expose the file's spelling and a client reading the declared
+ * name would find nothing.
+ */
+ @Test
+ void testDeclaredColumnsAreProjectedInsteadOfStar() {
+ String ddl = ddlFor(tableWithColumns("audit_events", "s3://audit/",
+ List.of(column("eventid"), column("tenantid")),
+ List.of(column("tenant"), column("year"))));
+
+ assertTrue(ddl.contains("\"eventid\" AS \"eventid\""), ddl);
+ assertTrue(ddl.contains("\"tenantid\" AS \"tenantid\""), ddl);
+ assertFalse(ddl.contains("SELECT * FROM"), ddl);
+ }
+
+ /** Partition keys are part of the schema and are what partition predicates filter on. */
+ @Test
+ void testPartitionKeysAreProjectedAlongsideDataColumns() {
+ String ddl = ddlFor(tableWithColumns("audit_events", "s3://audit/",
+ List.of(column("eventid")),
+ List.of(column("tenant"), column("year"), column("month"), column("day"))));
+
+ for (String name : List.of("eventid", "tenant", "year", "month", "day")) {
+ assertTrue(ddl.contains("\"" + name + "\" AS \"" + name + "\""), name + " missing from: " + ddl);
+ }
+ }
+
+ /** A partition key repeating a data column would bind twice and make the view ambiguous. */
+ @Test
+ void testDuplicateColumnAndPartitionKeyIsProjectedOnce() {
+ String ddl = ddlFor(tableWithColumns("t", "s3://b/",
+ List.of(column("tenant")), List.of(column("TENANT"))));
+
+ int first = ddl.indexOf("AS \"tenant\"");
+ assertTrue(first >= 0, ddl);
+ assertEquals(-1, ddl.indexOf("AS \"tenant\"", first + 1), "projected twice: " + ddl);
+ }
+
+ /** A table that declares no columns leaves inference in charge, as before. */
+ @Test
+ void testTableWithoutDeclaredColumnsStillUsesStar() {
+ String ddl = ddlFor(tableWithColumns("t", "s3://b/", null, null));
+
+ assertTrue(ddl.contains("SELECT * FROM"), ddl);
+ }
+
+ /** Column identifiers go through the same quoting as table and schema names. */
+ @Test
+ void testColumnIdentifierQuotingEscapesDoubleQuotes() {
+ String ddl = ddlFor(tableWithColumns("t", "s3://b/", List.of(column("we\"ird")), null));
+
+ assertTrue(ddl.contains("\"we\"\"ird\" AS \"we\"\"ird\""), ddl);
+ }
+
@Test
void testTwoDatabasesQualifiedViewsAndSchemaPerDb() {
Database shopDb = createDatabase("shop");