Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(athena): read a projecting table's partition paths, not its whole location (#3415)

Ovidiu6d6940bb8fc3
diff --git a/src/main/java/io/github/hectorvent/floci/services/athena/AthenaService.java b/src/main/java/io/github/hectorvent/floci/services/athena/AthenaService.java
index 6517470515..1f26cf11d7 100644
--- a/src/main/java/io/github/hectorvent/floci/services/athena/AthenaService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/athena/AthenaService.java
@@ -146,6 +146,9 @@ public class AthenaService {
// Submit async — caller gets the ID immediately while execution runs in background
vertx.executeBlocking(() -> {
+ // Athena rejects a query that leaves an injected partition column unconstrained. It fails
+ // the query rather than the submission, which is what throwing here produces.
+ PartitionProjection.assertInjectedColumnsFiltered(query, tablesForProjectionCheck(database));
String setupDdl = ddlBuilder.build(database);
if (outputLocation != null) {
ensureOutputBucket(outputLocation);
@@ -163,6 +166,20 @@ public class AthenaService {
return id;
}
+ /** Tables of the query's database, or none when the catalog cannot answer. */
+ private List<Table> tablesForProjectionCheck(String database) {
+ if (database == null || database.isBlank()) {
+ return List.of();
+ }
+ try {
+ List<Table> tables = glueService.getTables(database);
+ return tables == null ? List.of() : tables;
+ } catch (Exception e) {
+ LOG.debugv("Could not fetch tables for projection check on {0}: {1}", database, e.getMessage());
+ return List.of();
+ }
+ }
+
public QueryExecution getQueryExecution(String id) {
return queryStore.get(id)
.orElseThrow(() -> new AwsException("InvalidRequestException",
diff --git a/src/main/java/io/github/hectorvent/floci/services/athena/GlueViewDdlBuilder.java b/src/main/java/io/github/hectorvent/floci/services/athena/GlueViewDdlBuilder.java
index 0ac8693598..0cf84cbeb0 100644
--- a/src/main/java/io/github/hectorvent/floci/services/athena/GlueViewDdlBuilder.java
+++ b/src/main/java/io/github/hectorvent/floci/services/athena/GlueViewDdlBuilder.java
@@ -87,6 +87,7 @@ public class GlueViewDdlBuilder {
? location.substring(0, location.length() - 1)
: location;
String readFn = inferReadFunction(t);
+ String readPath = PartitionProjection.readPath(t, normalizedLocation);
String target = qualified
? quote(schemaOrNull) + "." + quote(t.getName())
: quote(t.getName());
@@ -95,7 +96,7 @@ public class GlueViewDdlBuilder {
.append(" AS SELECT ")
.append(buildProjection(t))
.append(" FROM ")
- .append(readExpression(readFn, normalizedLocation))
+ .append(readExpression(readFn, readPath))
.append(";\n");
} catch (Exception e) {
LOG.debugv("skip Glue table {0}.{1}: {2}", schemaOrNull, t != null ? t.getName() : "unknown", e.getMessage());
diff --git a/src/main/java/io/github/hectorvent/floci/services/athena/PartitionProjection.java b/src/main/java/io/github/hectorvent/floci/services/athena/PartitionProjection.java
new file mode 100644
index 0000000000..bb3d61d211
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/athena/PartitionProjection.java
@@ -0,0 +1,258 @@
+package io.github.hectorvent.floci.services.athena;
+
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.services.glue.model.Column;
+import io.github.hectorvent.floci.services.glue.model.Table;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Partition projection: the table properties describing where a table's partitions live and what
+ * values they take, so the partitions need not be listed from the catalog or from storage.
+ */
+final class PartitionProjection {
+
+ /** Table property switching partition projection on. */
+ private static final String ENABLED = "projection.enabled";
+
+ /** Table property giving the partition path layout when it is not the default. */
+ private static final String LOCATION_TEMPLATE = "storage.location.template";
+
+ /** {@code ${column}} placeholders in a partition location template. */
+ private static final Pattern TEMPLATE_PLACEHOLDER = Pattern.compile("\\$\\{[^}]*}");
+
+ /**
+ * Not part of a longer identifier, and not a quoted one either: {@code "limit"} is a column
+ * named after a keyword, and a clause keyword is never written in quotes.
+ */
+ private static final String KEYWORD_BOUNDS = "(?<![A-Za-z0-9_\"`])%s(?![A-Za-z0-9_\"`])";
+
+ /** The {@code WHERE} keyword, as a keyword rather than as an identifier spelling it. */
+ private static final Pattern WHERE_KEYWORD = Pattern.compile(
+ KEYWORD_BOUNDS.formatted("WHERE"), Pattern.CASE_INSENSITIVE);
+
+ /** The keywords that can end a {@code WHERE} clause by starting the next one. */
+ private static final Pattern CLAUSE_AFTER_WHERE = Pattern.compile(
+ KEYWORD_BOUNDS.formatted(
+ "(?:GROUP|HAVING|ORDER|WINDOW|LIMIT|OFFSET|FETCH|UNION|INTERSECT|EXCEPT)"),
+ Pattern.CASE_INSENSITIVE);
+
+ private PartitionProjection() {
+ }
+
+ static boolean enabled(Table table) {
+ return table != null && "true".equalsIgnoreCase(parameter(table, ENABLED));
+ }
+
+ /** Case-insensitive lookup, since table properties are not normalised on the way in. */
+ static String parameter(Table table, String name) {
+ Map<String, String> params = table == null ? null : table.getParameters();
+ if (params == null) {
+ return null;
+ }
+ for (Map.Entry<String, String> e : params.entrySet()) {
+ if (e.getKey() != null && e.getKey().equalsIgnoreCase(name)) {
+ return e.getValue();
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Returns the path to read for {@code table}: its partition layout when the table projects its
+ * partitions, and {@code normalizedLocation} unchanged otherwise.
+ *
+ * <p>A projecting table's data lives only under its partition paths, and those are what Athena
+ * reads - it builds them from the projection configuration rather than listing the location.
+ * Reading the location wholesale instead picks up whatever else is stored beside the data, which
+ * for a table rooted at a bucket means that bucket's other prefixes. A workgroup writing query
+ * results under the same root is the common way this bites: the first query succeeds, its output
+ * lands inside the read path, and the next query fails trying to read it as table data.
+ *
+ * <p>The layout comes from {@code storage.location.template} when set, and otherwise from the
+ * default {@code column=value} form over the partition keys. Values stay wildcards: narrowing
+ * them to the partitions a query actually needs requires that query's predicates, so the read
+ * stays wide and the query itself does the filtering.
+ */
+ static String readPath(Table table, String normalizedLocation) {
+ if (!enabled(table)) {
+ return normalizedLocation;
+ }
+
+ String template = parameter(table, LOCATION_TEMPLATE);
+ if (template != null && !template.isBlank()) {
+ String wildcarded = TEMPLATE_PLACEHOLDER.matcher(template).replaceAll("*");
+ return stripTrailingSlash(wildcarded);
+ }
+
+ List<Column> keys = table.getPartitionKeys();
+ if (keys == null || keys.isEmpty()) {
+ return normalizedLocation;
+ }
+ StringBuilder sb = new StringBuilder(normalizedLocation);
+ for (Column key : keys) {
+ if (key == null || key.getName() == null || key.getName().isBlank()) {
+ return normalizedLocation;
+ }
+ sb.append('/').append(key.getName()).append("=*");
+ }
+ return sb.toString();
+ }
+
+ /**
+ * Fails a query that names a projecting table but never constrains one of its {@code injected}
+ * partition columns in a {@code WHERE} clause.
+ *
+ * <p>An injected column has no generatable range: its values come from the query, so Athena
+ * requires a static equality condition on it in the {@code WHERE} clause and rejects the query
+ * otherwise. Deciding whether a condition is a <em>static equality</em> one needs the query's
+ * predicate tree, which is not available here, so this checks only whether the column is
+ * mentioned inside a {@code WHERE} clause - any of them, including a subquery's. That is
+ * strictly narrower than the real rule - a query filtering an injected column with, say, a
+ * range condition is rejected by Athena and accepted here - but it never rejects a query Athena
+ * would accept, and it catches the case that silently reads every partition.
+ *
+ * <p>Scoping to the {@code WHERE} clause is what makes the check worth having. A mention
+ * anywhere in the query text also matches the column's own appearance in a {@code SELECT} list
+ * or a {@code GROUP BY}, so {@code SELECT tenant, count(*) FROM audit_events GROUP BY tenant}
+ * would pass while filtering nothing at all - exactly the case this exists to catch.
+ */
+ static void assertInjectedColumnsFiltered(String query, List<Table> tables) {
+ if (query == null || tables == null) {
+ return;
+ }
+ // Literals and comments are not code: a column name inside either constrains nothing.
+ String code = maskLiteralsAndComments(query);
+ List<String> whereClauses = null;
+ for (Table table : tables) {
+ if (!enabled(table) || table.getName() == null || table.getPartitionKeys() == null) {
+ continue;
+ }
+ // A table the query never names cannot be constrained by it, and must not fail it.
+ if (!mentions(code, table.getName())) {
+ continue;
+ }
+ for (Column key : table.getPartitionKeys()) {
+ if (key == null || key.getName() == null) {
+ continue;
+ }
+ String type = parameter(table, "projection." + key.getName() + ".type");
+ if (!"injected".equalsIgnoreCase(type)) {
+ continue;
+ }
+ if (whereClauses == null) {
+ whereClauses = whereClauses(code);
+ }
+ if (whereClauses.stream().anyMatch(clause -> mentions(clause, key.getName()))) {
+ continue;
+ }
+ throw new AwsException("InvalidRequestException",
+ "CONSTRAINT_VIOLATION: For the injected projected partition column "
+ + key.getName() + ", the WHERE clause must contain only static equality "
+ + "conditions, and at least one such condition must be present.", 400);
+ }
+ }
+ }
+
+ /**
+ * The text of every {@code WHERE} clause in {@code code}, a subquery's included.
+ *
+ * <p>A clause runs from the keyword to whatever ends it: a following clause keyword, the
+ * {@code )} closing the subquery that holds it, a statement terminator, or the end of the
+ * query. Parentheses opened inside the clause are part of it, so a predicate with its own
+ * subquery is not cut short.
+ */
+ private static List<String> whereClauses(String code) {
+ Set<Integer> clauseStarts = new HashSet<>();
+ Matcher boundaries = CLAUSE_AFTER_WHERE.matcher(code);
+ while (boundaries.find()) {
+ clauseStarts.add(boundaries.start());
+ }
+
+ List<String> clauses = new ArrayList<>();
+ Matcher where = WHERE_KEYWORD.matcher(code);
+ while (where.find()) {
+ int start = where.end();
+ int depth = 0;
+ int i = start;
+ while (i < code.length()) {
+ char c = code.charAt(i);
+ if (c == '(') {
+ depth++;
+ } else if (c == ')') {
+ if (depth == 0) {
+ break;
+ }
+ depth--;
+ } else if (depth == 0 && (c == ';' || clauseStarts.contains(i))) {
+ break;
+ }
+ i++;
+ }
+ clauses.add(code.substring(start, i));
+ }
+ return clauses;
+ }
+
+ /**
+ * Blanks out single-quoted literals and comments, leaving everything else - and every index -
+ * where it was. Double-quoted and backquoted identifiers stay: those are references to a column,
+ * not text that happens to spell its name.
+ */
+ private static String maskLiteralsAndComments(String sql) {
+ char[] chars = sql.toCharArray();
+ int i = 0;
+ while (i < chars.length) {
+ int end;
+ if (chars[i] == '\'') {
+ end = i + 1;
+ while (end < chars.length) {
+ if (chars[end] != '\'') {
+ end++;
+ } else if (end + 1 < chars.length && chars[end + 1] == '\'') {
+ end += 2; // an escaped quote, still inside the literal
+ } else {
+ end++;
+ break;
+ }
+ }
+ } else if (chars[i] == '-' && i + 1 < chars.length && chars[i + 1] == '-') {
+ end = i;
+ while (end < chars.length && chars[end] != '\n') {
+ end++;
+ }
+ } else if (chars[i] == '/' && i + 1 < chars.length && chars[i + 1] == '*') {
+ end = i + 2;
+ while (end + 1 < chars.length && !(chars[end] == '*' && chars[end + 1] == '/')) {
+ end++;
+ }
+ end = Math.min(chars.length, end + 2);
+ } else {
+ i++;
+ continue;
+ }
+ for (int k = i; k < end; k++) {
+ if (chars[k] != '\n') {
+ chars[k] = ' ';
+ }
+ }
+ i = end;
+ }
+ return new String(chars);
+ }
+
+ private static boolean mentions(String code, String identifier) {
+ return Pattern.compile("(?<![A-Za-z0-9_])" + Pattern.quote(identifier) + "(?![A-Za-z0-9_])",
+ Pattern.CASE_INSENSITIVE).matcher(code).find();
+ }
+
+ private static String stripTrailingSlash(String value) {
+ return value.endsWith("/") ? value.substring(0, value.length() - 1) : value;
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/athena/PartitionProjectionTest.java b/src/test/java/io/github/hectorvent/floci/services/athena/PartitionProjectionTest.java
new file mode 100644
index 0000000000..ce8815a129
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/athena/PartitionProjectionTest.java
@@ -0,0 +1,210 @@
+package io.github.hectorvent.floci.services.athena;
+
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.services.glue.model.Column;
+import io.github.hectorvent.floci.services.glue.model.StorageDescriptor;
+import io.github.hectorvent.floci.services.glue.model.Table;
+import org.junit.jupiter.api.Test;
+
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class PartitionProjectionTest {
+
+ private static final String BUCKET = "s3://audit-logs";
+
+ private static Column column(String name) {
+ Column c = new Column();
+ c.setName(name);
+ c.setType("string");
+ return c;
+ }
+
+ private static Table table(String name, Map<String, String> params, List<Column> partitionKeys) {
+ Table t = new Table();
+ t.setName(name);
+ StorageDescriptor sd = new StorageDescriptor();
+ sd.setLocation(BUCKET + "/");
+ t.setStorageDescriptor(sd);
+ t.setParameters(params);
+ t.setPartitionKeys(partitionKeys);
+ return t;
+ }
+
+ private static Map<String, String> projecting(String... extra) {
+ Map<String, String> m = new LinkedHashMap<>();
+ m.put("projection.enabled", "true");
+ for (int i = 0; i + 1 < extra.length; i += 2) {
+ m.put(extra[i], extra[i + 1]);
+ }
+ return m;
+ }
+
+ // ---- read path ----
+
+ /**
+ * A workgroup writing results under the table's root is the case that breaks: reading the root
+ * wholesale picks the results back up as though they were table data.
+ */
+ @Test
+ void templateGivesAPartitionPathThatExcludesSiblingPrefixes() {
+ Table t = table("audit_events", projecting(
+ "storage.location.template", BUCKET + "/tenant=${tenant}/year=${year}/month=${month}/day=${day}"),
+ List.of(column("tenant"), column("year"), column("month"), column("day")));
+
+ String path = PartitionProjection.readPath(t, BUCKET);
+
+ assertEquals(BUCKET + "/tenant=*/year=*/month=*/day=*", path);
+ assertTrue(path.startsWith(BUCKET + "/tenant="), path);
+ }
+
+ /** Without a template the default column=value layout is built from the partition keys. */
+ @Test
+ void partitionKeysGiveTheDefaultLayoutWhenNoTemplateIsSet() {
+ Table t = table("t", projecting(), List.of(column("tenant"), column("year")));
+
+ assertEquals(BUCKET + "/tenant=*/year=*", PartitionProjection.readPath(t, BUCKET));
+ }
+
+ /** Projection off means the location is read as before. */
+ @Test
+ void withoutProjectionTheLocationIsUnchanged() {
+ Table t = table("t", Map.of(), List.of(column("tenant")));
+
+ assertEquals(BUCKET, PartitionProjection.readPath(t, BUCKET));
+ }
+
+ /** A projecting table declaring no partitions has no layout to build. */
+ @Test
+ void projectionWithoutPartitionKeysLeavesTheLocationUnchanged() {
+ assertEquals(BUCKET, PartitionProjection.readPath(table("t", projecting(), null), BUCKET));
+ }
+
+ @Test
+ void trailingSlashOnTheTemplateIsDropped() {
+ Table t = table("t", projecting("storage.location.template", BUCKET + "/tenant=${tenant}/"),
+ List.of(column("tenant")));
+
+ assertEquals(BUCKET + "/tenant=*", PartitionProjection.readPath(t, BUCKET));
+ }
+
+ // ---- injected columns ----
+
+ private Table injectedTenantTable() {
+ return table("audit_events", projecting("projection.tenant.type", "injected"),
+ List.of(column("tenant")));
+ }
+
+ @Test
+ void queryWithoutAnyMentionOfAnInjectedColumnIsRejected() {
+ AwsException e = assertThrows(AwsException.class, () ->
+ PartitionProjection.assertInjectedColumnsFiltered(
+ "SELECT * FROM audit_events", List.of(injectedTenantTable())));
+
+ assertTrue(e.getMessage().startsWith("CONSTRAINT_VIOLATION:"), e.getMessage());
+ assertTrue(e.getMessage().contains("tenant"), e.getMessage());
+ }
+
+ @Test
+ void queryConstrainingTheInjectedColumnIsAccepted() {
+ assertDoesNotThrow(() -> PartitionProjection.assertInjectedColumnsFiltered(
+ "SELECT * FROM audit_events WHERE tenant = 'abc'", List.of(injectedTenantTable())));
+ }
+
+ /** A table the query never names must not fail it. */
+ @Test
+ void unrelatedQueryAgainstAnotherTableIsUnaffected() {
+ assertDoesNotThrow(() -> PartitionProjection.assertInjectedColumnsFiltered(
+ "SELECT * FROM other_table", List.of(injectedTenantTable())));
+ }
+
+ /** A non-injected projected column generates its own values and needs no filter. */
+ @Test
+ void nonInjectedPartitionColumnsNeedNoFilter() {
+ Table t = table("t", projecting("projection.year.type", "integer"), List.of(column("year")));
+
+ assertDoesNotThrow(() -> PartitionProjection.assertInjectedColumnsFiltered("SELECT * FROM t", List.of(t)));
+ }
+
+ /** Identifiers match on word boundaries, so a longer name that contains one does not count. */
+ @Test
+ void aLongerIdentifierContainingTheColumnNameDoesNotCountAsAMention() {
+ assertThrows(AwsException.class, () -> PartitionProjection.assertInjectedColumnsFiltered(
+ "SELECT * FROM audit_events WHERE tenant_id = 'abc'", List.of(injectedTenantTable())));
+ }
+
+ /**
+ * The column names itself in the SELECT list and the GROUP BY, and filters nothing - the case
+ * the check exists to catch, and the one a query-wide text match lets through.
+ */
+ @Test
+ void mentionOutsideAWhereClauseDoesNotCount() {
+ assertThrows(AwsException.class, () -> PartitionProjection.assertInjectedColumnsFiltered(
+ "SELECT tenant, count(*) FROM audit_events GROUP BY tenant",
+ List.of(injectedTenantTable())));
+ }
+
+ /** A WHERE clause ends where the next one starts, so an ORDER BY mention is outside it. */
+ @Test
+ void mentionAfterTheWhereClauseEndsDoesNotCount() {
+ assertThrows(AwsException.class, () -> PartitionProjection.assertInjectedColumnsFiltered(
+ "SELECT * FROM audit_events WHERE year = 2026 ORDER BY tenant",
+ List.of(injectedTenantTable())));
+ }
+
+ /** A string that happens to spell the column name constrains nothing. */
+ @Test
+ void mentionInsideAStringLiteralDoesNotCount() {
+ assertThrows(AwsException.class, () -> PartitionProjection.assertInjectedColumnsFiltered(
+ "SELECT * FROM audit_events WHERE note = 'tenant'", List.of(injectedTenantTable())));
+ }
+
+ /** Nor does one in a comment. */
+ @Test
+ void mentionInsideACommentDoesNotCount() {
+ assertThrows(AwsException.class, () -> PartitionProjection.assertInjectedColumnsFiltered(
+ "SELECT * FROM audit_events -- filter by tenant one day\n WHERE year = 2026",
+ List.of(injectedTenantTable())));
+ assertThrows(AwsException.class, () -> PartitionProjection.assertInjectedColumnsFiltered(
+ "SELECT * FROM audit_events /* tenant */ WHERE year = 2026",
+ List.of(injectedTenantTable())));
+ }
+
+ /** A subquery's WHERE is still a WHERE, and still constrains the read. */
+ @Test
+ void filterInASubqueryCounts() {
+ assertDoesNotThrow(() -> PartitionProjection.assertInjectedColumnsFiltered(
+ "SELECT tenant, count(*) FROM (SELECT * FROM audit_events WHERE tenant = 'abc') "
+ + "GROUP BY tenant",
+ List.of(injectedTenantTable())));
+ }
+
+ /** A predicate carrying its own subquery is not cut short at the inner parentheses. */
+ @Test
+ void aPredicateWithItsOwnSubqueryIsReadWhole() {
+ assertDoesNotThrow(() -> PartitionProjection.assertInjectedColumnsFiltered(
+ "SELECT * FROM audit_events WHERE year IN (SELECT y FROM years) AND tenant = 'abc'",
+ List.of(injectedTenantTable())));
+ }
+
+ /** Double quotes mark an identifier in Athena, so a quoted column reference counts. */
+ @Test
+ void aQuotedColumnReferenceCounts() {
+ assertDoesNotThrow(() -> PartitionProjection.assertInjectedColumnsFiltered(
+ "SELECT * FROM audit_events WHERE \"tenant\" = 'abc'", List.of(injectedTenantTable())));
+ }
+
+ /** A column named after a clause keyword is quoted, and must not end the clause it sits in. */
+ @Test
+ void aQuotedIdentifierSpellingAClauseKeywordDoesNotEndTheClause() {
+ assertDoesNotThrow(() -> PartitionProjection.assertInjectedColumnsFiltered(
+ "SELECT * FROM audit_events WHERE \"limit\" = 10 AND tenant = 'abc'",
+ List.of(injectedTenantTable())));
+ }
+}