:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(warehouse): implement aws athena source (#118198)
Daniel RC0cfbd7992d89
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/SOURCES.md b/products/warehouse_sources/backend/temporal/data_imports/sources/SOURCES.mdindex 84cf54fb57ee..580e4affbe4a 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/SOURCES.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/SOURCES.md@@ -111,6 +111,7 @@ the row lists both.| aviationstack | HTTP | requests | ✅ || aviator | HTTP | requests | ✅ || awin | HTTP | requests | ✅ |+| aws_athena | HTTP | requests + botocore (SigV4 signing, model-driven serializer) | ✅ || aws_batch | HTTP | requests | ✅ || aws_budgets | HTTP | requests | ✅ || aws_cloudtrail | HTTP | requests | ✅ |@@ -1054,7 +1055,6 @@ doesn't conflict with concurrent PRs.- audiogo- autodesk_construction_cloud- automox-- aws_athena- aws_cloudformation- aws_connect- aws_healthdiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_athena/athena.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_athena/athena.pynew file mode 100644index 000000000000..6266467121b3--- /dev/null+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_athena/athena.py@@ -0,0 +1,689 @@+import re+import time+import uuid+import datetime as dt+import dataclasses+from collections.abc import Callable, Iterator+from contextlib import contextmanager+from decimal import Decimal+from functools import cache+from typing import Any, Optional+from zoneinfo import ZoneInfo++import pyarrow as pa+import requests+from botocore.auth import SigV4Auth+from botocore.awsrequest import AWSRequest+from botocore.credentials import Credentials+from botocore.loaders import Loader+from botocore.model import ServiceModel+from botocore.parsers import create_parser+from botocore.serialize import create_serializer+from structlog.types import FilteringBoundLogger+from tenacity import (+ RetryCallState,+ retry,+ retry_if_exception_type,+ stop_after_attempt,+ stop_before_delay,+ wait_exponential_jitter,+)++from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import (+ BinaryColumnReporter,+ restrict_schema_to_columns,+ table_from_iterator,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.aws_athena.settings import (+ ATHENA_SERVICE,+ DATA_CATALOG,+ DEFAULT_WORKGROUP,+ GLUE_PAGE_SIZE,+ GLUE_SERVICE,+ POLL_INITIAL_SECONDS,+ POLL_MAX_SECONDS,+ QUERY_DEADLINE_SECONDS,+ RESULTS_PAGE_SIZE,+ AwsService,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http.transport import NO_RETRY+from products.warehouse_sources.backend.temporal.data_imports.sources.common.interruptible_wait import (+ interruptible_wait,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.primary_keys import resolve_merge_keys+from products.warehouse_sources.backend.temporal.data_imports.sources.common.safe_point import reach_safe_point+from products.warehouse_sources.backend.temporal.data_imports.sources.common.sql import (+ AnsiIdentifierQuoter,+ Column,+ Table,+ resolve_table_projection,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.sql.implementation import (+ SQLSourceImplementation,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.sql.incremental import (+ IncrementalFieldFilter,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.sql.location import resolve_source_location+from products.warehouse_sources.backend.temporal.data_imports.sources.common.sql.query_builder import (+ ParamStyle,+ SafeSQL,+ SelectQueryBuilder,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse+from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.awsathena import (+ AwsAthenaSourceConfig,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.trino.trino import (+ filter_trino_incremental_fields,+ incremental_value_for_trino_column,+ trino_type_to_arrow,+)++# (connect, read) seconds for one try.+REQUEST_TIMEOUT = (10, 60)+MAX_ATTEMPTS = 6+# No new try starts after this, so one call with its tries and waits stays under five minutes.+RETRY_BUDGET_SECONDS = 180+RETRYABLE_STATUSES = frozenset({429, 500, 502, 503, 504})+ERROR_PREFIX = "AWS Athena request failed"+QUERY_ERROR_PREFIX = "AWS Athena query failed"+TABLE_NOT_FOUND = "TableNotFound"+TABLE_NOT_FOUND_MESSAGE = (+ "This table no longer exists in Glue, or it has no columns. Refresh the schemas, then turn the table back on."+)++# AWS answers these as HTTP 400 with the code in the body, so the transport cannot see them.+THROTTLING_ERROR_CODES = frozenset({"ThrottlingException", "TooManyRequestsException", "ThrottlingError"})++# Athena's `AthenaError.ErrorCategory`: 1 is a system error, 2 a user error, 3 anything else.+ATHENA_USER_ERROR_CATEGORY = 2+# The widest fixed-point numeric literal to render, which covers DECIMAL(38, 38) with room to spare.+MAX_DECIMAL_LITERAL_DIGITS = 100+++class AwsAthenaError(Exception):+ def __init__(self, message: str, code: str | None = None) -> None:+ super().__init__(message)+ self.code = code+++class AwsAthenaTransientError(AwsAthenaError):+ pass+++class AwsAthenaThrottledError(AwsAthenaTransientError):+ pass+++class AwsAthenaQueryEndedError(AwsAthenaError):+ """Athena ended the query itself, so there is nothing left to stop."""+++@cache+def aws_dns_suffix(region: str) -> Optional[str]:+ """The DNS suffix of the AWS partition a region belongs to, or None for an unknown region.++ China regions live under `amazonaws.com.cn`, so the suffix comes from botocore's partition table.+ """+ for partition in Loader().load_data("partitions")["partitions"]:+ if region in partition.get("regions", {}) or re.match(partition["regionRegex"], region):+ return partition["outputs"]["dnsSuffix"]+ return None+++@cache+def _service_model(service: AwsService) -> ServiceModel:+ return ServiceModel(Loader().load_service_model(service, "service-2"), service_name=service)+++def _error_from(parsed: dict[str, Any], status_code: int) -> AwsAthenaError:+ error = parsed.get("Error") or {}+ code = str(error.get("Code") or f"HTTP {status_code}").split("#")[-1]+ message = str(error.get("Message") or "")[:500]+ text = f"{ERROR_PREFIX}: {code} - {message}"+ if code in THROTTLING_ERROR_CODES:+ return AwsAthenaThrottledError(text, code)+ if status_code in RETRYABLE_STATUSES:+ return AwsAthenaTransientError(text, code)+ return AwsAthenaError(text, code)+++def _before_retry_wait(_state: RetryCallState) -> None:+ reach_safe_point()+++def _retry_wait(seconds: float) -> None:+ interruptible_wait(seconds)+++def send_once(+ session: requests.Session,+ credentials: Credentials,+ region: str,+ service: AwsService,+ operation: str,+ params: dict[str, Any],+) -> dict[str, Any]:+ """Serialize one call from its botocore model, sign it with SigV4 and send it once over the tracked session."""+ model = _service_model(service)+ operation_model = model.operation_model(operation)+ serialized = create_serializer(model.protocol, include_validation=False).serialize_to_request(+ params, operation_model+ )+ body = serialized["body"]+ if isinstance(body, str):+ body = body.encode()+ suffix = aws_dns_suffix(region)+ if suffix is None:+ raise ValueError(f"Unknown AWS region: {region}")+ url = f"https://{model.endpoint_prefix}.{region}.{suffix}{serialized['url_path']}"++ aws_request = AWSRequest(method="POST", url=url, data=body, headers=dict(serialized["headers"]))+ SigV4Auth(credentials, model.signing_name, region).add_auth(aws_request)++ response = session.post(url, data=body, headers=dict(aws_request.headers.items()), timeout=REQUEST_TIMEOUT)++ output_shape = operation_model.output_shape+ assert output_shape is not None # every operation this source calls returns a body+ parsed = create_parser(model.protocol).parse(+ {"status_code": response.status_code, "headers": dict(response.headers), "body": response.content},+ output_shape,+ )+ if response.status_code >= 400 or "Error" in parsed:+ raise _error_from(parsed, response.status_code)+ parsed.pop("ResponseMetadata", None)+ return parsed+++# The only retry layer: the session sends each try once, so tries and waits do not multiply.+@retry(+ retry=retry_if_exception_type((AwsAthenaTransientError, requests.ConnectionError, requests.Timeout)),+ stop=stop_after_attempt(MAX_ATTEMPTS) | stop_before_delay(RETRY_BUDGET_SECONDS),+ wait=wait_exponential_jitter(initial=2, max=30),+ sleep=_retry_wait,+ before_sleep=_before_retry_wait,+ reraise=True,+)+def send_operation(+ session: requests.Session,+ credentials: Credentials,+ region: str,+ service: AwsService,+ operation: str,+ params: dict[str, Any],+) -> dict[str, Any]:+ return send_once(session, credentials, region, service, operation, params)+++class AthenaClient:+ def __init__(+ self,+ aws_access_key_id: str,+ aws_secret_access_key: str,+ aws_session_token: Optional[str],+ aws_region: str,+ ) -> None:+ redact = tuple(value for value in (aws_secret_access_key, aws_session_token) if value)+ self._session = make_tracked_session(retry=NO_RETRY, redact_values=redact, allow_redirects=False)+ self._credentials = Credentials(aws_access_key_id, aws_secret_access_key, aws_session_token or None)+ self._region = aws_region.strip()++ def call(self, service: AwsService, operation: str, params: dict[str, Any]) -> dict[str, Any]:+ return send_operation(self._session, self._credentials, self._region, service, operation, params)++ def close(self) -> None:+ self._session.close()++ def stop_query(self, execution_id: str) -> None:+ """One try with no retry wait, because the caller is often on its way out of the run."""+ send_once(+ self._session,+ self._credentials,+ self._region,+ ATHENA_SERVICE,+ "StopQueryExecution",+ {"QueryExecutionId": execution_id},+ )+++def make_client(config: AwsAthenaSourceConfig) -> AthenaClient:+ return AthenaClient(+ config.aws_access_key_id, config.aws_secret_access_key, config.aws_session_token, config.aws_region+ )+++# Glue stores Hive type names. Athena's query engine is Trino, so they are mapped onto Trino's+# names and the Trino source's type handling is reused.+_HIVE_TO_TRINO = {"string": "varchar", "float": "real", "int": "integer", "binary": "varbinary"}+++def athena_type(glue_type: str) -> str:+ normalized = " ".join(glue_type.strip().lower().split())+ if normalized.startswith(("array<", "map<", "struct<", "uniontype<")):+ return "json"+ return _HIVE_TO_TRINO.get(normalized, normalized)+++class AthenaColumn(Column):+ def __init__(self, name: str, data_type: str, nullable: bool = True) -> None:+ self.name = name+ self.data_type = data_type+ self.nullable = nullable++ def to_arrow_field(self) -> pa.Field[pa.DataType]:+ return pa.field(self.name, trino_type_to_arrow(self.data_type), nullable=self.nullable)+++def _glue_columns(table: dict[str, Any]) -> list[AthenaColumn]:+ # Partition keys are queryable columns too; Glue keeps them apart from the storage columns.+ glue_columns = [+ *((table.get("StorageDescriptor") or {}).get("Columns") or []),+ *(table.get("PartitionKeys") or []),+ ]+ return [+ AthenaColumn(str(column["Name"]), athena_type(str(column.get("Type") or "string"))) for column in glue_columns+ ]+++def list_glue_tables(client: AthenaClient, database: str, names: Optional[list[str]]) -> dict[str, list[AthenaColumn]]:+ tables: dict[str, list[AthenaColumn]] = {}+ if names is not None:+ # A large catalog takes hundreds of GetTables pages, so named tables are fetched one by one.+ for name in names:+ try:+ body = client.call(GLUE_SERVICE, "GetTable", {"DatabaseName": database, "Name": name})+ except AwsAthenaError as error:+ if error.code == "EntityNotFoundException":+ continue+ raise+ if columns := _glue_columns(body.get("Table") or {}):+ tables[name] = columns+ return tables+ next_token: Optional[str] = None+ while True:+ params: dict[str, Any] = {"DatabaseName": database, "MaxResults": GLUE_PAGE_SIZE}+ if next_token:+ params["NextToken"] = next_token+ body = client.call(GLUE_SERVICE, "GetTables", params)+ for table in body.get("TableList") or []:+ name = str(table.get("Name") or "")+ columns = _glue_columns(table)+ # A table Glue lists with no columns (a Spark-registered Delta table, for one) can never be queried.+ if name and columns:+ tables[name] = columns+ next_token = body.get("NextToken")+ if not next_token:+ return tables+++def get_glue_table(client: AthenaClient, database: str, table_name: str) -> Table[AthenaColumn]:+ missing = AwsAthenaError(f"{ERROR_PREFIX}: {TABLE_NOT_FOUND} - table {table_name}", TABLE_NOT_FOUND)+ try:+ body = client.call(GLUE_SERVICE, "GetTable", {"DatabaseName": database, "Name": table_name})+ except AwsAthenaError as error:+ if error.code == "EntityNotFoundException":+ raise missing from error+ raise+ columns = _glue_columns(body.get("Table") or {})+ if not columns:+ raise missing+ return Table(name=table_name, parents=(database,), columns=columns)+++def _decimal_literal(value: Decimal) -> str:+ """Render a Decimal in fixed point, after refusing one that has no Athena literal.++ A row filter value arrives as text and becomes a Decimal, so `1e1000000000` and `NaN` both get+ this far. Athena's widest exact numeric is DECIMAL(38, 38) and its literals carry no exponent,+ so a value this wide matches no row; writing it out would allocate a character per exponent.+ """+ if not value.is_finite():+ raise ValueError(f"Cannot pass {value} as an Athena query parameter")+ _, digits, exponent = value.as_tuple()+ width = len(digits) + abs(int(exponent))+ if width > MAX_DECIMAL_LITERAL_DIGITS:+ raise ValueError(+ f"A numeric value of {width} digits is wider than the {MAX_DECIMAL_LITERAL_DIGITS} an "+ "Athena numeric literal holds. Use a value Athena can compare against the column."+ )+ return format(value, "f")+++def athena_literal(value: Any) -> str:+ """Render one query parameter as the SQL literal Athena's `ExecutionParameters` expect.++ Athena substitutes each parameter as literal text, so a string has to arrive quoted and+ escaped. Only values the query builder produces reach here: the watermark and row filter values.+ """+ if value is None:+ return "NULL"+ if isinstance(value, bool):+ return "TRUE" if value else "FALSE"+ if isinstance(value, int):+ return str(value)+ if isinstance(value, Decimal):+ return _decimal_literal(value)+ if isinstance(value, float):+ return repr(value)+ if isinstance(value, dt.datetime):+ if value.tzinfo is not None:+ return f"TIMESTAMP '{value.astimezone(dt.UTC):%Y-%m-%d %H:%M:%S.%f} UTC'"+ return f"TIMESTAMP '{value:%Y-%m-%d %H:%M:%S.%f}'"+ if isinstance(value, dt.date):+ return f"DATE '{value.isoformat()}'"+ if isinstance(value, str):+ return "'" + value.replace("'", "''") + "'"+ raise TypeError(f"Cannot pass a {type(value).__name__} as an Athena query parameter")+++def _parse_timestamp(text: str, with_zone: bool) -> dt.datetime:+ # A zoned value prints as `2024-01-15 10:30:00.123 UTC` or `... +05:30`, and fromisoformat takes+ # neither tail. Glue declares an Iceberg `timestamptz` as plain `timestamp`, so a naive column can+ # carry one too.+ value = text.strip()+ zone: dt.tzinfo | None = None+ head, separator, tail = value.rpartition(" ")+ if separator and " " in head and tail:+ if tail[0] in "+-":+ value = head + tail+ elif tail[0].isalpha():+ value, zone = head, dt.UTC if tail.upper() == "UTC" else ZoneInfo(tail)+ parsed = dt.datetime.fromisoformat(value)+ if zone is not None and parsed.tzinfo is None:+ parsed = parsed.replace(tzinfo=zone)+ if parsed.tzinfo is None:+ return parsed+ in_utc = parsed.astimezone(dt.UTC)+ return in_utc if with_zone else in_utc.replace(tzinfo=None)+++def value_parser(arrow_type: pa.DataType) -> Optional[Callable[[str], Any]]:+ """How to read Athena's text form of a value into the Arrow type the column lands as."""+ if pa.types.is_integer(arrow_type):+ return int+ if pa.types.is_floating(arrow_type):+ return float+ if pa.types.is_decimal(arrow_type):+ return Decimal+ if pa.types.is_boolean(arrow_type):+ return lambda text: text.strip().lower() == "true"+ if pa.types.is_date(arrow_type):+ return dt.date.fromisoformat+ if pa.types.is_timestamp(arrow_type):+ with_zone = arrow_type.tz is not None+ return lambda text: _parse_timestamp(text, with_zone)+ if pa.types.is_time(arrow_type):+ return dt.time.fromisoformat+ if pa.types.is_binary(arrow_type):+ # Athena prints varbinary as space-separated hex.+ return lambda text: bytes.fromhex(text.replace(" ", ""))+ return None++[email protected](frozen=True)+class QueryLocation:+ database: str+ workgroup: str+ output_location: Optional[str]+++def start_query(client: AthenaClient, query: SafeSQL, location: QueryLocation) -> str:+ params: dict[str, Any] = {+ "QueryString": query.sql,+ "QueryExecutionContext": {"Database": location.database, "Catalog": DATA_CATALOG},+ "WorkGroup": location.workgroup,+ # SDKs fill this in, but a direct call must send it. Retries of this call reuse it, so a+ # retried start does not run the query twice.+ "ClientRequestToken": str(uuid.uuid4()),+ }+ if query.params:+ params["ExecutionParameters"] = [athena_literal(value) for value in query.params]+ if location.output_location:+ params["ResultConfiguration"] = {"OutputLocation": location.output_location}+ return str(client.call(ATHENA_SERVICE, "StartQueryExecution", params)["QueryExecutionId"])+++def wait_for_query(client: AthenaClient, execution_id: str) -> None:+ deadline = time.monotonic() + QUERY_DEADLINE_SECONDS+ delay = POLL_INITIAL_SECONDS+ while True:+ execution = client.call(ATHENA_SERVICE, "GetQueryExecution", {"QueryExecutionId": execution_id})+ status = (execution.get("QueryExecution") or {}).get("Status") or {}+ state = status.get("State")+ if state == "SUCCEEDED":+ return+ if state in ("FAILED", "CANCELLED"):+ athena_error = status.get("AthenaError") or {}+ kind = "USER_ERROR" if athena_error.get("ErrorCategory") == ATHENA_USER_ERROR_CATEGORY else state+ reason = athena_error.get("ErrorMessage") or status.get("StateChangeReason") or ""+ raise AwsAthenaQueryEndedError(f"{QUERY_ERROR_PREFIX}: {kind} - {str(reason)[:500]}", kind)+ if time.monotonic() > deadline:+ raise AwsAthenaError(f"{QUERY_ERROR_PREFIX}: TIMEOUT - query {execution_id} is still {state}", "TIMEOUT")+ # Nothing is yielded while Athena runs the query, so give the pipeline a chance to hand off.+ reach_safe_point()+ interruptible_wait(delay)+ delay = min(delay * 2, POLL_MAX_SECONDS)+++def iter_result_pages(client: AthenaClient, execution_id: str) -> Iterator[tuple[list[str], list[list[Optional[str]]]]]:+ """Each page of results as (column names, rows of text values). The header row is dropped."""+ next_token: Optional[str] = None+ first_page = True+ while True:+ params: dict[str, Any] = {"QueryExecutionId": execution_id, "MaxResults": RESULTS_PAGE_SIZE}+ if next_token:+ params["NextToken"] = next_token+ body = client.call(ATHENA_SERVICE, "GetQueryResults", params)+ result_set = body.get("ResultSet") or {}+ names = [str(info.get("Name")) for info in (result_set.get("ResultSetMetadata") or {}).get("ColumnInfo") or []]+ rows = [[datum.get("VarCharValue") for datum in row.get("Data") or []] for row in result_set.get("Rows") or []]+ if first_page and rows:+ # A SELECT's first page starts with a row of column names.+ rows = rows[1:]+ first_page = False+ yield names, rows+ next_token = body.get("NextToken")+ if not next_token:+ return+++def build_athena_select(+ *,+ database: str,+ table: Table[AthenaColumn],+ incremental_field: Optional[str],+ incremental_field_type: Any,+ incremental_last_value: Any,+ enabled_columns: Optional[list[str]],+ primary_keys: Optional[list[str]],+ row_filters: Any,+) -> SafeSQL:+ last_value = incremental_last_value+ if incremental_field is not None and incremental_field_type is not None:+ column_type = next((column.data_type for column in table.columns if column.name == incremental_field), None)+ last_value = incremental_value_for_trino_column(incremental_last_value, incremental_field_type, column_type)+ return SelectQueryBuilder(quoter=AnsiIdentifierQuoter(), param_style=ParamStyle.QMARK).select_all(+ schema=database,+ table_name=table.name,+ incremental_field=incremental_field,+ incremental_field_type=incremental_field_type,+ incremental_last_value=last_value,+ enabled_columns=enabled_columns,+ primary_keys=primary_keys,+ row_filters=row_filters,+ )+++class AthenaImplementation(SQLSourceImplementation[AwsAthenaSourceConfig, AthenaClient, Any]):+ """Athena over its HTTPS API: Glue for the catalog, Athena to run the read.++ Athena exposes no primary keys, so the merge key comes from the schema's stored key or an `id`+ column. Partition sizing is not implemented: a size estimate would cost a scan.+ """++ @contextmanager+ def connect(self, config: AwsAthenaSourceConfig, *, team_id: int | None = None) -> Iterator[AthenaClient]:+ # Every request goes to an AWS endpoint picked from the region, so there is no host to check.+ client = make_client(config)+ try:+ yield client+ finally:+ client.close()++ def get_columns(+ self,+ conn: AthenaClient,+ config: AwsAthenaSourceConfig,+ names: list[str] | None,+ ) -> dict[str, list[tuple[str, str, bool]]]: # nosemgrep: tuple-return-prefer-dataclass -- SQLSource protocol+ return {+ table_name: [(column.name, column.data_type, column.nullable) for column in columns]+ for table_name, columns in list_glue_tables(conn, config.database.strip(), names).items()+ }++ def get_incremental_filter(self) -> IncrementalFieldFilter:+ return filter_trino_incremental_fields++ def build_pipeline(self, config: AwsAthenaSourceConfig, inputs: SourceInputs) -> SourceResponse:+ source_location = resolve_source_location(inputs, config_namespace=config.database.strip())+ database = source_location.schema+ table_name = source_location.table_name+ if not database or not table_name:+ raise ValueError("Database or table name is missing")++ logger: FilteringBoundLogger = inputs.logger+ incremental_field = inputs.incremental_field if inputs.should_use_incremental_field else None+ incremental_field_type = inputs.incremental_field_type if inputs.should_use_incremental_field else None+ location = QueryLocation(+ database=database,+ workgroup=(config.workgroup or "").strip() or DEFAULT_WORKGROUP,+ output_location=(config.output_location or "").strip() or None,+ )++ client = make_client(config)+ full_table = get_glue_table(client, database, table_name)+ primary_keys = resolve_merge_keys(inputs.primary_keys, None, [column.name for column in full_table.columns])++ def get_rows() -> Iterator[pa.Table]:+ try:+ projection = resolve_table_projection(+ full_table,+ enabled_columns=inputs.enabled_columns,+ primary_keys=primary_keys,+ incremental_field=incremental_field,+ )+ query = build_athena_select(+ database=database,+ table=full_table,+ incremental_field=incremental_field,+ incremental_field_type=incremental_field_type,+ incremental_last_value=inputs.db_incremental_field_last_value,+ enabled_columns=projection.enabled_columns,+ primary_keys=primary_keys,+ row_filters=inputs.row_filters,+ )+ arrow_schema = projection.table.to_arrow_schema()+ logger.debug(f"Athena query: {query.sql}")++ execution_id = start_query(client, query, location)+ try:+ wait_for_query(client, execution_id)+ except AwsAthenaQueryEndedError:+ raise+ except BaseException:+ # The next attempt starts a new query, so this one would only use capacity and bill scans.+ try:+ client.stop_query(execution_id)+ except Exception as stop_error:+ logger.warning("Could not stop Athena query", execution_id=execution_id, error=str(stop_error))+ raise++ binary_reporter = BinaryColumnReporter(logger)+ for names, rows in iter_result_pages(client, execution_id):+ if not rows:+ reach_safe_point()+ continue+ page_schema = restrict_schema_to_columns(arrow_schema, names)+ parsers = [+ value_parser(page_schema.field(name).type) if name in page_schema.names else None+ for name in names+ ]+ yield table_from_iterator(+ (+ {+ name: parser(value) if parser is not None and value is not None else value+ for name, parser, value in zip(names, parsers, row)+ }+ for row in rows+ ),+ page_schema,+ primary_keys=primary_keys,+ binary_reporter=binary_reporter,+ )++ finally:+ client.close()++ return SourceResponse(+ name=source_location.response_name,+ items=get_rows,+ primary_keys=primary_keys,+ )+++VALIDATION_ERROR_MESSAGES: dict[str, str] = {+ "UnrecognizedClientException": "AWS rejected the access key. Check the access key ID and secret access key, and that the key is still active.",+ "InvalidClientTokenId": "AWS rejected the access key. Check the access key ID and secret access key, and that the key is still active.",+ "InvalidSignatureException": "AWS rejected the request signature. Re-enter the secret access key.",+ "ExpiredTokenException": "The AWS session token has expired. Reconnect with fresh credentials.",+ "AccessDeniedException": "These AWS credentials cannot read the catalog or run queries. Grant glue:GetDatabase, glue:GetTables, glue:GetTable, glue:GetPartitions, glue:GetPartition, glue:BatchGetPartition, athena:GetWorkGroup, athena:StartQueryExecution, athena:GetQueryExecution, athena:GetQueryResults and athena:StopQueryExecution, plus read access to the tables' S3 data and read and write access to the query result location.",+ "EntityNotFoundException": "The Glue database does not exist in this region. Check the database name and region.",+}++NO_RESULT_LOCATION_MESSAGE = (+ "This workgroup has no query result location. Set one on the workgroup, or enter an S3 output location."+)+WORKGROUP_DISABLED_MESSAGE = "This Athena workgroup is disabled. Enable it, or pick another workgroup."+WORKGROUP_NOT_FOUND_MESSAGE = "The Athena workgroup does not exist in this region. Check the workgroup name."+GENERIC_VALIDATION_ERROR = (+ "AWS didn't accept the request. Check the access key can read Glue and Athena, then try again."+)+++def validate_credentials(config: AwsAthenaSourceConfig) -> tuple[bool, Optional[str]]:+ """Check the inputs, then the database and the workgroup: the two things every sync needs."""+ if not config.aws_access_key_id or not config.aws_secret_access_key:+ return False, "AWS access key ID and secret access key are required"+ if not config.aws_region or aws_dns_suffix(config.aws_region.strip()) is None:+ return False, "Enter an AWS region code, for example us-east-1."+ if not (config.database or "").strip():+ return False, "Enter the Glue database to sync tables from."+ output_location = (config.output_location or "").strip()+ if output_location and not output_location.startswith("s3://"):+ return False, "Enter the S3 output location as s3://bucket/path/."++ client = make_client(config)+ workgroup_name = (config.workgroup or "").strip() or DEFAULT_WORKGROUP+ try:+ client.call(GLUE_SERVICE, "GetDatabase", {"Name": config.database.strip()})+ workgroup = client.call(ATHENA_SERVICE, "GetWorkGroup", {"WorkGroup": workgroup_name}).get("WorkGroup") or {}+ except AwsAthenaError as error:+ code = error.code or ""+ if code == "InvalidRequestException" and "WorkGroup" in str(error):+ return False, WORKGROUP_NOT_FOUND_MESSAGE+ if code in VALIDATION_ERROR_MESSAGES:+ return False, VALIDATION_ERROR_MESSAGES[code]+ return False, GENERIC_VALIDATION_ERROR+ except Exception:+ return False, "Could not reach AWS. Check the region and try again."++ if workgroup.get("State") == "DISABLED":+ return False, WORKGROUP_DISABLED_MESSAGE+ configuration = workgroup.get("Configuration") or {}+ has_location = bool((configuration.get("ResultConfiguration") or {}).get("OutputLocation"))+ managed = bool((configuration.get("ManagedQueryResultsConfiguration") or {}).get("Enabled"))+ if not (has_location or managed or output_location):+ return False, NO_RESULT_LOCATION_MESSAGE+ return True, Nonediff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_athena/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_athena/settings.pynew file mode 100644index 000000000000..33394a9b80c0--- /dev/null+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_athena/settings.py@@ -0,0 +1,20 @@+from typing import Literal++AwsService = Literal["athena", "glue"]++ATHENA_SERVICE: AwsService = "athena"+GLUE_SERVICE: AwsService = "glue"++# The catalog Glue-backed databases live in. Federated catalogs are not supported yet.+DATA_CATALOG = "AwsDataCatalog"+DEFAULT_WORKGROUP = "primary"++# GetQueryResults returns at most 1,000 rows per call; GetTables at most 100 tables.+RESULTS_PAGE_SIZE = 1000+GLUE_PAGE_SIZE = 100++# Polling backs off from 1 to 10 seconds. A query that has not finished after an hour is given up,+# matching the longest DML timeout most workgroups allow.+POLL_INITIAL_SECONDS = 1.0+POLL_MAX_SECONDS = 10.0+QUERY_DEADLINE_SECONDS = 60 * 60diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_athena/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_athena/source.pyindex 9677bcfd2118..3c9fe136065d 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_athena/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_athena/source.py@@ -1,27 +1,164 @@-from typing import cast+from typing import Optional, cast-from products.warehouse_sources.backend.facade.source_config import DataWarehouseSourceCategory, SourceConfig-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import FieldType, SimpleSource+from products.warehouse_sources.backend.facade.codebase_detection import detect+from products.warehouse_sources.backend.facade.source_config import (+ DataWarehouseSourceCategory,+ ReleaseStatus,+ SourceConfig,+ SourceFieldInputConfig,+ SourceFieldInputConfigType,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.aws_athena.athena import (+ ERROR_PREFIX,+ QUERY_ERROR_PREFIX,+ TABLE_NOT_FOUND,+ TABLE_NOT_FOUND_MESSAGE,+ THROTTLING_ERROR_CODES,+ VALIDATION_ERROR_MESSAGES,+ AthenaImplementation,+ validate_credentials as validate_athena_credentials,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import FieldTypefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistry+from products.warehouse_sources.backend.temporal.data_imports.sources.common.sql.base import SQLSourcefrom products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.awsathena import (AwsAthenaSourceConfig,)from products.warehouse_sources.backend.types import ExternalDataSourceType+_ATHENA_IMPLEMENTATION = AthenaImplementation()+@SourceRegistry.register-class AwsAthenaSource(SimpleSource[AwsAthenaSourceConfig]):+class AwsAthenaSource(SQLSource[AwsAthenaSourceConfig]):+ codebase_detection = detect(+ npm=("@aws-sdk/client-athena", "athena-express"),+ pypi=("pyathena", "awswrangler"),+ cargo=("aws-sdk-athena",),+ go=("github.com/aws/aws-sdk-go-v2/service/athena",),+ maven=("software.amazon.awssdk:athena", "com.amazonaws:athena-jdbc"),+ rubygems=("aws-sdk-athena",),+ )++ api_docs_url = "https://docs.aws.amazon.com/athena/latest/APIReference/Welcome.html"++ @property+ def get_implementation(self) -> AthenaImplementation:+ return _ATHENA_IMPLEMENTATION+@propertydef source_type(self) -> ExternalDataSourceType:return ExternalDataSourceType.AWSATHENA+ @property+ def connection_host_fields(self) -> list[str]:+ # The region picks the endpoint the signed requests, and so the stored key, are sent to.+ return ["aws_region"]++ def validate_credentials(+ self,+ config: AwsAthenaSourceConfig,+ team_id: int,+ schema_name: Optional[str] = None,+ api_version: str | None = None,+ ) -> tuple[bool, str | None]:+ return validate_athena_credentials(config)++ def get_non_retryable_errors(self) -> dict[str, str | None]:+ errors: dict[str, str | None] = {+ **self.default_non_retryable_errors(),+ **{f"{ERROR_PREFIX}: {code}": message for code, message in VALIDATION_ERROR_MESSAGES.items()},+ }+ errors[f"{ERROR_PREFIX}: InvalidRequestException"] = (+ "Athena rejected the request. Check the workgroup, the output location and the table's columns, then retry."+ )+ errors[f"{ERROR_PREFIX}: {TABLE_NOT_FOUND}"] = TABLE_NOT_FOUND_MESSAGE+ errors[f"{QUERY_ERROR_PREFIX}: USER_ERROR"] = (+ "Athena could not run the query for this table. A column may have been renamed or dropped, or the "+ "credentials cannot read the table's S3 data. Fix that, then retry the sync."+ )+ return errors++ def get_retryable_errors(self) -> set[str]:+ return {f"{ERROR_PREFIX}: {code}" for code in THROTTLING_ERROR_CODES} | {+ f"{ERROR_PREFIX}: HTTP 5",+ f"{ERROR_PREFIX}: InternalServerException",+ }+@propertydef get_source_config(self) -> SourceConfig:return SourceConfig(name=ExternalDataSourceType.AWSATHENA,- category=DataWarehouseSourceCategory.ENGINEERING___MONITORING,- label="Amazon Web Services (Amazon Athena)",+ category=DataWarehouseSourceCategory.DATABASES,+ label="Amazon Athena",+ caption="""Sync tables from an AWS Glue database through Amazon Athena into the PostHog Data warehouse.++Create an IAM user or role with `glue:GetDatabase`, `glue:GetTables`, `glue:GetTable`, `glue:GetPartitions`, `glue:GetPartition`, `glue:BatchGetPartition`, `athena:GetWorkGroup`, `athena:StartQueryExecution`, `athena:GetQueryExecution`, `athena:GetQueryResults` and `athena:StopQueryExecution`, plus read access to the tables' S3 data and read and write access to the query result location. Paste its access key ID and secret access key.++Each sync runs one Athena query per table, which Athena bills by the data it scans. An incremental sync filters on the chosen column, so a partitioned table keyed on that column scans less.""",iconPath="/static/services/aws_athena.png",- fields=cast(list[FieldType], []),- unreleasedSource=True,+ docsUrl="https://posthog.com/docs/cdp/sources/aws-athena",+ releaseStatus=ReleaseStatus.ALPHA,+ keywords=["aws", "athena", "glue", "presto", "trino", "sql"],+ fields=cast(+ list[FieldType],+ [+ SourceFieldInputConfig(+ name="aws_access_key_id",+ label="AWS access key ID",+ type=SourceFieldInputConfigType.TEXT,+ required=True,+ placeholder="AKIA...",+ secret=False,+ ),+ SourceFieldInputConfig(+ name="aws_secret_access_key",+ label="AWS secret access key",+ type=SourceFieldInputConfigType.PASSWORD,+ required=True,+ placeholder="",+ secret=True,+ ),+ SourceFieldInputConfig(+ name="aws_session_token",+ label="AWS session token (only for temporary credentials)",+ type=SourceFieldInputConfigType.PASSWORD,+ required=False,+ placeholder="",+ secret=True,+ ),+ SourceFieldInputConfig(+ name="aws_region",+ label="AWS region",+ type=SourceFieldInputConfigType.TEXT,+ required=True,+ placeholder="us-east-1",+ secret=False,+ ),+ SourceFieldInputConfig(+ name="database",+ label="Glue database",+ type=SourceFieldInputConfigType.TEXT,+ required=True,+ placeholder="analytics",+ secret=False,+ ),+ SourceFieldInputConfig(+ name="workgroup",+ label="Athena workgroup (defaults to primary)",+ type=SourceFieldInputConfigType.TEXT,+ required=False,+ placeholder="primary",+ secret=False,+ ),+ SourceFieldInputConfig(+ name="output_location",+ label="S3 output location (only when the workgroup has none)",+ type=SourceFieldInputConfigType.TEXT,+ required=False,+ placeholder="s3://my-athena-results/posthog/",+ secret=False,+ ),+ ],+ ),)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_athena/tests/test_athena.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_athena/tests/test_athena.pynew file mode 100644index 000000000000..b83bbc6369d9--- /dev/null+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_athena/tests/test_athena.py@@ -0,0 +1,449 @@+import datetime as dt+from collections.abc import Callable, Mapping+from decimal import Decimal+from typing import Any++import pytest++import pyarrow as pa++from products.warehouse_sources.backend.temporal.data_imports.sources.aws_athena.athena import (+ NO_RESULT_LOCATION_MESSAGE,+ VALIDATION_ERROR_MESSAGES,+ WORKGROUP_DISABLED_MESSAGE,+ WORKGROUP_NOT_FOUND_MESSAGE,+ athena_literal,+ validate_credentials,+ value_parser,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.aws_athena.source import AwsAthenaSource+from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (+ RecordedRequest,+ ScriptedResponse,+ SourceDriver,+ scripted_network,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing.responders import UnexpectedRequest+from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.awsathena import (+ AwsAthenaSourceConfig,+)+from products.warehouse_sources.backend.types import IncrementalFieldType++ORDERS_TABLE = {+ "Table": {+ "Name": "orders",+ "DatabaseName": "analytics",+ "StorageDescriptor": {+ "Columns": [+ {"Name": "id", "Type": "bigint"},+ {"Name": "amount", "Type": "decimal(10,2)"},+ {"Name": "paid", "Type": "boolean"},+ {"Name": "updated_at", "Type": "timestamp"},+ {"Name": "tags", "Type": "array<string>"},+ {"Name": "note", "Type": "string"},+ ]+ },+ "PartitionKeys": [{"Name": "dt", "Type": "date"}],+ }+}+COLUMNS = ["id", "amount", "paid", "updated_at", "tags", "note", "dt"]+++def config(**overrides: Any) -> AwsAthenaSourceConfig:+ values: dict[str, Any] = {+ "aws_access_key_id": "AKIAEXAMPLE",+ "aws_secret_access_key": "secret",+ "aws_session_token": None,+ "aws_region": "us-east-1",+ "database": "analytics",+ "workgroup": None,+ "output_location": None,+ }+ values.update(overrides)+ return AwsAthenaSourceConfig(**values)+++def by_target(queues: Mapping[str, list[ScriptedResponse]]) -> Callable[[RecordedRequest], ScriptedResponse]:+ """Glue and Athena both post to `/`, so answers are routed by the operation in `X-Amz-Target`."""+ remaining = {target: list(answers) for target, answers in queues.items()}++ def respond(request: RecordedRequest) -> ScriptedResponse:+ target = request.headers.get("x-amz-target", "")+ queue = remaining.get(target.split(".")[-1])+ if not queue:+ raise UnexpectedRequest(f"no answer left for {target}")+ return queue.pop(0)++ return respond+++def execution(state: str, athena_error: dict[str, Any] | None = None) -> ScriptedResponse:+ status: dict[str, Any] = {"State": state}+ if athena_error:+ status["AthenaError"] = athena_error+ return ScriptedResponse(json={"QueryExecution": {"QueryExecutionId": "q-1", "Status": status}})+++def results(rows: list[list[str | None]], next_token: str | None = None, header: bool = False) -> ScriptedResponse:+ data = [[{"VarCharValue": value} if value is not None else {} for value in row] for row in rows]+ if header:+ data.insert(0, [{"VarCharValue": name} for name in COLUMNS])+ body: dict[str, Any] = {+ "ResultSet": {+ "Rows": [{"Data": row} for row in data],+ "ResultSetMetadata": {"ColumnInfo": [{"Name": name, "Type": "varchar"} for name in COLUMNS]},+ }+ }+ if next_token:+ body["NextToken"] = next_token+ return ScriptedResponse(json=body)+++def aws_error(code: str, message: str) -> ScriptedResponse:+ return ScriptedResponse(status=400, json={"__type": code, "Message": message})+++def table_rows(items: list[Any]) -> list[dict[str, Any]]:+ return [row for table in items for row in table.to_pylist()]+++class TestSync:+ def test_runs_the_query_waits_for_it_and_reads_every_result_page_as_typed_rows(self) -> None:+ result = SourceDriver(AwsAthenaSource(), config()).run(+ "orders",+ by_target(+ {+ "GetTable": [ScriptedResponse(json=ORDERS_TABLE)],+ "StartQueryExecution": [ScriptedResponse(json={"QueryExecutionId": "q-1"})],+ "GetQueryExecution": [execution("QUEUED"), execution("RUNNING"), execution("SUCCEEDED")],+ "GetQueryResults": [+ results(+ [["1", "12.50", "true", "2024-03-02 10:00:00.123", "[a, b]", "it's", "2024-03-02"]],+ next_token="p2",+ header=True,+ ),+ results([["2", None, "false", "2024-03-02 11:00:00", "[]", "", "2024-03-02"]]),+ ],+ }+ ),+ )++ assert result.raised is None+ targets = [request.headers["x-amz-target"] for request in result.requests]+ assert targets == [+ "AWSGlue.GetTable",+ "AmazonAthena.StartQueryExecution",+ "AmazonAthena.GetQueryExecution",+ "AmazonAthena.GetQueryExecution",+ "AmazonAthena.GetQueryExecution",+ "AmazonAthena.GetQueryResults",+ "AmazonAthena.GetQueryResults",+ ]+ start = result.requests[1].json()+ assert 'FROM "analytics"."orders"' in start["QueryString"]+ assert "WHERE" not in start["QueryString"]+ assert "ExecutionParameters" not in start+ assert start["QueryExecutionContext"] == {"Database": "analytics", "Catalog": "AwsDataCatalog"}+ assert start["WorkGroup"] == "primary"+ assert result.requests[6].json()["NextToken"] == "p2"+ # The header row on the first page is not data.+ assert table_rows(result.items) == [+ {+ "id": 1,+ "amount": Decimal("12.50"),+ "paid": True,+ "updated_at": dt.datetime(2024, 3, 2, 10, 0, 0, 123000),+ "tags": "[a, b]",+ "note": "it's",+ "dt": dt.date(2024, 3, 2),+ },+ {+ "id": 2,+ "amount": None,+ "paid": False,+ "updated_at": dt.datetime(2024, 3, 2, 11, 0),+ "tags": "[]",+ "note": "",+ "dt": dt.date(2024, 3, 2),+ },+ ]+ assert result.response is not None and result.response.primary_keys == ["id"]++ def test_an_incremental_read_passes_the_watermark_as_a_typed_execution_parameter(self) -> None:+ result = SourceDriver(+ AwsAthenaSource(), config(workgroup="analysts", output_location="s3://results/posthog/")+ ).run(+ "orders",+ by_target(+ {+ "GetTable": [ScriptedResponse(json=ORDERS_TABLE)],+ "StartQueryExecution": [ScriptedResponse(json={"QueryExecutionId": "q-1"})],+ "GetQueryExecution": [execution("SUCCEEDED")],+ "GetQueryResults": [results([], header=True)],+ }+ ),+ incremental_field="updated_at",+ incremental_field_type=IncrementalFieldType.Timestamp,+ db_incremental_field_last_value=dt.datetime(2024, 3, 1, 8, 30, tzinfo=dt.UTC),+ )++ assert result.raised is None+ start = result.requests[1].json()+ assert 'WHERE "updated_at" > ?' in start["QueryString"]+ assert start["QueryString"].endswith('ORDER BY "updated_at" ASC')+ # A `timestamp` column has no zone, so the UTC watermark is passed without one.+ assert start["ExecutionParameters"] == ["TIMESTAMP '2024-03-01 08:30:00.000000'"]+ assert start["WorkGroup"] == "analysts"+ assert start["ResultConfiguration"] == {"OutputLocation": "s3://results/posthog/"}++ def test_a_failed_query_fails_the_run_with_athenas_reason(self) -> None:+ result = SourceDriver(AwsAthenaSource(), config()).run(+ "orders",+ by_target(+ {+ "GetTable": [ScriptedResponse(json=ORDERS_TABLE)],+ "StartQueryExecution": [ScriptedResponse(json={"QueryExecutionId": "q-1"})],+ "GetQueryExecution": [+ execution(+ "FAILED",+ {+ "ErrorCategory": 2,+ "ErrorMessage": "COLUMN_NOT_FOUND: line 1:8: Column 'note' cannot be resolved",+ },+ )+ ],+ }+ ),+ )++ assert str(result.raised) == (+ "AWS Athena query failed: USER_ERROR - COLUMN_NOT_FOUND: line 1:8: Column 'note' cannot be resolved"+ )+ assert "AmazonAthena.StopQueryExecution" not in [request.headers["x-amz-target"] for request in result.requests]++ def test_a_retried_start_sends_the_same_idempotency_token(self) -> None:+ result = SourceDriver(AwsAthenaSource(), config()).run(+ "orders",+ by_target(+ {+ "GetTable": [ScriptedResponse(json=ORDERS_TABLE)],+ "StartQueryExecution": [+ ScriptedResponse(status=500, json={"__type": "InternalServerException", "Message": "Internal"}),+ ScriptedResponse(json={"QueryExecutionId": "q-1"}),+ ],+ "GetQueryExecution": [execution("SUCCEEDED")],+ "GetQueryResults": [results([], header=True)],+ }+ ),+ )++ assert result.raised is None+ tokens = [+ request.json().get("ClientRequestToken")+ for request in result.requests+ if request.headers["x-amz-target"] == "AmazonAthena.StartQueryExecution"+ ]+ assert len(tokens) == 2+ assert tokens[0] and tokens[0] == tokens[1]++ def test_a_query_still_running_at_the_deadline_is_stopped(self) -> None:+ result = SourceDriver(AwsAthenaSource(), config()).run(+ "orders",+ by_target(+ {+ "GetTable": [ScriptedResponse(json=ORDERS_TABLE)],+ "StartQueryExecution": [ScriptedResponse(json={"QueryExecutionId": "q-1"})],+ "GetQueryExecution": [execution("RUNNING")] * 1000,+ "StopQueryExecution": [ScriptedResponse(json={})],+ }+ ),+ )++ assert str(result.raised) == "AWS Athena query failed: TIMEOUT - query q-1 is still RUNNING"+ last = result.requests[-1]+ assert last.headers["x-amz-target"] == "AmazonAthena.StopQueryExecution"+ assert last.json() == {"QueryExecutionId": "q-1"}+++class TestSchemas:+ def test_lists_every_glue_table_with_its_partition_keys_and_incremental_candidates(self) -> None:+ orders = ORDERS_TABLE["Table"]+ events = {"Name": "events", "StorageDescriptor": {"Columns": [{"Name": "payload", "Type": "struct<a:int>"}]}}++ with scripted_network(+ by_target(+ {+ "GetTables": [+ ScriptedResponse(json={"TableList": [orders], "NextToken": "t2"}),+ ScriptedResponse(json={"TableList": [events]}),+ ]+ }+ )+ ) as network:+ schemas = AwsAthenaSource().get_schemas(config(), team_id=1)++ assert [request.json().get("NextToken") for request in network.requests_log] == [None, "t2"]+ by_name = {schema.name: schema for schema in schemas}+ assert set(by_name) == {"orders", "events"}+ assert [column[0] for column in by_name["orders"].columns] == COLUMNS+ assert {field["field"] for field in by_name["orders"].incremental_fields} == {+ "id",+ "amount",+ "updated_at",+ "dt",+ }+ assert by_name["orders"].detected_primary_keys == ["id"]+ assert by_name["events"].supports_incremental is False++ def test_named_tables_are_fetched_one_by_one_and_a_dropped_one_is_left_out(self) -> None:+ with scripted_network(+ by_target(+ {+ "GetTable": [+ ScriptedResponse(json=ORDERS_TABLE),+ aws_error("EntityNotFoundException", "Table gone not found"),+ ]+ }+ )+ ) as network:+ schemas = AwsAthenaSource().get_schemas(config(), team_id=1, names=["orders", "gone"])++ assert [request.headers["x-amz-target"] for request in network.requests_log] == [+ "AWSGlue.GetTable",+ "AWSGlue.GetTable",+ ]+ assert [schema.name for schema in schemas] == ["orders"]+++class TestAthenaLiteral:+ @pytest.mark.parametrize(+ ("value", "literal"),+ [+ ("it's", "'it''s'"),+ (+ dt.datetime(2024, 3, 1, 9, tzinfo=dt.timezone(dt.timedelta(hours=1))),+ "TIMESTAMP '2024-03-01 08:00:00.000000 UTC'",+ ),+ (dt.date(2024, 3, 1), "DATE '2024-03-01'"),+ (True, "TRUE"),+ (42, "42"),+ (Decimal("12.50"), "12.50"),+ # Athena takes no exponent in a numeric literal, and DECIMAL(38, 38) still fits.+ (Decimal("1E+2"), "100"),+ (Decimal("0." + "1" * 38), "0." + "1" * 38),+ (None, "NULL"),+ ],+ )+ def test_each_value_becomes_an_escaped_athena_literal(self, value: Any, literal: str) -> None:+ assert athena_literal(value) == literal++ def test_an_unsupported_value_is_refused_rather_than_stringified(self) -> None:+ with pytest.raises(TypeError):+ athena_literal(["a"])++ @pytest.mark.parametrize("text", ["1e1000000000", "1e-1000000000", "NaN", "Infinity"])+ def test_a_decimal_with_no_athena_literal_is_refused_before_it_is_written_out(self, text: str) -> None:+ # A row filter value arrives as text, so `1e1000000000` would otherwise render as a+ # gigabyte-long parameter before the query request leaves the worker.+ with pytest.raises(ValueError):+ athena_literal(Decimal(text))+++class TestValueParser:+ @pytest.mark.parametrize(+ "arrow_type, text, expected",+ [+ # Glue declares an Iceberg timestamptz as `timestamp`, and Athena prints it with its zone.+ (pa.timestamp("us"), "2024-01-15 10:30:00.123456 UTC", dt.datetime(2024, 1, 15, 10, 30, 0, 123456)),+ (pa.timestamp("us"), "2024-01-15 10:30:00.123 +05:30", dt.datetime(2024, 1, 15, 5, 0, 0, 123000)),+ (+ pa.timestamp("us", tz="UTC"),+ "2024-01-15 10:30:00 America/New_York",+ dt.datetime(2024, 1, 15, 15, 30, tzinfo=dt.UTC),+ ),+ (pa.timestamp("us"), "2024-01-15 10:30:00", dt.datetime(2024, 1, 15, 10, 30)),+ ],+ )+ def test_a_zoned_timestamp_lands_in_utc(self, arrow_type: pa.DataType, text: str, expected: dt.datetime) -> None:+ parser = value_parser(arrow_type)+ assert parser is not None+ assert parser(text) == expected+++class TestValidateCredentials:+ @pytest.mark.parametrize(+ ("overrides", "expected"),+ [+ ({"aws_secret_access_key": ""}, "AWS access key ID and secret access key are required"),+ ({"aws_region": "frankfurt"}, "Enter an AWS region code, for example us-east-1."),+ ({"database": " "}, "Enter the Glue database to sync tables from."),+ ({"output_location": "my-bucket/results"}, "Enter the S3 output location as s3://bucket/path/."),+ ],+ )+ def test_local_checks_fail_before_any_request(self, overrides: dict[str, Any], expected: str) -> None:+ with scripted_network([]) as network:+ assert validate_credentials(config(**overrides)) == (False, expected)++ assert network.requests == 0++ @pytest.mark.parametrize(+ ("overrides", "workgroup", "expected"),+ [+ (+ {},+ {"State": "ENABLED", "Configuration": {"ResultConfiguration": {"OutputLocation": "s3://r/"}}},+ (True, None),+ ),+ (+ {},+ {"State": "ENABLED", "Configuration": {"ManagedQueryResultsConfiguration": {"Enabled": True}}},+ (True, None),+ ),+ ({}, {"State": "ENABLED", "Configuration": {}}, (False, NO_RESULT_LOCATION_MESSAGE)),+ ({"output_location": "s3://r/"}, {"State": "ENABLED", "Configuration": {}}, (True, None)),+ ({}, {"State": "DISABLED", "Configuration": {}}, (False, WORKGROUP_DISABLED_MESSAGE)),+ ],+ )+ def test_the_workgroup_must_be_enabled_and_have_somewhere_to_write_results(+ self, overrides: dict[str, Any], workgroup: dict[str, Any], expected: tuple[bool, str | None]+ ) -> None:+ with scripted_network(+ by_target(+ {+ "GetDatabase": [ScriptedResponse(json={"Database": {"Name": "analytics"}})],+ "GetWorkGroup": [ScriptedResponse(json={"WorkGroup": {"Name": "primary", **workgroup}})],+ }+ )+ ):+ assert validate_credentials(config(**overrides)) == expected++ @pytest.mark.parametrize(+ ("answers", "expected"),+ [+ (+ {"GetDatabase": [aws_error("EntityNotFoundException", "Database analytics not found.")]},+ VALIDATION_ERROR_MESSAGES["EntityNotFoundException"],+ ),+ (+ {+ "GetDatabase": [ScriptedResponse(json={"Database": {"Name": "analytics"}})],+ "GetWorkGroup": [aws_error("InvalidRequestException", "WorkGroup primary is not found.")],+ },+ WORKGROUP_NOT_FOUND_MESSAGE,+ ),+ (+ {+ "GetDatabase": [+ aws_error(+ "AccessDeniedException",+ "User: arn:aws:iam::000000000000:user/example is not authorized to perform: glue:GetDatabase",+ )+ ]+ },+ VALIDATION_ERROR_MESSAGES["AccessDeniedException"],+ ),+ ],+ )+ def test_aws_errors_become_fixed_messages(self, answers: dict[str, list[ScriptedResponse]], expected: str) -> None:+ with scripted_network(by_target(answers)):+ assert validate_credentials(config()) == (False, expected)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/awsathena.py b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/awsathena.pyindex 6cd9fbafe9b5..b4d0e03daa7c 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/awsathena.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/awsathena.py@@ -6,4 +6,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common imp@config.configclass AwsAthenaSourceConfig(config.Config):- pass+ aws_access_key_id: str+ aws_secret_access_key: str+ aws_region: str+ database: str+ aws_session_token: str | None = None+ workgroup: str | None = None+ output_location: str | None = Nonediff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/source_contract_baseline.txt b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/source_contract_baseline.txtindex 31f836ed2b3b..6621b6817df2 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/source_contract_baseline.txt+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/source_contract_baseline.txt@@ -8,6 +8,7 @@ AppleSearchAds not-checkable:no-requestAppsFlyer fail:safe-pointAppsignal fail:safe-pointAttentive not-checkable:database+AwsAthena not-checkable:schemasAwsCostExplorer fail:safe-pointAzureBlob not-checkable:schemasAzureCostManagement not-checkable:no-requestdiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_credential_contract.py b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_credential_contract.pyindex 614c3892a4d5..293288380e79 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_credential_contract.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_credential_contract.py@@ -393,6 +393,7 @@ OK_NEEDS_VENDOR_BODY: set[str] = {"Ashby","Automox","Avalara",+ "AwsAthena","AwsBudgets","AwsComputeOptimizer","AwsCostAndUsageReport",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_error_contract.py b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_error_contract.pyindex 8e4339439749..d4a715944685 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_error_contract.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_error_contract.py@@ -78,6 +78,7 @@ NOT_CHECKABLE: dict[str, str] = {"AppStoreConnect": "no-request","AppleSearchAds": "no-request","Attentive": "no-request",+ "AwsAthena": "schemas","AzureBlob": "schemas","AzureCostManagement": "no-request","Baserow": "schemas",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_schema_contract.py b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_schema_contract.pyindex 3521de3ceae6..bc1ea1f6af11 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_schema_contract.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_schema_contract.py@@ -50,6 +50,7 @@ PASS = "pass"# `empty`: `get_schemas` returns no schema for a placeholder config.SCHEMA_FILTER_NOT_CHECKABLE: dict[str, str] = {"AmazonS3": "network",+ "AwsAthena": "network","AzureBlob": "network","BigQuery": "driver","Baserow": "network",@@ -100,6 +101,7 @@ SCHEMA_FILTER_NOT_CHECKABLE: dict[str, str] = {UNKNOWN_SCHEMA_NOT_CHECKABLE: dict[str, str] = {"AmazonS3": "request","Attentive": "webhook",+ "AwsAthena": "request","AzureBlob": "request","Baserow": "request","BigQuery": "driver",