:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(hogql): preserve query authorization during parameter binding
Marius Andra69adc2021c8c
diff --git a/docs/published/handbook/engineering/databases/hogql-python.md b/docs/published/handbook/engineering/databases/hogql-python.mdindex 4d0ec965e750..1148ac285084 100644--- a/docs/published/handbook/engineering/databases/hogql-python.md+++ b/docs/published/handbook/engineering/databases/hogql-python.md@@ -105,6 +105,20 @@ query_result.columns == ['event', 'timestamp'] # might be useful if you selectYou can mix and match `parse_expr` and `ast` nodes as you please. The example above _still_ took a shortcut for the where clause because it was easier to write.+Pass compiled ClickHouse SQL and `HogQLContext.values` together to the database client. The printer reserves `%(name)s` placeholders for values and rejects percent signs in identifiers. Do not apply brace formatting to compiled SQL: braces are valid identifier characters and must remain literal after authorization.++For trusted SQL templates passed to the Temporal `ClickHouseClient`, distinguish the layer that consumes each placeholder:++| Purpose | Plain Python string | Python f-string source | Consumed by |+| -------------------- | ------------------- | ---------------------- | ------------------------------------------------------- |+| Client-bound value | `%(name)s` | `%(name)s` | `prepare_query`, after SQL value escaping |+| Server-bound value | `{name:Type}` | `{{name:Type}}` | ClickHouse, using the HTTP `param_name` parameter |+| S3 partition marker | `{_partition_id}` | `{{_partition_id}}` | ClickHouse |+| Empty JSON object | `'{}'` | `'{{}}'` | Neither; SQL string data |+| Literal percent sign | `%%` | `%%` | `prepare_query`, when the parameter mapping is nonempty |++An f-string consumes one brace-escaping layer in its source. It does not reinterpret braces inside an interpolated string. `string.Template` substitutes `$name` and leaves braces unchanged. If a trusted template uses `.format()`, that call also needs its own brace escaping. Count these construction steps separately from database binding; do not halve braces across a query indiscriminately.+## Snowflake date formattingFor direct Snowflake queries, `formatDateTime` requires a literal format string and translates supported strftime specifiers into Snowflake format elements.diff --git a/posthog/api/proxy_record_diagnostics.py b/posthog/api/proxy_record_diagnostics.pyindex 06ed9bd6e094..00365aa66e5b 100644--- a/posthog/api/proxy_record_diagnostics.py+++ b/posthog/api/proxy_record_diagnostics.py@@ -191,7 +191,9 @@ def diagnose(record: ProxyRecord) -> DiagnosticReport:predate the Cloudflare path (where a Cloudflare lookup always comes back empty and wouldotherwise read as a false "we don't have a record of this proxy")."""- log = LOGGER.bind(+ # The configured logger context is thread-local; binding diagnostic fields+ # directly would attach them to unrelated logs after this request returns.+ log = structlog.threadlocal.as_immutable(LOGGER).bind(proxy_record_id=str(record.id),organization_id=str(record.organization_id),domain=record.domain,diff --git a/posthog/api/test/test_proxy_record_diagnostics.py b/posthog/api/test/test_proxy_record_diagnostics.pyindex 8f1d2bc559bf..12b45cf2ea6d 100644--- a/posthog/api/test/test_proxy_record_diagnostics.py+++ b/posthog/api/test/test_proxy_record_diagnostics.py@@ -6,6 +6,7 @@ from unittest.mock import MagicMock, patchfrom django.test import SimpleTestCase, TestCase, override_settingsimport requests+import structlogimport dns.resolverfrom parameterized import parameterized@@ -620,8 +621,10 @@ class TestDiagnoseOrchestrator(TestCase):# fired at all rather than on how a half-configured mock happens to fail.post_mock.return_value = MagicMock(status_code=200)+ log_context = dict(structlog.get_context(diagnostics.LOGGER.bind()))report = diagnostics.diagnose(_record(domain="169.254.169.254:80/pad.attacker.example", target=CF_TARGET))+ self.assertEqual(dict(structlog.get_context(diagnostics.LOGGER.bind())), log_context)post_mock.assert_not_called()http_get_mock.assert_not_called()socket_mock.assert_not_called()diff --git a/posthog/temporal/common/clickhouse.py b/posthog/temporal/common/clickhouse.pyindex 37de0a5aa72c..3e0351e00607 100644--- a/posthog/temporal/common/clickhouse.py+++ b/posthog/temporal/common/clickhouse.py@@ -10,7 +10,6 @@ import asyncioimport datetime as dtimport contextlibimport collections.abc-from string import Formatterfrom urllib.parse import urljoinfrom django.conf import settings@@ -313,23 +312,6 @@ def add_log_comment_param(params: dict[str, typing.Any], query_tags: typing.Optiparams[param_name] = query_tags.to_json()-class KeywordOnlyFormatter(Formatter):- """Formatter supporting only keyword arguments.-- Positional arguments are unchanged, missing keys are also left unchanged.- """-- def get_value(self, key, args, kwargs):- if isinstance(key, int):- # Returns '{n}' unchanged, where n is a numerical index.- return f"{{{key}}}"- try:- return kwargs[key]- except KeyError:- # Returns '{key}' unchanged- return f"{{{key}}}"--class ClickHouseClient:"""An asynchronous client to access ClickHouse via HTTP.@@ -429,23 +411,11 @@ class ClickHouseClient:if not query_parameters:return query- has_format_placeholders = re.search(r"(?<!{){[^{}]*}(?!})|{{[^{}]*}}", query)-format_parameters = {k: encode_clickhouse_data(v).decode("utf-8") for k, v in query_parameters.items()}-- if has_format_placeholders:- # Escape any curly brackets `{` or `}` in the format parameters so they are not parsed- # as format placeholders- escaped_parameters = {k: v.replace("{", "{{").replace("}", "}}") for k, v in format_parameters.items()}- query = query % escaped_parameters- # HogQL prints the empty-object sentinel '{}' inline for every property read on the native-JSON- # events table, and the formatter would auto-number it to '{0}'. An empty pair is never a placeholder.- query = re.sub(r"(?<!\{)\{\}(?!\})", "{{}}", query)- query = KeywordOnlyFormatter().format(query, **format_parameters)- else:- query = query % format_parameters-- return query+ # HogQL reserves percent placeholders for values and rejects percent signs in identifiers.+ # Braces can occur in authorized identifiers, so interpreting them here would change the+ # query after authorization. Leave typed ClickHouse parameters and all other braces intact.+ return query % format_parametersdef prepare_request_data(self, data: collections.abc.Sequence[typing.Any]) -> bytes | None:"""Prepare the request data sent by encoding it.@@ -842,8 +812,8 @@ class ClickHouseClient:"""query = """SELECT type, exception- FROM clusterAllReplicas({{cluster_name:String}}, system.query_log)- WHERE query_id = {{query_id:String}}+ FROM clusterAllReplicas({cluster_name:String}, system.query_log)+ WHERE query_id = {query_id:String}AND event_date >= yesterday() AND event_time >= now() - interval 24 hourFORMAT JSONEachRow"""@@ -907,8 +877,8 @@ class ClickHouseClient:"""query = """SELECT written_rows- FROM clusterAllReplicas({{cluster_name:String}}, system.query_log)- WHERE query_id = {{query_id:String}}+ FROM clusterAllReplicas({cluster_name:String}, system.query_log)+ WHERE query_id = {query_id:String}AND type = 'QueryFinish'AND is_initial_query = 1AND event_date >= yesterday() AND event_time >= now() - interval 24 hour@@ -948,8 +918,8 @@ class ClickHouseClient:"""query = """SELECT 1- FROM clusterAllReplicas({{cluster_name:String}}, system.processes)- WHERE query_id = {{query_id:String}}+ FROM clusterAllReplicas({cluster_name:String}, system.processes)+ WHERE query_id = {query_id:String}AND NOT is_cancelledLIMIT 1"""@@ -976,7 +946,7 @@ class ClickHouseClient:Arguments:query_id: The ID of the query to cancel."""- query = f"KILL QUERY ON CLUSTER '{settings.CLICKHOUSE_CLUSTER}' WHERE query_id = {{{{query_id:String}}}}"+ query = f"KILL QUERY ON CLUSTER '{settings.CLICKHOUSE_CLUSTER}' WHERE query_id = {{query_id:String}}"await self.execute_query(query,diff --git a/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py b/posthog/temporal/tests/data_modeling/test_materialize_view_activities.pyindex b959b55fdb71..4d1de8e44999 100644--- a/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py+++ b/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py@@ -24,7 +24,11 @@ from posthog.hogql.resolver import ResolverFactoryfrom posthog.clickhouse.query_tagging import get_query_tagsfrom posthog.models import Team, Userfrom posthog.sync import database_sync_to_async-from posthog.temporal.common.clickhouse import ClickHouseError, ClickHouseTooManySimultaneousQueriesError+from posthog.temporal.common.clickhouse import (+ ClickHouseClient,+ ClickHouseError,+ ClickHouseTooManySimultaneousQueriesError,+)from posthog.temporal.common.errors import NonReportableErrorfrom posthog.temporal.data_modeling.activities import (CreateDataModelingJobInputs,@@ -57,6 +61,7 @@ from posthog.temporal.data_modeling.activities.materialize_view_managed_warehousmaterialize_view_managed_warehouse_activity,)from posthog.temporal.data_modeling.activities.notify_materialization_failure import _SavedQueryViewers+from posthog.temporal.tests.utils.events import generate_test_events_in_clickhousefrom products.customer_analytics.backend.facade.temporal import stage_warehouse_account_property_files_activityfrom products.customer_analytics.backend.facade.temporal_contracts import StageAccountPropertySyncInput@@ -2048,6 +2053,41 @@ class _EmptyArrowClient:class TestHogqlTableModifiers:+ async def test_formatter_shaped_aliases_preserve_values_and_team_scope(+ self, ateam: Team, clickhouse_client: ClickHouseClient+ ) -> None:+ events, _, other_team_events = await generate_test_events_in_clickhouse(+ client=clickhouse_client,+ team_id=ateam.pk,+ start_time=dt.datetime(2024, 1, 1, tzinfo=dt.UTC),+ end_time=dt.datetime(2024, 1, 2, tzinfo=dt.UTC),+ count=2,+ count_outside_range=0,+ count_other_team=2,+ table="sharded_events",+ )++ @contextlib.asynccontextmanager+ async def get_test_client(**kwargs: object) -> AsyncIterator[ClickHouseClient]:+ yield clickhouse_client++ query = r"""+ SELECT 'quote\' backtick` slash\\ {log_comment} %(other)s' AS `{hogql_val_0}`,+ event AS `{log_comment}`, uuid AS `{{hogql_val_0}}`+ FROM events+ """+ with unittest.mock.patch(+ "posthog.temporal.data_modeling.activities.materialize_view.get_clickhouse_client", get_test_client+ ):+ batches = [batch async for batch in hogql_table(query, ateam, LOGGER.bind())]++ rows = [row for table, _ in batches for row in table.to_pylist()]+ assert len(rows) == len(events)+ assert all(list(row) == ["{hogql_val_0}", "{log_comment}", "{{hogql_val_0}}"] for row in rows)+ assert {row["{hogql_val_0}"] for row in rows} == {"quote' backtick` slash\\ {log_comment} %(other)s"}+ assert {row["{{hogql_val_0}}"] for row in rows} == {event["uuid"] for event in events}+ assert not {row["{{hogql_val_0}}"] for row in rows} & {event["uuid"] for event in other_team_events}+@pytest.mark.parametrize("clickhouse_type,expected_conversion",[diff --git a/posthog/temporal/tests/test_clickhouse.py b/posthog/temporal/tests/test_clickhouse.pyindex c9c361d0d6f4..b9a7a3414a43 100644--- a/posthog/temporal/tests/test_clickhouse.py+++ b/posthog/temporal/tests/test_clickhouse.py@@ -239,16 +239,33 @@ def test_post_query_sends_freshly_read_token(tmp_path):@pytest.mark.parametrize("query,query_parameters,expected",[+ (+ "SELECT %(hogql_val_0)s AS `{hogql_val_0}`",+ {"hogql_val_0": "` AS injected, 'quoted' -- \\"},+ "SELECT '` AS injected, \\'quoted\\' -- \\\\' AS `{hogql_val_0}`",+ ),+ (+ "SELECT %(value)s AS `{{value}}`, %(value)s AS `{value!r}`, %(value)s AS `{value.attr}`",+ {"value": "literal {value} and %(value)s"},+ "SELECT 'literal {value} and %(value)s' AS `{{value}}`, "+ "'literal {value} and %(value)s' AS `{value!r}`, "+ "'literal {value} and %(value)s' AS `{value.attr}`",+ ),+ (+ "SELECT {value:String}, %(value)s",+ {"value": "literal {value}"},+ "SELECT {value:String}, 'literal {value}'",+ ),(# the empty-object sentinel HogQL prints for a native-JSON property read must not become '{0}'- "SELECT if(notEquals(x, '{}'), x, %(fallback)s) FROM t WHERE team_id = {{team_id:Int64}}",+ "SELECT if(notEquals(x, '{}'), x, %(fallback)s) FROM t WHERE team_id = {team_id:Int64}",{"fallback": "{}"},"SELECT if(notEquals(x, '{}'), x, '{}') FROM t WHERE team_id = {team_id:Int64}",),(- "select * from events where event = {event}",+ "select * from events where event = '{event}'",{"event": "hello"},- "select * from events where event = 'hello'",+ "select * from events where event = '{event}'",),("select * from events where event = %(event)s",@@ -256,37 +273,35 @@ def test_post_query_sends_freshly_read_token(tmp_path):"select * from events where event = 'world'",),(- "select * from events where event = %(event)s and event != {another}",+ "select * from events where event = %(event)s and event != %(another)s",{"event": "index_{1}", "another": "event"},"select * from events where event = 'index_{1}' and event != 'event'",),(- "select * from events where event = %(event)s and event != {another}",+ "select * from events where event = %(event)s and event != %(another)s",{"event": "index_{something}", "another": "event"},"select * from events where event = 'index_{something}' and event != 'event'",),# a value containing an unbalanced brace used to crash the format pass(- "select * from events where event = %(event)s and event != {another}",+ "select * from events where event = %(event)s and event != %(another)s",{"event": "brace {", "another": "event"},"select * from events where event = 'brace {' and event != 'event'",),# a JSON string value used to be mangled by the format pass(- "select * from events where properties = %(props)s and event != {another}",+ "select * from events where properties = %(props)s and event != %(another)s",{"props": '{"a": 1}', "another": "event"},"select * from events where properties = '{\"a\": 1}' and event != 'event'",),# a value matching another parameter's name must not be substituted again(- "select * from events where event = %(event)s and event != {another}",+ "select * from events where event = %(event)s and event != %(another)s",{"event": "literal {another}", "another": "event"},"select * from events where event = 'literal {another}' and event != 'event'",),- # the INSERT INTO FUNCTION s3(...) shape used by batch exports: intentional escapes- # collapse while a value containing '%' and braces survives verbatim(- "INSERT INTO FUNCTION s3('bucket/export_{{_partition_id}}.arrow') SELECT %(v)s SETTINGS log_comment={log_comment}",+ "INSERT INTO FUNCTION s3('bucket/export_{_partition_id}.arrow') SELECT %(v)s SETTINGS log_comment=%(log_comment)s",{"v": '100%-{"a": 1}', "log_comment": "comment"},"INSERT INTO FUNCTION s3('bucket/export_{_partition_id}.arrow') SELECT '100%-{\"a\": 1}' SETTINGS log_comment='comment'",),diff --git a/products/batch_exports/backend/temporal/pipeline/internal_stage.py b/products/batch_exports/backend/temporal/pipeline/internal_stage.pyindex 651abb2cf84b..b1a2e0b966c7 100644--- a/products/batch_exports/backend/temporal/pipeline/internal_stage.py+++ b/products/batch_exports/backend/temporal/pipeline/internal_stage.py@@ -572,10 +572,10 @@ async def _get_query(filter_distinct_ids = """HAVING(- _timestamp >= {interval_start}::DateTime64+ _timestamp >= %(interval_start)s::DateTime64)AND (- _timestamp < {interval_end}::DateTime64+ _timestamp < %(interval_end)s::DateTime64)"""else:diff --git a/products/batch_exports/backend/temporal/record_batch_model.py b/products/batch_exports/backend/temporal/record_batch_model.pyindex e4a357f4025f..68a81deee377 100644--- a/products/batch_exports/backend/temporal/record_batch_model.py+++ b/products/batch_exports/backend/temporal/record_batch_model.py@@ -92,7 +92,7 @@ class RecordBatchModel(abc.ABC):# A bit of a hack: the query references neither half of this, but both are required:# - we need to call `get_log_comment` in order to tag the queries# - we need to pass non-empty `values` to `ClickHouseClient.prepare_query` to avoid it returning- # early and not resolving the `{{_partition_id}}` and `%%` escapes in the s3 function+ # early and not resolving the `%%` escapes in the s3 function# call.values={"log_comment": self.get_log_comment()},)@@ -338,7 +338,7 @@ class SessionsRecordBatchModel(RecordBatchModel):stack=[],)- query_settings = "max_execution_time={max_execution_time_seconds}, log_comment={log_comment}"+ query_settings = "max_execution_time=%(max_execution_time_seconds)s, log_comment=%(log_comment)s"if "settings" not in printed.lower():printed += f" SETTINGS {query_settings}"else:diff --git a/products/batch_exports/backend/temporal/sql/common.py b/products/batch_exports/backend/temporal/sql/common.pyindex 8498ac80f4d3..5ed9857c72a1 100644--- a/products/batch_exports/backend/temporal/sql/common.py+++ b/products/batch_exports/backend/temporal/sql/common.py@@ -16,7 +16,7 @@ def get_s3_function_call(s3_folder: str, credentials: AWSKeyPair | None, num_parNote: We use %% for the modulo operator because the ClickHouse client uses % forparameter substitution, so %% produces a literal % in the final query."""- s3_url = f"{s3_folder}/export_{{{{_partition_id}}}}.arrow"+ s3_url = f"{s3_folder}/export_{{_partition_id}}.arrow"if credentials is not None:# Escape single quotes by doubling them (ClickHouse SQL escaping)escaped_key = credentials.access_key_id.replace("'", "''")diff --git a/products/batch_exports/backend/temporal/sql/events.py b/products/batch_exports/backend/temporal/sql/events.pyindex 60230b6de0b8..c21ef9a07710 100644--- a/products/batch_exports/backend/temporal/sql/events.py+++ b/products/batch_exports/backend/temporal/sql/events.py@@ -45,14 +45,14 @@ FROM (FROMeventsPREWHERE- COALESCE(events.inserted_at, events._timestamp) >= {{interval_start:DateTime64}}- AND COALESCE(events.inserted_at, events._timestamp) < {{interval_end:DateTime64}}+ COALESCE(events.inserted_at, events._timestamp) >= {interval_start:DateTime64}+ AND COALESCE(events.inserted_at, events._timestamp) < {interval_end:DateTime64}WHERE- team_id = {{team_id:Int64}}- AND events.timestamp >= {{interval_start:DateTime64}} - INTERVAL {{lookback_days:Int32}} DAY- AND events.timestamp < {{interval_end:DateTime64}} + INTERVAL 1 DAY- AND (length({{include_events:Array(String)}}) = 0 OR event IN {{include_events:Array(String)}})- AND (length({{exclude_events:Array(String)}}) = 0 OR event NOT IN {{exclude_events:Array(String)}})+ team_id = {team_id:Int64}+ AND events.timestamp >= {interval_start:DateTime64} - INTERVAL {lookback_days:Int32} DAY+ AND events.timestamp < {interval_end:DateTime64} + INTERVAL 1 DAY+ AND (length({include_events:Array(String)}) = 0 OR event IN {include_events:Array(String)})+ AND (length({exclude_events:Array(String)}) = 0 OR event NOT IN {exclude_events:Array(String)})$filters$order) AS events@@ -88,21 +88,21 @@ FROM (FROMeventsPREWHERE- COALESCE(events.inserted_at, events._timestamp) >= {{interval_start:DateTime64}}- AND COALESCE(events.inserted_at, events._timestamp) < {{interval_end:DateTime64}}+ COALESCE(events.inserted_at, events._timestamp) >= {interval_start:DateTime64}+ AND COALESCE(events.inserted_at, events._timestamp) < {interval_end:DateTime64}WHERE- team_id = {{team_id:Int64}}- AND events.timestamp >= {{interval_start:DateTime64}} - INTERVAL {{lookback_days:Int32}} DAY- AND events.timestamp < {{interval_end:DateTime64}} + INTERVAL 1 DAY- AND (length({{include_events:Array(String)}}) = 0 OR event IN {{include_events:Array(String)}})- AND (length({{exclude_events:Array(String)}}) = 0 OR event NOT IN {{exclude_events:Array(String)}})+ team_id = {team_id:Int64}+ AND events.timestamp >= {interval_start:DateTime64} - INTERVAL {lookback_days:Int32} DAY+ AND events.timestamp < {interval_end:DateTime64} + INTERVAL 1 DAY+ AND (length({include_events:Array(String)}) = 0 OR event IN {include_events:Array(String)})+ AND (length({exclude_events:Array(String)}) = 0 OR event NOT IN {exclude_events:Array(String)})$filters) AS eventsSETTINGS-- This is half of configured MAX_MEMORY_USAGE for batch exports.max_bytes_before_external_sort=50000000000,optimize_aggregation_in_order=1,- log_comment={log_comment}+ log_comment=%(log_comment)s""")@@ -113,12 +113,12 @@ SELECT$fieldsFROM eventsPREWHERE- COALESCE(events.timestamp) >= {{interval_start:DateTime64}}- AND COALESCE(events.timestamp) < {{interval_end:DateTime64}}+ COALESCE(events.timestamp) >= {interval_start:DateTime64}+ AND COALESCE(events.timestamp) < {interval_end:DateTime64}WHERE- team_id = {{team_id:Int64}}- AND (length({{include_events:Array(String)}}) = 0 OR event IN {{include_events:Array(String)}})- AND (length({{exclude_events:Array(String)}}) = 0 OR event NOT IN {{exclude_events:Array(String)}})+ team_id = {team_id:Int64}+ AND (length({include_events:Array(String)}) = 0 OR event IN {include_events:Array(String)})+ AND (length({exclude_events:Array(String)}) = 0 OR event NOT IN {exclude_events:Array(String)})$filtersFORMAT ArrowStreamSETTINGS@@ -137,17 +137,17 @@ SELECTFROMeventsWHERE- team_id = {{team_id:Int64}}- AND events.timestamp >= {{interval_start:DateTime64}}- AND events.timestamp < {{interval_end:DateTime64}}- AND (length({{include_events:Array(String)}}) = 0 OR event IN {{include_events:Array(String)}})- AND (length({{exclude_events:Array(String)}}) = 0 OR event NOT IN {{exclude_events:Array(String)}})+ team_id = {team_id:Int64}+ AND events.timestamp >= {interval_start:DateTime64}+ AND events.timestamp < {interval_end:DateTime64}+ AND (length({include_events:Array(String)}) = 0 OR event IN {include_events:Array(String)})+ AND (length({exclude_events:Array(String)}) = 0 OR event NOT IN {exclude_events:Array(String)})$filtersSETTINGS-- This is half of configured MAX_MEMORY_USAGE for batch exports.max_bytes_before_external_sort=50000000000,optimize_aggregation_in_order=1,- log_comment={log_comment}+ log_comment=%(log_comment)s""")@@ -174,12 +174,12 @@ FROM (FROMevents_recent AS eventsPREWHERE- events_recent.inserted_at >= {{interval_start:DateTime64}}- AND events_recent.inserted_at < {{interval_end:DateTime64}}+ events_recent.inserted_at >= {interval_start:DateTime64}+ AND events_recent.inserted_at < {interval_end:DateTime64}WHERE- team_id = {{team_id:Int64}}- AND (length({{include_events:Array(String)}}) = 0 OR event IN {{include_events:Array(String)}})- AND (length({{exclude_events:Array(String)}}) = 0 OR event NOT IN {{exclude_events:Array(String)}})+ team_id = {team_id:Int64}+ AND (length({include_events:Array(String)}) = 0 OR event IN {include_events:Array(String)})+ AND (length({exclude_events:Array(String)}) = 0 OR event NOT IN {exclude_events:Array(String)})$filters$order) AS events@@ -216,12 +216,12 @@ FROM (FROMevents_recent AS eventsPREWHERE- events_recent.inserted_at >= {{interval_start:DateTime64}}- AND events_recent.inserted_at < {{interval_end:DateTime64}}+ events_recent.inserted_at >= {interval_start:DateTime64}+ AND events_recent.inserted_at < {interval_end:DateTime64}WHERE- team_id = {{team_id:Int64}}- AND (length({{include_events:Array(String)}}) = 0 OR event IN {{include_events:Array(String)}})- AND (length({{exclude_events:Array(String)}}) = 0 OR event NOT IN {{exclude_events:Array(String)}})+ team_id = {team_id:Int64}+ AND (length({include_events:Array(String)}) = 0 OR event IN {include_events:Array(String)})+ AND (length({exclude_events:Array(String)}) = 0 OR event NOT IN {exclude_events:Array(String)})$filters) AS eventsSETTINGS@@ -229,7 +229,7 @@ SETTINGSmax_bytes_before_external_sort=50000000000,max_replica_delay_for_distributed_queries=1,optimize_aggregation_in_order=1,- log_comment={log_comment}+ log_comment=%(log_comment)s""")@@ -256,12 +256,12 @@ FROM (FROMdistributed_events_recent AS eventsPREWHERE- distributed_events_recent.inserted_at >= {interval_start}::DateTime64- AND distributed_events_recent.inserted_at < {interval_end}::DateTime64+ distributed_events_recent.inserted_at >= %(interval_start)s::DateTime64+ AND distributed_events_recent.inserted_at < %(interval_end)s::DateTime64WHERE- team_id = {team_id}::Int64- AND (length({include_events}::Array(String)) = 0 OR event IN {include_events}::Array(String))- AND (length({exclude_events}::Array(String)) = 0 OR event NOT IN {exclude_events}::Array(String))+ team_id = %(team_id)s::Int64+ AND (length(%(include_events)s::Array(String)) = 0 OR event IN %(include_events)s::Array(String))+ AND (length(%(exclude_events)s::Array(String)) = 0 OR event NOT IN %(exclude_events)s::Array(String))$filters$order) AS events@@ -299,12 +299,12 @@ FROM (FROMdistributed_events_recent AS eventsPREWHERE- distributed_events_recent.inserted_at >= {interval_start}::DateTime64- AND distributed_events_recent.inserted_at < {interval_end}::DateTime64+ distributed_events_recent.inserted_at >= %(interval_start)s::DateTime64+ AND distributed_events_recent.inserted_at < %(interval_end)s::DateTime64WHERE- team_id = {team_id}::Int64- AND (length({include_events}::Array(String)) = 0 OR event IN {include_events}::Array(String))- AND (length({exclude_events}::Array(String)) = 0 OR event NOT IN {exclude_events}::Array(String))+ team_id = %(team_id)s::Int64+ AND (length(%(include_events)s::Array(String)) = 0 OR event IN %(include_events)s::Array(String))+ AND (length(%(exclude_events)s::Array(String)) = 0 OR event NOT IN %(exclude_events)s::Array(String))$filters) AS eventsSETTINGS@@ -313,7 +313,7 @@ SETTINGSmax_replica_delay_for_distributed_queries=60,fallback_to_stale_replicas_for_distributed_queries=0,optimize_aggregation_in_order=1,- log_comment={log_comment}+ log_comment=%(log_comment)s""")@@ -340,12 +340,12 @@ FROM (FROMeventsPREWHERE- COALESCE(events.inserted_at, events._timestamp) >= {{interval_start:DateTime64}}- AND COALESCE(events.inserted_at, events._timestamp) < {{interval_end:DateTime64}}+ COALESCE(events.inserted_at, events._timestamp) >= {interval_start:DateTime64}+ AND COALESCE(events.inserted_at, events._timestamp) < {interval_end:DateTime64}WHERE- team_id = {{team_id:Int64}}- AND (length({{include_events:Array(String)}}) = 0 OR event IN {{include_events:Array(String)}})- AND (length({{exclude_events:Array(String)}}) = 0 OR event NOT IN {{exclude_events:Array(String)}})+ team_id = {team_id:Int64}+ AND (length({include_events:Array(String)}) = 0 OR event IN {include_events:Array(String)})+ AND (length({exclude_events:Array(String)}) = 0 OR event NOT IN {exclude_events:Array(String)})$filters$order) AS events@@ -381,19 +381,19 @@ FROM (FROMeventsPREWHERE- COALESCE(events.inserted_at, events._timestamp) >= {{interval_start:DateTime64}}- AND COALESCE(events.inserted_at, events._timestamp) < {{interval_end:DateTime64}}+ COALESCE(events.inserted_at, events._timestamp) >= {interval_start:DateTime64}+ AND COALESCE(events.inserted_at, events._timestamp) < {interval_end:DateTime64}WHERE- team_id = {{team_id:Int64}}- AND (length({{include_events:Array(String)}}) = 0 OR event IN {{include_events:Array(String)}})- AND (length({{exclude_events:Array(String)}}) = 0 OR event NOT IN {{exclude_events:Array(String)}})+ team_id = {team_id:Int64}+ AND (length({include_events:Array(String)}) = 0 OR event IN {include_events:Array(String)})+ AND (length({exclude_events:Array(String)}) = 0 OR event NOT IN {exclude_events:Array(String)})$filters) AS eventsSETTINGS-- This is half of configured MAX_MEMORY_USAGE for batch exports.max_bytes_before_external_sort=50000000000,optimize_aggregation_in_order=1,- log_comment={log_comment}+ log_comment=%(log_comment)s""")@@ -420,11 +420,11 @@ FROM (FROMeventsWHERE- team_id = {{team_id:Int64}}- AND events.timestamp >= {{interval_start:DateTime64}}- AND events.timestamp < {{interval_end:DateTime64}}- AND (length({{include_events:Array(String)}}) = 0 OR event IN {{include_events:Array(String)}})- AND (length({{exclude_events:Array(String)}}) = 0 OR event NOT IN {{exclude_events:Array(String)}})+ team_id = {team_id:Int64}+ AND events.timestamp >= {interval_start:DateTime64}+ AND events.timestamp < {interval_end:DateTime64}+ AND (length({include_events:Array(String)}) = 0 OR event IN {include_events:Array(String)})+ AND (length({exclude_events:Array(String)}) = 0 OR event NOT IN {exclude_events:Array(String)})$filters$order) as events@@ -460,26 +460,26 @@ FROM (FROMeventsWHERE- team_id = {{team_id:Int64}}- AND events.timestamp >= {{interval_start:DateTime64}}- AND events.timestamp < {{interval_end:DateTime64}}- AND (length({{include_events:Array(String)}}) = 0 OR event IN {{include_events:Array(String)}})- AND (length({{exclude_events:Array(String)}}) = 0 OR event NOT IN {{exclude_events:Array(String)}})+ team_id = {team_id:Int64}+ AND events.timestamp >= {interval_start:DateTime64}+ AND events.timestamp < {interval_end:DateTime64}+ AND (length({include_events:Array(String)}) = 0 OR event IN {include_events:Array(String)})+ AND (length({exclude_events:Array(String)}) = 0 OR event NOT IN {exclude_events:Array(String)})$filters) as eventsSETTINGS-- This is half of configured MAX_MEMORY_USAGE for batch exports.max_bytes_before_external_sort=50000000000,optimize_aggregation_in_order=1,- log_comment={log_comment}+ log_comment=%(log_comment)s""")EVENT_COUNT_BY_INTERVAL = """SELECT- toStartOfInterval(_inserted_at, INTERVAL {interval}) AS interval_start,- interval_start + INTERVAL {interval} AS interval_end,+ toStartOfInterval(_inserted_at, INTERVAL %(interval)s) AS interval_start,+ interval_start + INTERVAL %(interval)s AS interval_end,COUNT(*) as total_countFROM (SELECT DISTINCT ON (team_id, event, cityHash64(events_recent.distinct_id), cityHash64(events_recent.uuid))@@ -487,12 +487,12 @@ FROM (FROMevents_recentPREWHERE- events_recent.inserted_at >= {overall_interval_start}::DateTime64- AND events_recent.inserted_at < {overall_interval_end}::DateTime64+ events_recent.inserted_at >= %(overall_interval_start)s::DateTime64+ AND events_recent.inserted_at < %(overall_interval_end)s::DateTime64WHERE- team_id = {team_id}::Int64- AND (length({include_events}::Array(String)) = 0 OR event IN {include_events}::Array(String))- AND (length({exclude_events}::Array(String)) = 0 OR event NOT IN {exclude_events}::Array(String))+ team_id = %(team_id)s::Int64+ AND (length(%(include_events)s::Array(String)) = 0 OR event IN %(include_events)s::Array(String))+ AND (length(%(exclude_events)s::Array(String)) = 0 OR event NOT IN %(exclude_events)s::Array(String)))GROUP BY interval_startORDER BY interval_start ASC@@ -502,18 +502,10 @@ SETTINGS"""-# The `{{}}` literals below are not a typo. `ClickHouseClient.prepare_query` runs `str.format` over-# the whole query, which turns `{{}}` back into `{}`. A bare `{}` would parse as a positional field-# and reach ClickHouse as `{0}`, so the `nullIf` would never match an empty object. The rebuilt document-# gets the same doubling for the same reason.# `toJSONString` prints a stored dotted key as `a%2Eb` unless the query sets `json_type_escape_dots_in_keys`,# which is why the native query's SETTINGS carry it.-_PROPERTIES_DOCUMENT = (- event_document_sql("properties", "temporary_properties", "properties.`$feature_flags`")- .replace("{", "{{")- .replace("}", "}}")-)-_PERSON_PROPERTIES_DOCUMENT = person_document_sql("person_properties").replace("{", "{{").replace("}", "}}")+_PROPERTIES_DOCUMENT = event_document_sql("properties", "temporary_properties", "properties.`$feature_flags`")+_PERSON_PROPERTIES_DOCUMENT = person_document_sql("person_properties")SERIALIZED_EVENTS_JSON_SOURCE = """(SELECT * REPLACE (toString(uuid) AS uuid,@@ -521,10 +513,10 @@ SERIALIZED_EVENTS_JSON_SOURCE = """(__PROPERTIES_DOCUMENT__ AS properties,__PERSON_PROPERTIES_DOCUMENT__ AS person_properties),- nullIf(toJSONString(temporary_properties.^`$set`), '{{}}') AS set,- nullIf(toJSONString(temporary_properties.^`$set_once`), '{{}}') AS set_once,+ nullIf(toJSONString(temporary_properties.^`$set`), '{}') AS set,+ nullIf(toJSONString(temporary_properties.^`$set_once`), '{}') AS set_once,nullIf(toJSONString(temporary_properties.^`$unset`), '[]') AS unset,- nullIf(toJSONString(temporary_properties.^`$group_set`), '{{}}') AS group_set+ nullIf(toJSONString(temporary_properties.^`$group_set`), '{}') AS group_setFROM events_json)""".replace("__PROPERTIES_DOCUMENT__", _PROPERTIES_DOCUMENT).replace("__PERSON_PROPERTIES_DOCUMENT__", _PERSON_PROPERTIES_DOCUMENT@@ -545,11 +537,11 @@ FROM (SELECT DISTINCT ON (team_id, event, cityHash64(events.distinct_id), cityHash64(events.uuid))*, timestamp AS _inserted_atFROM {SERIALIZED_EVENTS_JSON_SOURCE} AS events- WHERE team_id = {{{{team_id:Int64}}}}- AND ({{interval_start}}::Nullable(DateTime64) IS NULL OR timestamp >= {{interval_start}}::Nullable(DateTime64))- AND timestamp < {{{{interval_end:DateTime64}}}}- AND (length({{{{include_events:Array(String)}}}}) = 0 OR event IN {{{{include_events:Array(String)}}}})- AND (length({{{{exclude_events:Array(String)}}}}) = 0 OR event NOT IN {{{{exclude_events:Array(String)}}}})+ WHERE team_id = {{team_id:Int64}}+ AND (%(interval_start)s::Nullable(DateTime64) IS NULL OR timestamp >= %(interval_start)s::Nullable(DateTime64))+ AND timestamp < {{interval_end:DateTime64}}+ AND (length({{include_events:Array(String)}}) = 0 OR event IN {{include_events:Array(String)}})+ AND (length({{exclude_events:Array(String)}}) = 0 OR event NOT IN {{exclude_events:Array(String)}}){"AND " + filters if filters else ""}{order}) AS events@@ -561,5 +553,5 @@ SETTINGSoptimize_aggregation_in_order=1,json_type_escape_dots_in_keys=1,output_format_json_escape_forward_slashes=0-{", log_comment={log_comment}" if s3_function else ""}+{", log_comment=%(log_comment)s" if s3_function else ""}"""diff --git a/products/batch_exports/backend/temporal/sql/persons.py b/products/batch_exports/backend/temporal/sql/persons.pyindex 15db3d6c385f..9f0eba17ca6b 100644--- a/products/batch_exports/backend/temporal/sql/persons.py+++ b/products/batch_exports/backend/temporal/sql/persons.py@@ -20,22 +20,22 @@ FROM (FROMpersonWHERE- team_id = {team_id}::Int64+ team_id = %(team_id)s::Int64AND id in (SELECTidFROMpersonWHERE- team_id = {team_id}::Int64- AND _timestamp >= {interval_start}::DateTime64- AND _timestamp < {interval_end}::DateTime64+ team_id = %(team_id)s::Int64+ AND _timestamp >= %(interval_start)s::DateTime64+ AND _timestamp < %(interval_end)s::DateTime64)GROUP BYidHAVING- argMax(_timestamp, person.version) >= {interval_start}::DateTime64- AND argMax(_timestamp, person.version) < {interval_end}::DateTime64+ argMax(_timestamp, person.version) >= %(interval_start)s::DateTime64+ AND argMax(_timestamp, person.version) < %(interval_end)s::DateTime64),distinct_ids AS (-- new distinct_ids@@ -44,22 +44,22 @@ FROM (FROMperson_distinct_id2WHERE- team_id = {team_id}::Int64+ team_id = %(team_id)s::Int64AND distinct_id in (SELECTdistinct_idFROMperson_distinct_id2WHERE- team_id = {team_id}::Int64- AND _timestamp >= {interval_start}::DateTime64- AND _timestamp < {interval_end}::DateTime64+ team_id = %(team_id)s::Int64+ AND _timestamp >= %(interval_start)s::DateTime64+ AND _timestamp < %(interval_end)s::DateTime64)GROUP BYdistinct_idHAVING- argMax(_timestamp, person_distinct_id2.version) >= {interval_start}::DateTime64- AND argMax(_timestamp, person_distinct_id2.version) < {interval_end}::DateTime64+ argMax(_timestamp, person_distinct_id2.version) >= %(interval_start)s::DateTime64+ AND argMax(_timestamp, person_distinct_id2.version) < %(interval_end)s::DateTime64UNION DISTINCT@@ -69,7 +69,7 @@ FROM (FROMperson_distinct_id2WHERE- team_id = {team_id}::Int64+ team_id = %(team_id)s::Int64AND person_id IN new_persons),latest_distinct_ids AS (@@ -81,7 +81,7 @@ FROM (FROMperson_distinct_id2WHERE- team_id = {team_id}::Int64+ team_id = %(team_id)s::Int64AND distinct_id IN distinct_idsGROUP BYdistinct_id@@ -94,7 +94,7 @@ FROM (FROMpersonWHERE- team_id = {team_id}::Int64+ team_id = %(team_id)s::Int64ANDid IN (SELECT@@ -116,21 +116,21 @@ FROM (toBool(p.is_deleted) AS is_deleted,multiIf((- pd._timestamp >= {interval_start}::DateTime64- AND pd._timestamp < {interval_end}::DateTime64+ pd._timestamp >= %(interval_start)s::DateTime64+ AND pd._timestamp < %(interval_end)s::DateTime64)AND NOT (- p._timestamp >= {interval_start}::DateTime64- AND p._timestamp < {interval_end}::DateTime64+ p._timestamp >= %(interval_start)s::DateTime64+ AND p._timestamp < %(interval_end)s::DateTime64),pd._timestamp,(- p._timestamp >= {interval_start}::DateTime64- AND p._timestamp < {interval_end}::DateTime64+ p._timestamp >= %(interval_start)s::DateTime64+ AND p._timestamp < %(interval_end)s::DateTime64)AND NOT (- pd._timestamp >= {interval_start}::DateTime64- AND pd._timestamp < {interval_end}::DateTime64+ pd._timestamp >= %(interval_start)s::DateTime64+ AND pd._timestamp < %(interval_end)s::DateTime64),p._timestamp,least(p._timestamp, pd._timestamp)@@ -140,7 +140,7 @@ FROM (INNER JOIN latest_distinct_ids pdON p.id = pd.person_id2WHERE- p.team_id = {team_id}::Int64+ p.team_id = %(team_id)s::Int64AND (p.id, p.version, p._timestamp) in latest_person_versionsORDER BY_inserted_at@@ -172,22 +172,22 @@ FROM (FROMpersonWHERE- team_id = {team_id}::Int64+ team_id = %(team_id)s::Int64AND id in (SELECTidFROMpersonWHERE- team_id = {team_id}::Int64- AND _timestamp >= {interval_start}::DateTime64- AND _timestamp < {interval_end}::DateTime64+ team_id = %(team_id)s::Int64+ AND _timestamp >= %(interval_start)s::DateTime64+ AND _timestamp < %(interval_end)s::DateTime64)GROUP BYidHAVING- argMax(_timestamp, person.version) >= {interval_start}::DateTime64- AND argMax(_timestamp, person.version) < {interval_end}::DateTime64+ argMax(_timestamp, person.version) >= %(interval_start)s::DateTime64+ AND argMax(_timestamp, person.version) < %(interval_end)s::DateTime64),distinct_ids AS (-- new distinct_ids@@ -196,22 +196,22 @@ FROM (FROMperson_distinct_id2WHERE- team_id = {team_id}::Int64+ team_id = %(team_id)s::Int64AND distinct_id in (SELECTdistinct_idFROMperson_distinct_id2WHERE- team_id = {team_id}::Int64- AND _timestamp >= {interval_start}::DateTime64- AND _timestamp < {interval_end}::DateTime64+ team_id = %(team_id)s::Int64+ AND _timestamp >= %(interval_start)s::DateTime64+ AND _timestamp < %(interval_end)s::DateTime64)GROUP BYdistinct_idHAVING- argMax(_timestamp, person_distinct_id2.version) >= {interval_start}::DateTime64- AND argMax(_timestamp, person_distinct_id2.version) < {interval_end}::DateTime64+ argMax(_timestamp, person_distinct_id2.version) >= %(interval_start)s::DateTime64+ AND argMax(_timestamp, person_distinct_id2.version) < %(interval_end)s::DateTime64UNION DISTINCT@@ -221,7 +221,7 @@ FROM (FROMperson_distinct_id2WHERE- team_id = {team_id}::Int64+ team_id = %(team_id)s::Int64AND person_id IN new_persons),latest_distinct_ids AS (@@ -233,7 +233,7 @@ FROM (FROMperson_distinct_id2WHERE- team_id = {team_id}::Int64+ team_id = %(team_id)s::Int64AND distinct_id IN distinct_idsGROUP BYdistinct_id@@ -247,7 +247,7 @@ FROM (FROMpersonWHERE- team_id = {team_id}::Int64+ team_id = %(team_id)s::Int64ANDid IN (SELECT@@ -269,21 +269,21 @@ FROM (toBool(p.is_deleted) AS is_deleted,multiIf((- pd._timestamp >= {interval_start}::DateTime64- AND pd._timestamp < {interval_end}::DateTime64+ pd._timestamp >= %(interval_start)s::DateTime64+ AND pd._timestamp < %(interval_end)s::DateTime64)AND NOT (- p._timestamp >= {interval_start}::DateTime64- AND p._timestamp < {interval_end}::DateTime64+ p._timestamp >= %(interval_start)s::DateTime64+ AND p._timestamp < %(interval_end)s::DateTime64),pd._timestamp,(- p._timestamp >= {interval_start}::DateTime64- AND p._timestamp < {interval_end}::DateTime64+ p._timestamp >= %(interval_start)s::DateTime64+ AND p._timestamp < %(interval_end)s::DateTime64)AND NOT (- pd._timestamp >= {interval_start}::DateTime64- AND pd._timestamp < {interval_end}::DateTime64+ pd._timestamp >= %(interval_start)s::DateTime64+ AND pd._timestamp < %(interval_end)s::DateTime64),p._timestamp,least(p._timestamp, pd._timestamp)@@ -293,14 +293,14 @@ FROM (INNER JOIN latest_distinct_ids pdON p.id = pd.person_id2WHERE- p.team_id = {team_id}::Int64+ p.team_id = %(team_id)s::Int64AND (p.id, p.version, p._timestamp) in latest_person_versions) AS personsSETTINGSmax_bytes_before_external_group_by=50000000000,max_bytes_before_external_sort=50000000000,optimize_aggregation_in_order=1,- log_comment={log_comment}+ log_comment=%(log_comment)s""")@@ -315,11 +315,11 @@ SELECTp.created_at AS created_at,toBool(p.is_deleted) AS is_deleted,multiIf(- pd._timestamp < {interval_end}::DateTime64- AND NOT p._timestamp < {interval_end}::DateTime64,+ pd._timestamp < %(interval_end)s::DateTime64+ AND NOT p._timestamp < %(interval_end)s::DateTime64,pd._timestamp,- p._timestamp < {interval_end}::DateTime64- AND NOT pd._timestamp < {interval_end}::DateTime64,+ p._timestamp < %(interval_end)s::DateTime64+ AND NOT pd._timestamp < %(interval_end)s::DateTime64,p._timestamp,least(p._timestamp, pd._timestamp)) AS _inserted_at@@ -333,7 +333,7 @@ FROM (FROMperson_distinct_id2PREWHERE- team_id = {team_id}::Int64+ team_id = %(team_id)s::Int64GROUP BYteam_id,distinct_id@@ -350,17 +350,17 @@ INNER JOIN (FROMpersonPREWHERE- team_id = {team_id}::Int64+ team_id = %(team_id)s::Int64GROUP BYteam_id,id) AS p ON p.id = pd.person_id AND p.team_id = pd.team_idWHERE- pd.team_id = {team_id}::Int64- AND p.team_id = {team_id}::Int64+ pd.team_id = %(team_id)s::Int64+ AND p.team_id = %(team_id)s::Int64AND (- pd._timestamp < {interval_end}::DateTime64- OR p._timestamp < {interval_end}::DateTime64+ pd._timestamp < %(interval_end)s::DateTime64+ OR p._timestamp < %(interval_end)s::DateTime64)ORDER BY_inserted_at@@ -384,11 +384,11 @@ SELECTp.created_at AS created_at,toBool(p.is_deleted) AS is_deleted,multiIf(- pd._timestamp < {interval_end}::DateTime64- AND NOT p._timestamp < {interval_end}::DateTime64,+ pd._timestamp < %(interval_end)s::DateTime64+ AND NOT p._timestamp < %(interval_end)s::DateTime64,pd._timestamp,- p._timestamp < {interval_end}::DateTime64- AND NOT pd._timestamp < {interval_end}::DateTime64,+ p._timestamp < %(interval_end)s::DateTime64+ AND NOT pd._timestamp < %(interval_end)s::DateTime64,p._timestamp,least(p._timestamp, pd._timestamp)) AS _inserted_at@@ -402,7 +402,7 @@ FROM (FROMperson_distinct_id2PREWHERE- team_id = {team_id}::Int64+ team_id = %(team_id)s::Int64GROUP BYteam_id,distinct_id@@ -419,21 +419,21 @@ INNER JOIN (FROMpersonPREWHERE- team_id = {team_id}::Int64+ team_id = %(team_id)s::Int64GROUP BYteam_id,id) AS p ON p.id = pd.person_id AND p.team_id = pd.team_idWHERE- pd.team_id = {team_id}::Int64- AND p.team_id = {team_id}::Int64+ pd.team_id = %(team_id)s::Int64+ AND p.team_id = %(team_id)s::Int64AND (- pd._timestamp < {interval_end}::DateTime64- OR p._timestamp < {interval_end}::DateTime64+ pd._timestamp < %(interval_end)s::DateTime64+ OR p._timestamp < %(interval_end)s::DateTime64)SETTINGSmax_bytes_before_external_group_by=50000000000,max_bytes_before_external_sort=50000000000,optimize_aggregation_in_order=1,- log_comment={log_comment}+ log_comment=%(log_comment)s""")diff --git a/products/batch_exports/backend/tests/temporal/destinations/s3/utils.py b/products/batch_exports/backend/tests/temporal/destinations/s3/utils.pyindex f97a92696186..4c403de99f4b 100644--- a/products/batch_exports/backend/tests/temporal/destinations/s3/utils.py+++ b/products/batch_exports/backend/tests/temporal/destinations/s3/utils.py@@ -170,7 +170,7 @@ async def assert_metrics_in_clickhouse(SELECT metric_kind, metric_name, countFROM sharded_app_metrics2WHERE app_source = 'batch_export'- AND app_source_id = {{batch_export_id:String}}+ AND app_source_id = {batch_export_id:String}FORMAT JSONEachRow \"""diff --git a/products/batch_exports/backend/tests/temporal/pipeline/test_internal_stage.py b/products/batch_exports/backend/tests/temporal/pipeline/test_internal_stage.pyindex 4e914cca6daa..f2472fe51ef3 100644--- a/products/batch_exports/backend/tests/temporal/pipeline/test_internal_stage.py+++ b/products/batch_exports/backend/tests/temporal/pipeline/test_internal_stage.py@@ -1410,6 +1410,18 @@ class TestHogQLModel:lambda events, persons: [(e["event"], e["distinct_id"], "Chrome") for e in events],id="events-with-special-chars",),+ pytest.param(+ r"""+ SELECT 'quote\' backtick` slash\\ {log_comment} %(other)s' AS `{hogql_val_0}`,+ event AS `{log_comment}`, distinct_id AS `{{hogql_val_0}}`+ FROM events+ """,+ ["{hogql_val_0}", "{log_comment}", "{{hogql_val_0}}"],+ lambda events, persons: [+ ("quote' backtick` slash\\ {log_comment} %(other)s", e["event"], e["distinct_id"]) for e in events+ ],+ id="formatter-shaped-aliases",+ ),pytest.param("SELECT toString(id) AS id, properties.utm_medium AS utm_medium FROM persons",["id", "utm_medium"],diff --git a/products/batch_exports/backend/tests/temporal/test_record_batch_model.py b/products/batch_exports/backend/tests/temporal/test_record_batch_model.pyindex 4782cc6374c9..d06cd547c1c1 100644--- a/products/batch_exports/backend/tests/temporal/test_record_batch_model.py+++ b/products/batch_exports/backend/tests/temporal/test_record_batch_model.py@@ -256,8 +256,7 @@ class TestSessionsRecordBatchModel:)assert "INSERT INTO FUNCTION" in printed_query- # parition_id is a ClickHouse variable, so we need to escape it- assert "https://test-bucket.s3.amazonaws.com/test-prefix/export_{{_partition_id}}.arrow" in printed_query+ assert "https://test-bucket.s3.amazonaws.com/test-prefix/export_{_partition_id}.arrow" in printed_queryassert "PARTITION BY rand() %% 5" in printed_queryassert f"equals(raw_sessions.team_id, {ateam.id})" in printed_queryassert (@@ -302,10 +301,10 @@ class TestSessionsRecordBatchModel:)assert "INSERT INTO FUNCTION" in printed_query- assert "https://test-bucket.s3.amazonaws.com/test-prefix/export_{{_partition_id}}.arrow" in printed_query+ assert "https://test-bucket.s3.amazonaws.com/test-prefix/export_{_partition_id}.arrow" in printed_query# For keyless auth, the s3() call should only have 2 parameters (url, format), not 4assert (- "s3('https://test-bucket.s3.amazonaws.com/test-prefix/export_{{_partition_id}}.arrow', 'ArrowStream')"+ "s3('https://test-bucket.s3.amazonaws.com/test-prefix/export_{_partition_id}.arrow', 'ArrowStream')"in printed_query)assert "PARTITION BY rand() %% 5" in printed_query@@ -510,7 +509,7 @@ class TestHogQLQueryRecordBatchModel:)assert "INSERT INTO FUNCTION" in printed_query- assert "https://test-bucket.s3.amazonaws.com/test-prefix/export_{{_partition_id}}.arrow" in printed_query+ assert "https://test-bucket.s3.amazonaws.com/test-prefix/export_{_partition_id}.arrow" in printed_queryassert "PARTITION BY rand() %% 5" in printed_queryassert f"equals(events.team_id, {ateam.id})" in printed_query# the user's query is wrapped as-is: settings are sent as query parameters, so we never write a