Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(hogql): preserve query authorization during parameter binding

Marius Andra69adc2021c8c
diff --git a/docs/published/handbook/engineering/databases/hogql-python.md b/docs/published/handbook/engineering/databases/hogql-python.md
index 4d0ec965e750..1148ac285084 100644
--- a/docs/published/handbook/engineering/databases/hogql-python.md
+++ b/docs/published/handbook/engineering/databases/hogql-python.md
@@ -105,6 +105,20 @@ query_result.columns == ['event', 'timestamp'] # might be useful if you select
You can mix and match `parse_expr` and `ast` nodes as you please. The example above _still_ took a shortcut for the where clause because it was easier to write.
+Pass compiled ClickHouse SQL and `HogQLContext.values` together to the database client. The printer reserves `%(name)s` placeholders for values and rejects percent signs in identifiers. Do not apply brace formatting to compiled SQL: braces are valid identifier characters and must remain literal after authorization.
+
+For trusted SQL templates passed to the Temporal `ClickHouseClient`, distinguish the layer that consumes each placeholder:
+
+| Purpose | Plain Python string | Python f-string source | Consumed by |
+| -------------------- | ------------------- | ---------------------- | ------------------------------------------------------- |
+| Client-bound value | `%(name)s` | `%(name)s` | `prepare_query`, after SQL value escaping |
+| Server-bound value | `{name:Type}` | `{{name:Type}}` | ClickHouse, using the HTTP `param_name` parameter |
+| S3 partition marker | `{_partition_id}` | `{{_partition_id}}` | ClickHouse |
+| Empty JSON object | `'{}'` | `'{{}}'` | Neither; SQL string data |
+| Literal percent sign | `%%` | `%%` | `prepare_query`, when the parameter mapping is nonempty |
+
+An f-string consumes one brace-escaping layer in its source. It does not reinterpret braces inside an interpolated string. `string.Template` substitutes `$name` and leaves braces unchanged. If a trusted template uses `.format()`, that call also needs its own brace escaping. Count these construction steps separately from database binding; do not halve braces across a query indiscriminately.
+
## Snowflake date formatting
For direct Snowflake queries, `formatDateTime` requires a literal format string and translates supported strftime specifiers into Snowflake format elements.
diff --git a/posthog/api/proxy_record_diagnostics.py b/posthog/api/proxy_record_diagnostics.py
index 06ed9bd6e094..00365aa66e5b 100644
--- a/posthog/api/proxy_record_diagnostics.py
+++ b/posthog/api/proxy_record_diagnostics.py
@@ -191,7 +191,9 @@ def diagnose(record: ProxyRecord) -> DiagnosticReport:
predate the Cloudflare path (where a Cloudflare lookup always comes back empty and would
otherwise read as a false "we don't have a record of this proxy").
"""
- log = LOGGER.bind(
+ # The configured logger context is thread-local; binding diagnostic fields
+ # directly would attach them to unrelated logs after this request returns.
+ log = structlog.threadlocal.as_immutable(LOGGER).bind(
proxy_record_id=str(record.id),
organization_id=str(record.organization_id),
domain=record.domain,
diff --git a/posthog/api/test/test_proxy_record_diagnostics.py b/posthog/api/test/test_proxy_record_diagnostics.py
index 8f1d2bc559bf..12b45cf2ea6d 100644
--- a/posthog/api/test/test_proxy_record_diagnostics.py
+++ b/posthog/api/test/test_proxy_record_diagnostics.py
@@ -6,6 +6,7 @@ from unittest.mock import MagicMock, patch
from django.test import SimpleTestCase, TestCase, override_settings
import requests
+import structlog
import dns.resolver
from parameterized import parameterized
@@ -620,8 +621,10 @@ class TestDiagnoseOrchestrator(TestCase):
# fired at all rather than on how a half-configured mock happens to fail.
post_mock.return_value = MagicMock(status_code=200)
+ log_context = dict(structlog.get_context(diagnostics.LOGGER.bind()))
report = diagnostics.diagnose(_record(domain="169.254.169.254:80/pad.attacker.example", target=CF_TARGET))
+ self.assertEqual(dict(structlog.get_context(diagnostics.LOGGER.bind())), log_context)
post_mock.assert_not_called()
http_get_mock.assert_not_called()
socket_mock.assert_not_called()
diff --git a/posthog/temporal/common/clickhouse.py b/posthog/temporal/common/clickhouse.py
index 37de0a5aa72c..3e0351e00607 100644
--- a/posthog/temporal/common/clickhouse.py
+++ b/posthog/temporal/common/clickhouse.py
@@ -10,7 +10,6 @@ import asyncio
import datetime as dt
import contextlib
import collections.abc
-from string import Formatter
from urllib.parse import urljoin
from django.conf import settings
@@ -313,23 +312,6 @@ def add_log_comment_param(params: dict[str, typing.Any], query_tags: typing.Opti
params[param_name] = query_tags.to_json()
-class KeywordOnlyFormatter(Formatter):
- """Formatter supporting only keyword arguments.
-
- Positional arguments are unchanged, missing keys are also left unchanged.
- """
-
- def get_value(self, key, args, kwargs):
- if isinstance(key, int):
- # Returns '{n}' unchanged, where n is a numerical index.
- return f"{{{key}}}"
- try:
- return kwargs[key]
- except KeyError:
- # Returns '{key}' unchanged
- return f"{{{key}}}"
-
-
class ClickHouseClient:
"""An asynchronous client to access ClickHouse via HTTP.
@@ -429,23 +411,11 @@ class ClickHouseClient:
if not query_parameters:
return query
- has_format_placeholders = re.search(r"(?<!{){[^{}]*}(?!})|{{[^{}]*}}", query)
-
format_parameters = {k: encode_clickhouse_data(v).decode("utf-8") for k, v in query_parameters.items()}
-
- if has_format_placeholders:
- # Escape any curly brackets `{` or `}` in the format parameters so they are not parsed
- # as format placeholders
- escaped_parameters = {k: v.replace("{", "{{").replace("}", "}}") for k, v in format_parameters.items()}
- query = query % escaped_parameters
- # HogQL prints the empty-object sentinel '{}' inline for every property read on the native-JSON
- # events table, and the formatter would auto-number it to '{0}'. An empty pair is never a placeholder.
- query = re.sub(r"(?<!\{)\{\}(?!\})", "{{}}", query)
- query = KeywordOnlyFormatter().format(query, **format_parameters)
- else:
- query = query % format_parameters
-
- return query
+ # HogQL reserves percent placeholders for values and rejects percent signs in identifiers.
+ # Braces can occur in authorized identifiers, so interpreting them here would change the
+ # query after authorization. Leave typed ClickHouse parameters and all other braces intact.
+ return query % format_parameters
def prepare_request_data(self, data: collections.abc.Sequence[typing.Any]) -> bytes | None:
"""Prepare the request data sent by encoding it.
@@ -842,8 +812,8 @@ class ClickHouseClient:
"""
query = """
SELECT type, exception
- FROM clusterAllReplicas({{cluster_name:String}}, system.query_log)
- WHERE query_id = {{query_id:String}}
+ FROM clusterAllReplicas({cluster_name:String}, system.query_log)
+ WHERE query_id = {query_id:String}
AND event_date >= yesterday() AND event_time >= now() - interval 24 hour
FORMAT JSONEachRow
"""
@@ -907,8 +877,8 @@ class ClickHouseClient:
"""
query = """
SELECT written_rows
- FROM clusterAllReplicas({{cluster_name:String}}, system.query_log)
- WHERE query_id = {{query_id:String}}
+ FROM clusterAllReplicas({cluster_name:String}, system.query_log)
+ WHERE query_id = {query_id:String}
AND type = 'QueryFinish'
AND is_initial_query = 1
AND event_date >= yesterday() AND event_time >= now() - interval 24 hour
@@ -948,8 +918,8 @@ class ClickHouseClient:
"""
query = """
SELECT 1
- FROM clusterAllReplicas({{cluster_name:String}}, system.processes)
- WHERE query_id = {{query_id:String}}
+ FROM clusterAllReplicas({cluster_name:String}, system.processes)
+ WHERE query_id = {query_id:String}
AND NOT is_cancelled
LIMIT 1
"""
@@ -976,7 +946,7 @@ class ClickHouseClient:
Arguments:
query_id: The ID of the query to cancel.
"""
- query = f"KILL QUERY ON CLUSTER '{settings.CLICKHOUSE_CLUSTER}' WHERE query_id = {{{{query_id:String}}}}"
+ query = f"KILL QUERY ON CLUSTER '{settings.CLICKHOUSE_CLUSTER}' WHERE query_id = {{query_id:String}}"
await self.execute_query(
query,
diff --git a/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py b/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py
index b959b55fdb71..4d1de8e44999 100644
--- a/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py
+++ b/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py
@@ -24,7 +24,11 @@ from posthog.hogql.resolver import ResolverFactory
from posthog.clickhouse.query_tagging import get_query_tags
from posthog.models import Team, User
from posthog.sync import database_sync_to_async
-from posthog.temporal.common.clickhouse import ClickHouseError, ClickHouseTooManySimultaneousQueriesError
+from posthog.temporal.common.clickhouse import (
+ ClickHouseClient,
+ ClickHouseError,
+ ClickHouseTooManySimultaneousQueriesError,
+)
from posthog.temporal.common.errors import NonReportableError
from posthog.temporal.data_modeling.activities import (
CreateDataModelingJobInputs,
@@ -57,6 +61,7 @@ from posthog.temporal.data_modeling.activities.materialize_view_managed_warehous
materialize_view_managed_warehouse_activity,
)
from posthog.temporal.data_modeling.activities.notify_materialization_failure import _SavedQueryViewers
+from posthog.temporal.tests.utils.events import generate_test_events_in_clickhouse
from products.customer_analytics.backend.facade.temporal import stage_warehouse_account_property_files_activity
from products.customer_analytics.backend.facade.temporal_contracts import StageAccountPropertySyncInput
@@ -2048,6 +2053,41 @@ class _EmptyArrowClient:
class TestHogqlTableModifiers:
+ async def test_formatter_shaped_aliases_preserve_values_and_team_scope(
+ self, ateam: Team, clickhouse_client: ClickHouseClient
+ ) -> None:
+ events, _, other_team_events = await generate_test_events_in_clickhouse(
+ client=clickhouse_client,
+ team_id=ateam.pk,
+ start_time=dt.datetime(2024, 1, 1, tzinfo=dt.UTC),
+ end_time=dt.datetime(2024, 1, 2, tzinfo=dt.UTC),
+ count=2,
+ count_outside_range=0,
+ count_other_team=2,
+ table="sharded_events",
+ )
+
+ @contextlib.asynccontextmanager
+ async def get_test_client(**kwargs: object) -> AsyncIterator[ClickHouseClient]:
+ yield clickhouse_client
+
+ query = r"""
+ SELECT 'quote\' backtick` slash\\ {log_comment} %(other)s' AS `{hogql_val_0}`,
+ event AS `{log_comment}`, uuid AS `{{hogql_val_0}}`
+ FROM events
+ """
+ with unittest.mock.patch(
+ "posthog.temporal.data_modeling.activities.materialize_view.get_clickhouse_client", get_test_client
+ ):
+ batches = [batch async for batch in hogql_table(query, ateam, LOGGER.bind())]
+
+ rows = [row for table, _ in batches for row in table.to_pylist()]
+ assert len(rows) == len(events)
+ assert all(list(row) == ["{hogql_val_0}", "{log_comment}", "{{hogql_val_0}}"] for row in rows)
+ assert {row["{hogql_val_0}"] for row in rows} == {"quote' backtick` slash\\ {log_comment} %(other)s"}
+ assert {row["{{hogql_val_0}}"] for row in rows} == {event["uuid"] for event in events}
+ assert not {row["{{hogql_val_0}}"] for row in rows} & {event["uuid"] for event in other_team_events}
+
@pytest.mark.parametrize(
"clickhouse_type,expected_conversion",
[
diff --git a/posthog/temporal/tests/test_clickhouse.py b/posthog/temporal/tests/test_clickhouse.py
index c9c361d0d6f4..b9a7a3414a43 100644
--- a/posthog/temporal/tests/test_clickhouse.py
+++ b/posthog/temporal/tests/test_clickhouse.py
@@ -239,16 +239,33 @@ def test_post_query_sends_freshly_read_token(tmp_path):
@pytest.mark.parametrize(
"query,query_parameters,expected",
[
+ (
+ "SELECT %(hogql_val_0)s AS `{hogql_val_0}`",
+ {"hogql_val_0": "` AS injected, 'quoted' -- \\"},
+ "SELECT '` AS injected, \\'quoted\\' -- \\\\' AS `{hogql_val_0}`",
+ ),
+ (
+ "SELECT %(value)s AS `{{value}}`, %(value)s AS `{value!r}`, %(value)s AS `{value.attr}`",
+ {"value": "literal {value} and %(value)s"},
+ "SELECT 'literal {value} and %(value)s' AS `{{value}}`, "
+ "'literal {value} and %(value)s' AS `{value!r}`, "
+ "'literal {value} and %(value)s' AS `{value.attr}`",
+ ),
+ (
+ "SELECT {value:String}, %(value)s",
+ {"value": "literal {value}"},
+ "SELECT {value:String}, 'literal {value}'",
+ ),
(
# the empty-object sentinel HogQL prints for a native-JSON property read must not become '{0}'
- "SELECT if(notEquals(x, '{}'), x, %(fallback)s) FROM t WHERE team_id = {{team_id:Int64}}",
+ "SELECT if(notEquals(x, '{}'), x, %(fallback)s) FROM t WHERE team_id = {team_id:Int64}",
{"fallback": "{}"},
"SELECT if(notEquals(x, '{}'), x, '{}') FROM t WHERE team_id = {team_id:Int64}",
),
(
- "select * from events where event = {event}",
+ "select * from events where event = '{event}'",
{"event": "hello"},
- "select * from events where event = 'hello'",
+ "select * from events where event = '{event}'",
),
(
"select * from events where event = %(event)s",
@@ -256,37 +273,35 @@ def test_post_query_sends_freshly_read_token(tmp_path):
"select * from events where event = 'world'",
),
(
- "select * from events where event = %(event)s and event != {another}",
+ "select * from events where event = %(event)s and event != %(another)s",
{"event": "index_{1}", "another": "event"},
"select * from events where event = 'index_{1}' and event != 'event'",
),
(
- "select * from events where event = %(event)s and event != {another}",
+ "select * from events where event = %(event)s and event != %(another)s",
{"event": "index_{something}", "another": "event"},
"select * from events where event = 'index_{something}' and event != 'event'",
),
# a value containing an unbalanced brace used to crash the format pass
(
- "select * from events where event = %(event)s and event != {another}",
+ "select * from events where event = %(event)s and event != %(another)s",
{"event": "brace {", "another": "event"},
"select * from events where event = 'brace {' and event != 'event'",
),
# a JSON string value used to be mangled by the format pass
(
- "select * from events where properties = %(props)s and event != {another}",
+ "select * from events where properties = %(props)s and event != %(another)s",
{"props": '{"a": 1}', "another": "event"},
"select * from events where properties = '{\"a\": 1}' and event != 'event'",
),
# a value matching another parameter's name must not be substituted again
(
- "select * from events where event = %(event)s and event != {another}",
+ "select * from events where event = %(event)s and event != %(another)s",
{"event": "literal {another}", "another": "event"},
"select * from events where event = 'literal {another}' and event != 'event'",
),
- # the INSERT INTO FUNCTION s3(...) shape used by batch exports: intentional escapes
- # collapse while a value containing '%' and braces survives verbatim
(
- "INSERT INTO FUNCTION s3('bucket/export_{{_partition_id}}.arrow') SELECT %(v)s SETTINGS log_comment={log_comment}",
+ "INSERT INTO FUNCTION s3('bucket/export_{_partition_id}.arrow') SELECT %(v)s SETTINGS log_comment=%(log_comment)s",
{"v": '100%-{"a": 1}', "log_comment": "comment"},
"INSERT INTO FUNCTION s3('bucket/export_{_partition_id}.arrow') SELECT '100%-{\"a\": 1}' SETTINGS log_comment='comment'",
),
diff --git a/products/batch_exports/backend/temporal/pipeline/internal_stage.py b/products/batch_exports/backend/temporal/pipeline/internal_stage.py
index 651abb2cf84b..b1a2e0b966c7 100644
--- a/products/batch_exports/backend/temporal/pipeline/internal_stage.py
+++ b/products/batch_exports/backend/temporal/pipeline/internal_stage.py
@@ -572,10 +572,10 @@ async def _get_query(
filter_distinct_ids = """
HAVING
(
- _timestamp >= {interval_start}::DateTime64
+ _timestamp >= %(interval_start)s::DateTime64
)
AND (
- _timestamp < {interval_end}::DateTime64
+ _timestamp < %(interval_end)s::DateTime64
)
"""
else:
diff --git a/products/batch_exports/backend/temporal/record_batch_model.py b/products/batch_exports/backend/temporal/record_batch_model.py
index e4a357f4025f..68a81deee377 100644
--- a/products/batch_exports/backend/temporal/record_batch_model.py
+++ b/products/batch_exports/backend/temporal/record_batch_model.py
@@ -92,7 +92,7 @@ class RecordBatchModel(abc.ABC):
# A bit of a hack: the query references neither half of this, but both are required:
# - we need to call `get_log_comment` in order to tag the queries
# - we need to pass non-empty `values` to `ClickHouseClient.prepare_query` to avoid it returning
- # early and not resolving the `{{_partition_id}}` and `%%` escapes in the s3 function
+ # early and not resolving the `%%` escapes in the s3 function
# call.
values={"log_comment": self.get_log_comment()},
)
@@ -338,7 +338,7 @@ class SessionsRecordBatchModel(RecordBatchModel):
stack=[],
)
- query_settings = "max_execution_time={max_execution_time_seconds}, log_comment={log_comment}"
+ query_settings = "max_execution_time=%(max_execution_time_seconds)s, log_comment=%(log_comment)s"
if "settings" not in printed.lower():
printed += f" SETTINGS {query_settings}"
else:
diff --git a/products/batch_exports/backend/temporal/sql/common.py b/products/batch_exports/backend/temporal/sql/common.py
index 8498ac80f4d3..5ed9857c72a1 100644
--- a/products/batch_exports/backend/temporal/sql/common.py
+++ b/products/batch_exports/backend/temporal/sql/common.py
@@ -16,7 +16,7 @@ def get_s3_function_call(s3_folder: str, credentials: AWSKeyPair | None, num_par
Note: We use %% for the modulo operator because the ClickHouse client uses % for
parameter substitution, so %% produces a literal % in the final query.
"""
- s3_url = f"{s3_folder}/export_{{{{_partition_id}}}}.arrow"
+ s3_url = f"{s3_folder}/export_{{_partition_id}}.arrow"
if credentials is not None:
# Escape single quotes by doubling them (ClickHouse SQL escaping)
escaped_key = credentials.access_key_id.replace("'", "''")
diff --git a/products/batch_exports/backend/temporal/sql/events.py b/products/batch_exports/backend/temporal/sql/events.py
index 60230b6de0b8..c21ef9a07710 100644
--- a/products/batch_exports/backend/temporal/sql/events.py
+++ b/products/batch_exports/backend/temporal/sql/events.py
@@ -45,14 +45,14 @@ FROM (
FROM
events
PREWHERE
- COALESCE(events.inserted_at, events._timestamp) >= {{interval_start:DateTime64}}
- AND COALESCE(events.inserted_at, events._timestamp) < {{interval_end:DateTime64}}
+ COALESCE(events.inserted_at, events._timestamp) >= {interval_start:DateTime64}
+ AND COALESCE(events.inserted_at, events._timestamp) < {interval_end:DateTime64}
WHERE
- team_id = {{team_id:Int64}}
- AND events.timestamp >= {{interval_start:DateTime64}} - INTERVAL {{lookback_days:Int32}} DAY
- AND events.timestamp < {{interval_end:DateTime64}} + INTERVAL 1 DAY
- AND (length({{include_events:Array(String)}}) = 0 OR event IN {{include_events:Array(String)}})
- AND (length({{exclude_events:Array(String)}}) = 0 OR event NOT IN {{exclude_events:Array(String)}})
+ team_id = {team_id:Int64}
+ AND events.timestamp >= {interval_start:DateTime64} - INTERVAL {lookback_days:Int32} DAY
+ AND events.timestamp < {interval_end:DateTime64} + INTERVAL 1 DAY
+ AND (length({include_events:Array(String)}) = 0 OR event IN {include_events:Array(String)})
+ AND (length({exclude_events:Array(String)}) = 0 OR event NOT IN {exclude_events:Array(String)})
$filters
$order
) AS events
@@ -88,21 +88,21 @@ FROM (
FROM
events
PREWHERE
- COALESCE(events.inserted_at, events._timestamp) >= {{interval_start:DateTime64}}
- AND COALESCE(events.inserted_at, events._timestamp) < {{interval_end:DateTime64}}
+ COALESCE(events.inserted_at, events._timestamp) >= {interval_start:DateTime64}
+ AND COALESCE(events.inserted_at, events._timestamp) < {interval_end:DateTime64}
WHERE
- team_id = {{team_id:Int64}}
- AND events.timestamp >= {{interval_start:DateTime64}} - INTERVAL {{lookback_days:Int32}} DAY
- AND events.timestamp < {{interval_end:DateTime64}} + INTERVAL 1 DAY
- AND (length({{include_events:Array(String)}}) = 0 OR event IN {{include_events:Array(String)}})
- AND (length({{exclude_events:Array(String)}}) = 0 OR event NOT IN {{exclude_events:Array(String)}})
+ team_id = {team_id:Int64}
+ AND events.timestamp >= {interval_start:DateTime64} - INTERVAL {lookback_days:Int32} DAY
+ AND events.timestamp < {interval_end:DateTime64} + INTERVAL 1 DAY
+ AND (length({include_events:Array(String)}) = 0 OR event IN {include_events:Array(String)})
+ AND (length({exclude_events:Array(String)}) = 0 OR event NOT IN {exclude_events:Array(String)})
$filters
) AS events
SETTINGS
-- This is half of configured MAX_MEMORY_USAGE for batch exports.
max_bytes_before_external_sort=50000000000,
optimize_aggregation_in_order=1,
- log_comment={log_comment}
+ log_comment=%(log_comment)s
"""
)
@@ -113,12 +113,12 @@ SELECT
$fields
FROM events
PREWHERE
- COALESCE(events.timestamp) >= {{interval_start:DateTime64}}
- AND COALESCE(events.timestamp) < {{interval_end:DateTime64}}
+ COALESCE(events.timestamp) >= {interval_start:DateTime64}
+ AND COALESCE(events.timestamp) < {interval_end:DateTime64}
WHERE
- team_id = {{team_id:Int64}}
- AND (length({{include_events:Array(String)}}) = 0 OR event IN {{include_events:Array(String)}})
- AND (length({{exclude_events:Array(String)}}) = 0 OR event NOT IN {{exclude_events:Array(String)}})
+ team_id = {team_id:Int64}
+ AND (length({include_events:Array(String)}) = 0 OR event IN {include_events:Array(String)})
+ AND (length({exclude_events:Array(String)}) = 0 OR event NOT IN {exclude_events:Array(String)})
$filters
FORMAT ArrowStream
SETTINGS
@@ -137,17 +137,17 @@ SELECT
FROM
events
WHERE
- team_id = {{team_id:Int64}}
- AND events.timestamp >= {{interval_start:DateTime64}}
- AND events.timestamp < {{interval_end:DateTime64}}
- AND (length({{include_events:Array(String)}}) = 0 OR event IN {{include_events:Array(String)}})
- AND (length({{exclude_events:Array(String)}}) = 0 OR event NOT IN {{exclude_events:Array(String)}})
+ team_id = {team_id:Int64}
+ AND events.timestamp >= {interval_start:DateTime64}
+ AND events.timestamp < {interval_end:DateTime64}
+ AND (length({include_events:Array(String)}) = 0 OR event IN {include_events:Array(String)})
+ AND (length({exclude_events:Array(String)}) = 0 OR event NOT IN {exclude_events:Array(String)})
$filters
SETTINGS
-- This is half of configured MAX_MEMORY_USAGE for batch exports.
max_bytes_before_external_sort=50000000000,
optimize_aggregation_in_order=1,
- log_comment={log_comment}
+ log_comment=%(log_comment)s
"""
)
@@ -174,12 +174,12 @@ FROM (
FROM
events_recent AS events
PREWHERE
- events_recent.inserted_at >= {{interval_start:DateTime64}}
- AND events_recent.inserted_at < {{interval_end:DateTime64}}
+ events_recent.inserted_at >= {interval_start:DateTime64}
+ AND events_recent.inserted_at < {interval_end:DateTime64}
WHERE
- team_id = {{team_id:Int64}}
- AND (length({{include_events:Array(String)}}) = 0 OR event IN {{include_events:Array(String)}})
- AND (length({{exclude_events:Array(String)}}) = 0 OR event NOT IN {{exclude_events:Array(String)}})
+ team_id = {team_id:Int64}
+ AND (length({include_events:Array(String)}) = 0 OR event IN {include_events:Array(String)})
+ AND (length({exclude_events:Array(String)}) = 0 OR event NOT IN {exclude_events:Array(String)})
$filters
$order
) AS events
@@ -216,12 +216,12 @@ FROM (
FROM
events_recent AS events
PREWHERE
- events_recent.inserted_at >= {{interval_start:DateTime64}}
- AND events_recent.inserted_at < {{interval_end:DateTime64}}
+ events_recent.inserted_at >= {interval_start:DateTime64}
+ AND events_recent.inserted_at < {interval_end:DateTime64}
WHERE
- team_id = {{team_id:Int64}}
- AND (length({{include_events:Array(String)}}) = 0 OR event IN {{include_events:Array(String)}})
- AND (length({{exclude_events:Array(String)}}) = 0 OR event NOT IN {{exclude_events:Array(String)}})
+ team_id = {team_id:Int64}
+ AND (length({include_events:Array(String)}) = 0 OR event IN {include_events:Array(String)})
+ AND (length({exclude_events:Array(String)}) = 0 OR event NOT IN {exclude_events:Array(String)})
$filters
) AS events
SETTINGS
@@ -229,7 +229,7 @@ SETTINGS
max_bytes_before_external_sort=50000000000,
max_replica_delay_for_distributed_queries=1,
optimize_aggregation_in_order=1,
- log_comment={log_comment}
+ log_comment=%(log_comment)s
"""
)
@@ -256,12 +256,12 @@ FROM (
FROM
distributed_events_recent AS events
PREWHERE
- distributed_events_recent.inserted_at >= {interval_start}::DateTime64
- AND distributed_events_recent.inserted_at < {interval_end}::DateTime64
+ distributed_events_recent.inserted_at >= %(interval_start)s::DateTime64
+ AND distributed_events_recent.inserted_at < %(interval_end)s::DateTime64
WHERE
- team_id = {team_id}::Int64
- AND (length({include_events}::Array(String)) = 0 OR event IN {include_events}::Array(String))
- AND (length({exclude_events}::Array(String)) = 0 OR event NOT IN {exclude_events}::Array(String))
+ team_id = %(team_id)s::Int64
+ AND (length(%(include_events)s::Array(String)) = 0 OR event IN %(include_events)s::Array(String))
+ AND (length(%(exclude_events)s::Array(String)) = 0 OR event NOT IN %(exclude_events)s::Array(String))
$filters
$order
) AS events
@@ -299,12 +299,12 @@ FROM (
FROM
distributed_events_recent AS events
PREWHERE
- distributed_events_recent.inserted_at >= {interval_start}::DateTime64
- AND distributed_events_recent.inserted_at < {interval_end}::DateTime64
+ distributed_events_recent.inserted_at >= %(interval_start)s::DateTime64
+ AND distributed_events_recent.inserted_at < %(interval_end)s::DateTime64
WHERE
- team_id = {team_id}::Int64
- AND (length({include_events}::Array(String)) = 0 OR event IN {include_events}::Array(String))
- AND (length({exclude_events}::Array(String)) = 0 OR event NOT IN {exclude_events}::Array(String))
+ team_id = %(team_id)s::Int64
+ AND (length(%(include_events)s::Array(String)) = 0 OR event IN %(include_events)s::Array(String))
+ AND (length(%(exclude_events)s::Array(String)) = 0 OR event NOT IN %(exclude_events)s::Array(String))
$filters
) AS events
SETTINGS
@@ -313,7 +313,7 @@ SETTINGS
max_replica_delay_for_distributed_queries=60,
fallback_to_stale_replicas_for_distributed_queries=0,
optimize_aggregation_in_order=1,
- log_comment={log_comment}
+ log_comment=%(log_comment)s
"""
)
@@ -340,12 +340,12 @@ FROM (
FROM
events
PREWHERE
- COALESCE(events.inserted_at, events._timestamp) >= {{interval_start:DateTime64}}
- AND COALESCE(events.inserted_at, events._timestamp) < {{interval_end:DateTime64}}
+ COALESCE(events.inserted_at, events._timestamp) >= {interval_start:DateTime64}
+ AND COALESCE(events.inserted_at, events._timestamp) < {interval_end:DateTime64}
WHERE
- team_id = {{team_id:Int64}}
- AND (length({{include_events:Array(String)}}) = 0 OR event IN {{include_events:Array(String)}})
- AND (length({{exclude_events:Array(String)}}) = 0 OR event NOT IN {{exclude_events:Array(String)}})
+ team_id = {team_id:Int64}
+ AND (length({include_events:Array(String)}) = 0 OR event IN {include_events:Array(String)})
+ AND (length({exclude_events:Array(String)}) = 0 OR event NOT IN {exclude_events:Array(String)})
$filters
$order
) AS events
@@ -381,19 +381,19 @@ FROM (
FROM
events
PREWHERE
- COALESCE(events.inserted_at, events._timestamp) >= {{interval_start:DateTime64}}
- AND COALESCE(events.inserted_at, events._timestamp) < {{interval_end:DateTime64}}
+ COALESCE(events.inserted_at, events._timestamp) >= {interval_start:DateTime64}
+ AND COALESCE(events.inserted_at, events._timestamp) < {interval_end:DateTime64}
WHERE
- team_id = {{team_id:Int64}}
- AND (length({{include_events:Array(String)}}) = 0 OR event IN {{include_events:Array(String)}})
- AND (length({{exclude_events:Array(String)}}) = 0 OR event NOT IN {{exclude_events:Array(String)}})
+ team_id = {team_id:Int64}
+ AND (length({include_events:Array(String)}) = 0 OR event IN {include_events:Array(String)})
+ AND (length({exclude_events:Array(String)}) = 0 OR event NOT IN {exclude_events:Array(String)})
$filters
) AS events
SETTINGS
-- This is half of configured MAX_MEMORY_USAGE for batch exports.
max_bytes_before_external_sort=50000000000,
optimize_aggregation_in_order=1,
- log_comment={log_comment}
+ log_comment=%(log_comment)s
"""
)
@@ -420,11 +420,11 @@ FROM (
FROM
events
WHERE
- team_id = {{team_id:Int64}}
- AND events.timestamp >= {{interval_start:DateTime64}}
- AND events.timestamp < {{interval_end:DateTime64}}
- AND (length({{include_events:Array(String)}}) = 0 OR event IN {{include_events:Array(String)}})
- AND (length({{exclude_events:Array(String)}}) = 0 OR event NOT IN {{exclude_events:Array(String)}})
+ team_id = {team_id:Int64}
+ AND events.timestamp >= {interval_start:DateTime64}
+ AND events.timestamp < {interval_end:DateTime64}
+ AND (length({include_events:Array(String)}) = 0 OR event IN {include_events:Array(String)})
+ AND (length({exclude_events:Array(String)}) = 0 OR event NOT IN {exclude_events:Array(String)})
$filters
$order
) as events
@@ -460,26 +460,26 @@ FROM (
FROM
events
WHERE
- team_id = {{team_id:Int64}}
- AND events.timestamp >= {{interval_start:DateTime64}}
- AND events.timestamp < {{interval_end:DateTime64}}
- AND (length({{include_events:Array(String)}}) = 0 OR event IN {{include_events:Array(String)}})
- AND (length({{exclude_events:Array(String)}}) = 0 OR event NOT IN {{exclude_events:Array(String)}})
+ team_id = {team_id:Int64}
+ AND events.timestamp >= {interval_start:DateTime64}
+ AND events.timestamp < {interval_end:DateTime64}
+ AND (length({include_events:Array(String)}) = 0 OR event IN {include_events:Array(String)})
+ AND (length({exclude_events:Array(String)}) = 0 OR event NOT IN {exclude_events:Array(String)})
$filters
) as events
SETTINGS
-- This is half of configured MAX_MEMORY_USAGE for batch exports.
max_bytes_before_external_sort=50000000000,
optimize_aggregation_in_order=1,
- log_comment={log_comment}
+ log_comment=%(log_comment)s
"""
)
EVENT_COUNT_BY_INTERVAL = """
SELECT
- toStartOfInterval(_inserted_at, INTERVAL {interval}) AS interval_start,
- interval_start + INTERVAL {interval} AS interval_end,
+ toStartOfInterval(_inserted_at, INTERVAL %(interval)s) AS interval_start,
+ interval_start + INTERVAL %(interval)s AS interval_end,
COUNT(*) as total_count
FROM (
SELECT DISTINCT ON (team_id, event, cityHash64(events_recent.distinct_id), cityHash64(events_recent.uuid))
@@ -487,12 +487,12 @@ FROM (
FROM
events_recent
PREWHERE
- events_recent.inserted_at >= {overall_interval_start}::DateTime64
- AND events_recent.inserted_at < {overall_interval_end}::DateTime64
+ events_recent.inserted_at >= %(overall_interval_start)s::DateTime64
+ AND events_recent.inserted_at < %(overall_interval_end)s::DateTime64
WHERE
- team_id = {team_id}::Int64
- AND (length({include_events}::Array(String)) = 0 OR event IN {include_events}::Array(String))
- AND (length({exclude_events}::Array(String)) = 0 OR event NOT IN {exclude_events}::Array(String))
+ team_id = %(team_id)s::Int64
+ AND (length(%(include_events)s::Array(String)) = 0 OR event IN %(include_events)s::Array(String))
+ AND (length(%(exclude_events)s::Array(String)) = 0 OR event NOT IN %(exclude_events)s::Array(String))
)
GROUP BY interval_start
ORDER BY interval_start ASC
@@ -502,18 +502,10 @@ SETTINGS
"""
-# The `{{}}` literals below are not a typo. `ClickHouseClient.prepare_query` runs `str.format` over
-# the whole query, which turns `{{}}` back into `{}`. A bare `{}` would parse as a positional field
-# and reach ClickHouse as `{0}`, so the `nullIf` would never match an empty object. The rebuilt document
-# gets the same doubling for the same reason.
# `toJSONString` prints a stored dotted key as `a%2Eb` unless the query sets `json_type_escape_dots_in_keys`,
# which is why the native query's SETTINGS carry it.
-_PROPERTIES_DOCUMENT = (
- event_document_sql("properties", "temporary_properties", "properties.`$feature_flags`")
- .replace("{", "{{")
- .replace("}", "}}")
-)
-_PERSON_PROPERTIES_DOCUMENT = person_document_sql("person_properties").replace("{", "{{").replace("}", "}}")
+_PROPERTIES_DOCUMENT = event_document_sql("properties", "temporary_properties", "properties.`$feature_flags`")
+_PERSON_PROPERTIES_DOCUMENT = person_document_sql("person_properties")
SERIALIZED_EVENTS_JSON_SOURCE = """(
SELECT * REPLACE (
toString(uuid) AS uuid,
@@ -521,10 +513,10 @@ SERIALIZED_EVENTS_JSON_SOURCE = """(
__PROPERTIES_DOCUMENT__ AS properties,
__PERSON_PROPERTIES_DOCUMENT__ AS person_properties
),
- nullIf(toJSONString(temporary_properties.^`$set`), '{{}}') AS set,
- nullIf(toJSONString(temporary_properties.^`$set_once`), '{{}}') AS set_once,
+ nullIf(toJSONString(temporary_properties.^`$set`), '{}') AS set,
+ nullIf(toJSONString(temporary_properties.^`$set_once`), '{}') AS set_once,
nullIf(toJSONString(temporary_properties.^`$unset`), '[]') AS unset,
- nullIf(toJSONString(temporary_properties.^`$group_set`), '{{}}') AS group_set
+ nullIf(toJSONString(temporary_properties.^`$group_set`), '{}') AS group_set
FROM events_json
)""".replace("__PROPERTIES_DOCUMENT__", _PROPERTIES_DOCUMENT).replace(
"__PERSON_PROPERTIES_DOCUMENT__", _PERSON_PROPERTIES_DOCUMENT
@@ -545,11 +537,11 @@ FROM (
SELECT DISTINCT ON (team_id, event, cityHash64(events.distinct_id), cityHash64(events.uuid))
*, timestamp AS _inserted_at
FROM {SERIALIZED_EVENTS_JSON_SOURCE} AS events
- WHERE team_id = {{{{team_id:Int64}}}}
- AND ({{interval_start}}::Nullable(DateTime64) IS NULL OR timestamp >= {{interval_start}}::Nullable(DateTime64))
- AND timestamp < {{{{interval_end:DateTime64}}}}
- AND (length({{{{include_events:Array(String)}}}}) = 0 OR event IN {{{{include_events:Array(String)}}}})
- AND (length({{{{exclude_events:Array(String)}}}}) = 0 OR event NOT IN {{{{exclude_events:Array(String)}}}})
+ WHERE team_id = {{team_id:Int64}}
+ AND (%(interval_start)s::Nullable(DateTime64) IS NULL OR timestamp >= %(interval_start)s::Nullable(DateTime64))
+ AND timestamp < {{interval_end:DateTime64}}
+ AND (length({{include_events:Array(String)}}) = 0 OR event IN {{include_events:Array(String)}})
+ AND (length({{exclude_events:Array(String)}}) = 0 OR event NOT IN {{exclude_events:Array(String)}})
{"AND " + filters if filters else ""}
{order}
) AS events
@@ -561,5 +553,5 @@ SETTINGS
optimize_aggregation_in_order=1,
json_type_escape_dots_in_keys=1,
output_format_json_escape_forward_slashes=0
-{", log_comment={log_comment}" if s3_function else ""}
+{", log_comment=%(log_comment)s" if s3_function else ""}
"""
diff --git a/products/batch_exports/backend/temporal/sql/persons.py b/products/batch_exports/backend/temporal/sql/persons.py
index 15db3d6c385f..9f0eba17ca6b 100644
--- a/products/batch_exports/backend/temporal/sql/persons.py
+++ b/products/batch_exports/backend/temporal/sql/persons.py
@@ -20,22 +20,22 @@ FROM (
FROM
person
WHERE
- team_id = {team_id}::Int64
+ team_id = %(team_id)s::Int64
AND id in (
SELECT
id
FROM
person
WHERE
- team_id = {team_id}::Int64
- AND _timestamp >= {interval_start}::DateTime64
- AND _timestamp < {interval_end}::DateTime64
+ team_id = %(team_id)s::Int64
+ AND _timestamp >= %(interval_start)s::DateTime64
+ AND _timestamp < %(interval_end)s::DateTime64
)
GROUP BY
id
HAVING
- argMax(_timestamp, person.version) >= {interval_start}::DateTime64
- AND argMax(_timestamp, person.version) < {interval_end}::DateTime64
+ argMax(_timestamp, person.version) >= %(interval_start)s::DateTime64
+ AND argMax(_timestamp, person.version) < %(interval_end)s::DateTime64
),
distinct_ids AS (
-- new distinct_ids
@@ -44,22 +44,22 @@ FROM (
FROM
person_distinct_id2
WHERE
- team_id = {team_id}::Int64
+ team_id = %(team_id)s::Int64
AND distinct_id in (
SELECT
distinct_id
FROM
person_distinct_id2
WHERE
- team_id = {team_id}::Int64
- AND _timestamp >= {interval_start}::DateTime64
- AND _timestamp < {interval_end}::DateTime64
+ team_id = %(team_id)s::Int64
+ AND _timestamp >= %(interval_start)s::DateTime64
+ AND _timestamp < %(interval_end)s::DateTime64
)
GROUP BY
distinct_id
HAVING
- argMax(_timestamp, person_distinct_id2.version) >= {interval_start}::DateTime64
- AND argMax(_timestamp, person_distinct_id2.version) < {interval_end}::DateTime64
+ argMax(_timestamp, person_distinct_id2.version) >= %(interval_start)s::DateTime64
+ AND argMax(_timestamp, person_distinct_id2.version) < %(interval_end)s::DateTime64
UNION DISTINCT
@@ -69,7 +69,7 @@ FROM (
FROM
person_distinct_id2
WHERE
- team_id = {team_id}::Int64
+ team_id = %(team_id)s::Int64
AND person_id IN new_persons
),
latest_distinct_ids AS (
@@ -81,7 +81,7 @@ FROM (
FROM
person_distinct_id2
WHERE
- team_id = {team_id}::Int64
+ team_id = %(team_id)s::Int64
AND distinct_id IN distinct_ids
GROUP BY
distinct_id
@@ -94,7 +94,7 @@ FROM (
FROM
person
WHERE
- team_id = {team_id}::Int64
+ team_id = %(team_id)s::Int64
AND
id IN (
SELECT
@@ -116,21 +116,21 @@ FROM (
toBool(p.is_deleted) AS is_deleted,
multiIf(
(
- pd._timestamp >= {interval_start}::DateTime64
- AND pd._timestamp < {interval_end}::DateTime64
+ pd._timestamp >= %(interval_start)s::DateTime64
+ AND pd._timestamp < %(interval_end)s::DateTime64
)
AND NOT (
- p._timestamp >= {interval_start}::DateTime64
- AND p._timestamp < {interval_end}::DateTime64
+ p._timestamp >= %(interval_start)s::DateTime64
+ AND p._timestamp < %(interval_end)s::DateTime64
),
pd._timestamp,
(
- p._timestamp >= {interval_start}::DateTime64
- AND p._timestamp < {interval_end}::DateTime64
+ p._timestamp >= %(interval_start)s::DateTime64
+ AND p._timestamp < %(interval_end)s::DateTime64
)
AND NOT (
- pd._timestamp >= {interval_start}::DateTime64
- AND pd._timestamp < {interval_end}::DateTime64
+ pd._timestamp >= %(interval_start)s::DateTime64
+ AND pd._timestamp < %(interval_end)s::DateTime64
),
p._timestamp,
least(p._timestamp, pd._timestamp)
@@ -140,7 +140,7 @@ FROM (
INNER JOIN latest_distinct_ids pd
ON p.id = pd.person_id2
WHERE
- p.team_id = {team_id}::Int64
+ p.team_id = %(team_id)s::Int64
AND (p.id, p.version, p._timestamp) in latest_person_versions
ORDER BY
_inserted_at
@@ -172,22 +172,22 @@ FROM (
FROM
person
WHERE
- team_id = {team_id}::Int64
+ team_id = %(team_id)s::Int64
AND id in (
SELECT
id
FROM
person
WHERE
- team_id = {team_id}::Int64
- AND _timestamp >= {interval_start}::DateTime64
- AND _timestamp < {interval_end}::DateTime64
+ team_id = %(team_id)s::Int64
+ AND _timestamp >= %(interval_start)s::DateTime64
+ AND _timestamp < %(interval_end)s::DateTime64
)
GROUP BY
id
HAVING
- argMax(_timestamp, person.version) >= {interval_start}::DateTime64
- AND argMax(_timestamp, person.version) < {interval_end}::DateTime64
+ argMax(_timestamp, person.version) >= %(interval_start)s::DateTime64
+ AND argMax(_timestamp, person.version) < %(interval_end)s::DateTime64
),
distinct_ids AS (
-- new distinct_ids
@@ -196,22 +196,22 @@ FROM (
FROM
person_distinct_id2
WHERE
- team_id = {team_id}::Int64
+ team_id = %(team_id)s::Int64
AND distinct_id in (
SELECT
distinct_id
FROM
person_distinct_id2
WHERE
- team_id = {team_id}::Int64
- AND _timestamp >= {interval_start}::DateTime64
- AND _timestamp < {interval_end}::DateTime64
+ team_id = %(team_id)s::Int64
+ AND _timestamp >= %(interval_start)s::DateTime64
+ AND _timestamp < %(interval_end)s::DateTime64
)
GROUP BY
distinct_id
HAVING
- argMax(_timestamp, person_distinct_id2.version) >= {interval_start}::DateTime64
- AND argMax(_timestamp, person_distinct_id2.version) < {interval_end}::DateTime64
+ argMax(_timestamp, person_distinct_id2.version) >= %(interval_start)s::DateTime64
+ AND argMax(_timestamp, person_distinct_id2.version) < %(interval_end)s::DateTime64
UNION DISTINCT
@@ -221,7 +221,7 @@ FROM (
FROM
person_distinct_id2
WHERE
- team_id = {team_id}::Int64
+ team_id = %(team_id)s::Int64
AND person_id IN new_persons
),
latest_distinct_ids AS (
@@ -233,7 +233,7 @@ FROM (
FROM
person_distinct_id2
WHERE
- team_id = {team_id}::Int64
+ team_id = %(team_id)s::Int64
AND distinct_id IN distinct_ids
GROUP BY
distinct_id
@@ -247,7 +247,7 @@ FROM (
FROM
person
WHERE
- team_id = {team_id}::Int64
+ team_id = %(team_id)s::Int64
AND
id IN (
SELECT
@@ -269,21 +269,21 @@ FROM (
toBool(p.is_deleted) AS is_deleted,
multiIf(
(
- pd._timestamp >= {interval_start}::DateTime64
- AND pd._timestamp < {interval_end}::DateTime64
+ pd._timestamp >= %(interval_start)s::DateTime64
+ AND pd._timestamp < %(interval_end)s::DateTime64
)
AND NOT (
- p._timestamp >= {interval_start}::DateTime64
- AND p._timestamp < {interval_end}::DateTime64
+ p._timestamp >= %(interval_start)s::DateTime64
+ AND p._timestamp < %(interval_end)s::DateTime64
),
pd._timestamp,
(
- p._timestamp >= {interval_start}::DateTime64
- AND p._timestamp < {interval_end}::DateTime64
+ p._timestamp >= %(interval_start)s::DateTime64
+ AND p._timestamp < %(interval_end)s::DateTime64
)
AND NOT (
- pd._timestamp >= {interval_start}::DateTime64
- AND pd._timestamp < {interval_end}::DateTime64
+ pd._timestamp >= %(interval_start)s::DateTime64
+ AND pd._timestamp < %(interval_end)s::DateTime64
),
p._timestamp,
least(p._timestamp, pd._timestamp)
@@ -293,14 +293,14 @@ FROM (
INNER JOIN latest_distinct_ids pd
ON p.id = pd.person_id2
WHERE
- p.team_id = {team_id}::Int64
+ p.team_id = %(team_id)s::Int64
AND (p.id, p.version, p._timestamp) in latest_person_versions
) AS persons
SETTINGS
max_bytes_before_external_group_by=50000000000,
max_bytes_before_external_sort=50000000000,
optimize_aggregation_in_order=1,
- log_comment={log_comment}
+ log_comment=%(log_comment)s
""")
@@ -315,11 +315,11 @@ SELECT
p.created_at AS created_at,
toBool(p.is_deleted) AS is_deleted,
multiIf(
- pd._timestamp < {interval_end}::DateTime64
- AND NOT p._timestamp < {interval_end}::DateTime64,
+ pd._timestamp < %(interval_end)s::DateTime64
+ AND NOT p._timestamp < %(interval_end)s::DateTime64,
pd._timestamp,
- p._timestamp < {interval_end}::DateTime64
- AND NOT pd._timestamp < {interval_end}::DateTime64,
+ p._timestamp < %(interval_end)s::DateTime64
+ AND NOT pd._timestamp < %(interval_end)s::DateTime64,
p._timestamp,
least(p._timestamp, pd._timestamp)
) AS _inserted_at
@@ -333,7 +333,7 @@ FROM (
FROM
person_distinct_id2
PREWHERE
- team_id = {team_id}::Int64
+ team_id = %(team_id)s::Int64
GROUP BY
team_id,
distinct_id
@@ -350,17 +350,17 @@ INNER JOIN (
FROM
person
PREWHERE
- team_id = {team_id}::Int64
+ team_id = %(team_id)s::Int64
GROUP BY
team_id,
id
) AS p ON p.id = pd.person_id AND p.team_id = pd.team_id
WHERE
- pd.team_id = {team_id}::Int64
- AND p.team_id = {team_id}::Int64
+ pd.team_id = %(team_id)s::Int64
+ AND p.team_id = %(team_id)s::Int64
AND (
- pd._timestamp < {interval_end}::DateTime64
- OR p._timestamp < {interval_end}::DateTime64
+ pd._timestamp < %(interval_end)s::DateTime64
+ OR p._timestamp < %(interval_end)s::DateTime64
)
ORDER BY
_inserted_at
@@ -384,11 +384,11 @@ SELECT
p.created_at AS created_at,
toBool(p.is_deleted) AS is_deleted,
multiIf(
- pd._timestamp < {interval_end}::DateTime64
- AND NOT p._timestamp < {interval_end}::DateTime64,
+ pd._timestamp < %(interval_end)s::DateTime64
+ AND NOT p._timestamp < %(interval_end)s::DateTime64,
pd._timestamp,
- p._timestamp < {interval_end}::DateTime64
- AND NOT pd._timestamp < {interval_end}::DateTime64,
+ p._timestamp < %(interval_end)s::DateTime64
+ AND NOT pd._timestamp < %(interval_end)s::DateTime64,
p._timestamp,
least(p._timestamp, pd._timestamp)
) AS _inserted_at
@@ -402,7 +402,7 @@ FROM (
FROM
person_distinct_id2
PREWHERE
- team_id = {team_id}::Int64
+ team_id = %(team_id)s::Int64
GROUP BY
team_id,
distinct_id
@@ -419,21 +419,21 @@ INNER JOIN (
FROM
person
PREWHERE
- team_id = {team_id}::Int64
+ team_id = %(team_id)s::Int64
GROUP BY
team_id,
id
) AS p ON p.id = pd.person_id AND p.team_id = pd.team_id
WHERE
- pd.team_id = {team_id}::Int64
- AND p.team_id = {team_id}::Int64
+ pd.team_id = %(team_id)s::Int64
+ AND p.team_id = %(team_id)s::Int64
AND (
- pd._timestamp < {interval_end}::DateTime64
- OR p._timestamp < {interval_end}::DateTime64
+ pd._timestamp < %(interval_end)s::DateTime64
+ OR p._timestamp < %(interval_end)s::DateTime64
)
SETTINGS
max_bytes_before_external_group_by=50000000000,
max_bytes_before_external_sort=50000000000,
optimize_aggregation_in_order=1,
- log_comment={log_comment}
+ log_comment=%(log_comment)s
""")
diff --git a/products/batch_exports/backend/tests/temporal/destinations/s3/utils.py b/products/batch_exports/backend/tests/temporal/destinations/s3/utils.py
index f97a92696186..4c403de99f4b 100644
--- a/products/batch_exports/backend/tests/temporal/destinations/s3/utils.py
+++ b/products/batch_exports/backend/tests/temporal/destinations/s3/utils.py
@@ -170,7 +170,7 @@ async def assert_metrics_in_clickhouse(
SELECT metric_kind, metric_name, count
FROM sharded_app_metrics2
WHERE app_source = 'batch_export'
- AND app_source_id = {{batch_export_id:String}}
+ AND app_source_id = {batch_export_id:String}
FORMAT JSONEachRow \
"""
diff --git a/products/batch_exports/backend/tests/temporal/pipeline/test_internal_stage.py b/products/batch_exports/backend/tests/temporal/pipeline/test_internal_stage.py
index 4e914cca6daa..f2472fe51ef3 100644
--- a/products/batch_exports/backend/tests/temporal/pipeline/test_internal_stage.py
+++ b/products/batch_exports/backend/tests/temporal/pipeline/test_internal_stage.py
@@ -1410,6 +1410,18 @@ class TestHogQLModel:
lambda events, persons: [(e["event"], e["distinct_id"], "Chrome") for e in events],
id="events-with-special-chars",
),
+ pytest.param(
+ r"""
+ SELECT 'quote\' backtick` slash\\ {log_comment} %(other)s' AS `{hogql_val_0}`,
+ event AS `{log_comment}`, distinct_id AS `{{hogql_val_0}}`
+ FROM events
+ """,
+ ["{hogql_val_0}", "{log_comment}", "{{hogql_val_0}}"],
+ lambda events, persons: [
+ ("quote' backtick` slash\\ {log_comment} %(other)s", e["event"], e["distinct_id"]) for e in events
+ ],
+ id="formatter-shaped-aliases",
+ ),
pytest.param(
"SELECT toString(id) AS id, properties.utm_medium AS utm_medium FROM persons",
["id", "utm_medium"],
diff --git a/products/batch_exports/backend/tests/temporal/test_record_batch_model.py b/products/batch_exports/backend/tests/temporal/test_record_batch_model.py
index 4782cc6374c9..d06cd547c1c1 100644
--- a/products/batch_exports/backend/tests/temporal/test_record_batch_model.py
+++ b/products/batch_exports/backend/tests/temporal/test_record_batch_model.py
@@ -256,8 +256,7 @@ class TestSessionsRecordBatchModel:
)
assert "INSERT INTO FUNCTION" in printed_query
- # parition_id is a ClickHouse variable, so we need to escape it
- assert "https://test-bucket.s3.amazonaws.com/test-prefix/export_{{_partition_id}}.arrow" in printed_query
+ assert "https://test-bucket.s3.amazonaws.com/test-prefix/export_{_partition_id}.arrow" in printed_query
assert "PARTITION BY rand() %% 5" in printed_query
assert f"equals(raw_sessions.team_id, {ateam.id})" in printed_query
assert (
@@ -302,10 +301,10 @@ class TestSessionsRecordBatchModel:
)
assert "INSERT INTO FUNCTION" in printed_query
- assert "https://test-bucket.s3.amazonaws.com/test-prefix/export_{{_partition_id}}.arrow" in printed_query
+ assert "https://test-bucket.s3.amazonaws.com/test-prefix/export_{_partition_id}.arrow" in printed_query
# For keyless auth, the s3() call should only have 2 parameters (url, format), not 4
assert (
- "s3('https://test-bucket.s3.amazonaws.com/test-prefix/export_{{_partition_id}}.arrow', 'ArrowStream')"
+ "s3('https://test-bucket.s3.amazonaws.com/test-prefix/export_{_partition_id}.arrow', 'ArrowStream')"
in printed_query
)
assert "PARTITION BY rand() %% 5" in printed_query
@@ -510,7 +509,7 @@ class TestHogQLQueryRecordBatchModel:
)
assert "INSERT INTO FUNCTION" in printed_query
- assert "https://test-bucket.s3.amazonaws.com/test-prefix/export_{{_partition_id}}.arrow" in printed_query
+ assert "https://test-bucket.s3.amazonaws.com/test-prefix/export_{_partition_id}.arrow" in printed_query
assert "PARTITION BY rand() %% 5" in printed_query
assert f"equals(events.team_id, {ateam.id})" in printed_query
# the user's query is wrapped as-is: settings are sent as query parameters, so we never write a