Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(signals): isolate pipeline and ranking clickhouse reads (#115522)

Oliver Browne0672cdfedef6
diff --git a/.env.services b/.env.services
index 0103510fad27..d95bebb9a4d4 100644
--- a/.env.services
+++ b/.env.services
@@ -25,6 +25,8 @@ CLICKHOUSE_BILLING_USER=billing
CLICKHOUSE_BILLING_PASSWORD=billingpass
CLICKHOUSE_DICT_READER_USER=dict_reader
CLICKHOUSE_DICT_READER_PASSWORD=dictreaderpass
+CLICKHOUSE_SIGNALS_USER=signals
+CLICKHOUSE_SIGNALS_PASSWORD=signalspass
# Redis
REDIS_URL=redis://redis7:6379/
diff --git a/docker/clickhouse/users-dev.xml b/docker/clickhouse/users-dev.xml
index f229e1196f84..58d171ca6e5a 100644
--- a/docker/clickhouse/users-dev.xml
+++ b/docker/clickhouse/users-dev.xml
@@ -92,6 +92,10 @@
<readonly><readonly/></readonly>
</constraints>
</autoresearch>
+ <signals>
+ <profile>default</profile>
+ <readonly>2</readonly>
+ </signals>
</profiles>
<!-- Users and ACL. -->
@@ -316,6 +320,12 @@
<query>GRANT SELECT ON system.query_log</query>
</grants>
</autoresearch>
+ <signals>
+ <password>signalspass</password>
+ <networks><ip>::/0</ip></networks>
+ <profile>signals</profile>
+ <quota>default</quota>
+ </signals>
</users>
<!-- Quotas. -->
diff --git a/docker/clickhouse/users.xml b/docker/clickhouse/users.xml
index c08699dcbe23..c3d9ac377ce6 100644
--- a/docker/clickhouse/users.xml
+++ b/docker/clickhouse/users.xml
@@ -71,6 +71,10 @@
<profile>default</profile>
</app>
+ <signals>
+ <profile>default</profile>
+ <readonly>2</readonly>
+ </signals>
</profiles>
<!-- Users and ACL. -->
@@ -206,6 +210,12 @@
<query>GRANT SELECT ON posthog_test.*</query>
</grants>
</dict_reader>
+ <signals>
+ <password>signalspass</password>
+ <networks><ip>::/0</ip></networks>
+ <profile>signals</profile>
+ <quota>default</quota>
+ </signals>
</users>
<!-- Quotas. -->
diff --git a/docs/internal/self-driving-temporal-worker-runbook.md b/docs/internal/self-driving-temporal-worker-runbook.md
index ef41e8a2e5a0..60d02988e684 100644
--- a/docs/internal/self-driving-temporal-worker-runbook.md
+++ b/docs/internal/self-driving-temporal-worker-runbook.md
@@ -32,6 +32,18 @@ Both live in `products/signals/backend/ranking/sweep.py`.
The video-export fleet does not register them.
All other Signals workflows (ingestion, grouping, summaries, scouts) stay on the video-export fleet.
+## ClickHouse identity
+
+Signals pipeline and Dagster ranking reads use the `signals` user. Web/API reads, diagnostic commands, and Scout/MCP query execution keep their existing credentials.
+Existing workload routing and query limits stay unchanged.
+The video-export, self-driving, Django/Celery, and Dagster workloads need `CLICKHOUSE_SIGNALS_USER=signals` and `CLICKHOUSE_SIGNALS_PASSWORD_FILE=/var/run/secrets/clickhouse-auth/token`.
+
+Deploy the ClickHouse user and ServiceAccount mappings first, then chart credentials, then the application.
+Missing credentials fail queries.
+After rollout, verify `user` or `initial_user` is `signals` in `system.query_log` and check workflow completion and capacity errors.
+
+Local development and tests use the Docker ClickHouse user; `.env.services` supplies hobby credentials.
+
## The ranking sweep schedule
`create_inbox_ranking_scoring_schedule` in `products/signals/backend/ranking/schedule.py` registers the schedule.
diff --git a/posthog/clickhouse/client/connection.py b/posthog/clickhouse/client/connection.py
index c39f05ceee08..3b430731f72e 100644
--- a/posthog/clickhouse/client/connection.py
+++ b/posthog/clickhouse/client/connection.py
@@ -102,6 +102,7 @@ class ClickHouseUser(StrEnum):
# Notebook frame materializations (Temporal worker streaming to the object store)
NOTEBOOKS = "notebooks"
ERROR_TRACKING = "error_tracking"
+ SIGNALS = "signals"
ENDPOINTS = "endpoints"
BILLING = "billing"
BUSINESS_KNOWLEDGE = "business_knowledge"
@@ -218,13 +219,19 @@ def init_clickhouse_users() -> Mapping[ClickHouseUser, ClickHouseCredentials]:
user = os.getenv(f"CLICKHOUSE_{u.name.upper()}_USER")
password = os.getenv(f"CLICKHOUSE_{u.name.upper()}_PASSWORD")
password_file = os.getenv(f"CLICKHOUSE_{u.name.upper()}_PASSWORD_FILE")
+ if (
+ u == ClickHouseUser.SIGNALS
+ and (data_stores.DEBUG or data_stores.TEST)
+ and not any((user, password, password_file))
+ ):
+ user, password = "signals", "signalspass"
secret = password or password_file
if user and secret:
user_dict[u] = ClickHouseCredentials(
user=user,
password=password or "",
password_file=password_file,
- require_password=u == ClickHouseUser.DELETION_EXECUTOR,
+ require_password=u in (ClickHouseUser.DELETION_EXECUTOR, ClickHouseUser.SIGNALS),
)
elif bool(user) != bool(secret):
logging.warning(f"only one of clickhouse user/password provided, check your config")
@@ -247,8 +254,8 @@ def get_clickhouse_creds(user: ClickHouseUser) -> ClickHouseCredentials:
This function retrieves the credentials associated with a given ClickHouse
user. Most missing dedicated users fall back to the ClickHouseUser.DEFAULT
credentials, which belong to the pod's CLICKHOUSE_USER.
- Business Knowledge fails closed so a deployment cannot silently bypass its
- isolated query limit.
+ Business Knowledge and Signals fail closed so a deployment cannot silently
+ bypass their isolated query limits.
The user and a password or token file must be properly passed as ENVs:
CLICKHOUSE_<USER_NAME>_USER
@@ -266,6 +273,11 @@ def get_clickhouse_creds(user: ClickHouseUser) -> ClickHouseCredentials:
"Business Knowledge ClickHouse credentials are missing; set "
"CLICKHOUSE_BUSINESS_KNOWLEDGE_USER and CLICKHOUSE_BUSINESS_KNOWLEDGE_PASSWORD"
)
+ if user == ClickHouseUser.SIGNALS:
+ raise RuntimeError(
+ "Signals ClickHouse credentials are missing; set CLICKHOUSE_SIGNALS_USER and "
+ "CLICKHOUSE_SIGNALS_PASSWORD or CLICKHOUSE_SIGNALS_PASSWORD_FILE"
+ )
if user == ClickHouseUser.DELETION_EXECUTOR:
raise RuntimeError(
"Data deletion request executor ClickHouse credentials are missing; set "
@@ -452,7 +464,12 @@ def get_kwargs_for_client(
return {**base_kwargs, **user_settings}
# Note that `readonly` does nothing if the relevant vars are not set!
- if readonly and settings.READONLY_CLICKHOUSE_USER is not None and settings.READONLY_CLICKHOUSE_PASSWORD:
+ if (
+ readonly
+ and ch_user != ClickHouseUser.SIGNALS
+ and settings.READONLY_CLICKHOUSE_USER is not None
+ and settings.READONLY_CLICKHOUSE_PASSWORD
+ ):
return {
"user": settings.READONLY_CLICKHOUSE_USER,
"password": settings.READONLY_CLICKHOUSE_PASSWORD,
@@ -606,8 +623,9 @@ class RefreshingChPool(ClickHouseChPool):
super().__init__(**kwargs)
def pull(self, key: str | None = None) -> SyncClient:
+ password = self._credential_provider()
client = super().pull(key)
- client.connection.password = self._credential_provider()
+ client.connection.password = password
return client
diff --git a/posthog/clickhouse/client/test/test_connection.py b/posthog/clickhouse/client/test/test_connection.py
index 12b0e466badd..8d7f086986bc 100644
--- a/posthog/clickhouse/client/test/test_connection.py
+++ b/posthog/clickhouse/client/test/test_connection.py
@@ -1,6 +1,8 @@
import json
import time
import base64
+from pathlib import Path
+from typing import TYPE_CHECKING
import pytest
from unittest.mock import patch
@@ -24,6 +26,9 @@ from posthog.clickhouse.client.connection import (
)
from posthog.clickhouse.client.execute import sync_execute
+if TYPE_CHECKING:
+ from pytest_django.fixtures import Settings
+
def _file_backed_default(monkeypatch, path, *, fallback="fallback"):
creds = ClickHouseCredentials(user="default", password=fallback, password_file=str(path))
@@ -90,6 +95,38 @@ def test_connection_pool_creation_with_team_id(settings):
assert team_pool.connection_args["host"] == "clicky"
[email protected]("user", [ClickHouseUser.SIGNALS, ClickHouseUser.DEFAULT, ClickHouseUser.ERROR_TRACKING])
+def test_signals_readonly_queries_keep_dedicated_credentials_and_offline_host(
+ settings: "Settings", monkeypatch: pytest.MonkeyPatch, tmp_path: Path, user: ClickHouseUser
+) -> None:
+ settings.READONLY_CLICKHOUSE_USER = "shared-readonly"
+ settings.READONLY_CLICKHOUSE_PASSWORD = "shared-password"
+ settings.CLICKHOUSE_OFFLINE_CLUSTER_HOST = "offline.example.com"
+ token = tmp_path / "signals-token"
+ token.write_text("signals-token-value")
+ monkeypatch.setattr(
+ connection,
+ "__user_dict",
+ {
+ ClickHouseUser.SIGNALS: ClickHouseCredentials(user="signals", password="", password_file=str(token)),
+ ClickHouseUser.DEFAULT: ClickHouseCredentials(user="default", password="default-password"),
+ ClickHouseUser.ERROR_TRACKING: ClickHouseCredentials(user="error_tracking", password="error-password"),
+ },
+ )
+
+ kwargs = get_http_kwargs(Workload.OFFLINE, readonly=True, ch_user=user)
+
+ if user == ClickHouseUser.SIGNALS:
+ assert kwargs == {
+ "user": "signals",
+ "password": "signals-token-value",
+ "host": "offline.example.com",
+ "verify": False,
+ }
+ else:
+ assert kwargs == {"user": "shared-readonly", "password": "shared-password"}
+
+
def test_native_events_pool_is_separate_from_legacy_pool(settings, monkeypatch):
from posthog.hogql_queries.utils.parallel import run_in_parallel_threads
from posthog.models.event.new_events_schema import new_events_cluster_host_scope
@@ -196,24 +233,49 @@ def test_read_password_uses_static_only_for_an_armed_user_with_an_expired_token(
assert creds.read_password() == (password if expect_static else token)
-def test_password_file_env_registers_file_backed_user(monkeypatch, tmp_path):
[email protected]("user", [ClickHouseUser.COHORTS, ClickHouseUser.SIGNALS])
+def test_password_file_env_registers_file_backed_user(monkeypatch, tmp_path, user):
token = tmp_path / "cohorts-token"
token.write_text("cohorts-token-value")
- monkeypatch.setenv("CLICKHOUSE_COHORTS_USER", "cohorts")
- monkeypatch.setenv("CLICKHOUSE_COHORTS_PASSWORD_FILE", str(token))
- monkeypatch.delenv("CLICKHOUSE_COHORTS_PASSWORD", raising=False)
+ monkeypatch.setenv(f"CLICKHOUSE_{user.name}_USER", user.value)
+ monkeypatch.setenv(f"CLICKHOUSE_{user.name}_PASSWORD_FILE", str(token))
+ monkeypatch.delenv(f"CLICKHOUSE_{user.name}_PASSWORD", raising=False)
- creds = init_clickhouse_users()[ClickHouseUser.COHORTS]
- assert creds.user == "cohorts"
+ creds = init_clickhouse_users()[user]
+ assert creds.user == user.value
assert creds.read_password() == "cohorts-token-value"
-def test_business_knowledge_credentials_do_not_fall_back_to_default(monkeypatch):
+ "debug,test,explicit_user,expected_user",
+ [
+ (False, False, None, None),
+ (True, False, None, "signals"),
+ (False, True, None, "signals"),
+ (True, False, "custom-signals", None),
+ ],
+)
+def test_signals_development_credentials_do_not_mask_deployment_config(
+ monkeypatch: pytest.MonkeyPatch, debug: bool, test: bool, explicit_user: str | None, expected_user: str | None
+) -> None:
+ monkeypatch.setattr(connection.data_stores, "DEBUG", debug)
+ monkeypatch.setattr(connection.data_stores, "TEST", test)
+ for suffix in ("USER", "PASSWORD", "PASSWORD_FILE"):
+ monkeypatch.delenv(f"CLICKHOUSE_SIGNALS_{suffix}", raising=False)
+ if explicit_user is not None:
+ monkeypatch.setenv("CLICKHOUSE_SIGNALS_USER", explicit_user)
+
+ creds = init_clickhouse_users().get(ClickHouseUser.SIGNALS)
+ assert (creds.user if creds else None) == expected_user
+
+
[email protected]("user", [ClickHouseUser.BUSINESS_KNOWLEDGE, ClickHouseUser.SIGNALS])
+def test_dedicated_credentials_do_not_fall_back_to_default(monkeypatch, user):
default_creds = ClickHouseCredentials(user="default", password="default-password")
monkeypatch.setattr(connection, "__user_dict", {ClickHouseUser.DEFAULT: default_creds})
- with pytest.raises(RuntimeError, match="CLICKHOUSE_BUSINESS_KNOWLEDGE_USER"):
- get_clickhouse_creds(ClickHouseUser.BUSINESS_KNOWLEDGE)
+ with pytest.raises(RuntimeError, match=f"CLICKHOUSE_{user.name}_USER"):
+ get_clickhouse_creds(user)
def test_deletion_executor_credentials_do_not_fall_back_to_default(monkeypatch):
@@ -290,11 +352,13 @@ def test_default_client_sends_the_pod_token_only_without_explicit_credentials(
assert (client.connection.user, client.connection.password) == (expected_user, expected_password)
-def test_refreshing_pool_stamps_current_credential(tmp_path):
[email protected]("failure", [None, "missing", "empty"])
+def test_refreshing_pool_stamps_current_credential(tmp_path: Path, failure: str | None) -> None:
token = tmp_path / "token"
token.write_text("tok-0")
+ creds = ClickHouseCredentials(user="signals", password="", password_file=str(token), require_password=True)
pool = RefreshingChPool(
- credential_provider=lambda: token.read_text().strip(),
+ credential_provider=creds.read_password,
host="localhost",
port=1,
connections_min=1,
@@ -305,6 +369,16 @@ def test_refreshing_pool_stamps_current_credential(tmp_path):
assert client.connection.password == "tok-0"
pool.push(client)
+ if failure is not None:
+ if failure == "missing":
+ token.unlink()
+ else:
+ token.write_text("")
+ for _ in range(3):
+ with pytest.raises(RuntimeError, match="signals"):
+ with pool.get_client():
+ pytest.fail("Missing credentials must fail closed")
+
token.write_text("tok-1")
client = pool.pull()
assert client.connection.password == "tok-1"
diff --git a/products/product_analytics/backend/facade/queries.py b/products/product_analytics/backend/facade/queries.py
index 9380cbfa5b1c..0833adc8038f 100644
--- a/products/product_analytics/backend/facade/queries.py
+++ b/products/product_analytics/backend/facade/queries.py
@@ -48,6 +48,7 @@ __all__ = sorted((*_LAZY, "run_cached_trends_query"))
if TYPE_CHECKING:
+ from posthog.clickhouse.client.connection import ClickHouseUser
from posthog.models import Team
# Static view for mypy and IDEs only; runtime resolves through __getattr__ below, so these
@@ -77,7 +78,12 @@ if TYPE_CHECKING:
def run_cached_trends_query(
- *, query: dict[str, Any], team: Team, max_execution_time_seconds: int, cache_age_seconds: int
+ *,
+ query: dict[str, Any],
+ team: Team,
+ max_execution_time_seconds: int,
+ cache_age_seconds: int,
+ ch_user: ClickHouseUser | None = None,
) -> TrendsQueryRunResult:
"""Run a Trends query through the blocking recent-cache path."""
@@ -92,6 +98,7 @@ def run_cached_trends_query(
query=query,
team=team,
hogql_settings=HogQLGlobalSettings(max_execution_time=max_execution_time_seconds),
+ ch_user=ch_user,
)
runner.is_query_service = is_api_key_access_method(get_query_tag_value("access_method"))
response = runner.run(
diff --git a/products/product_analytics/backend/hogql_queries/trends/test/test_trends_query_runner.py b/products/product_analytics/backend/hogql_queries/trends/test/test_trends_query_runner.py
index 2937a0868ac9..842f5493c51c 100644
--- a/products/product_analytics/backend/hogql_queries/trends/test/test_trends_query_runner.py
+++ b/products/product_analytics/backend/hogql_queries/trends/test/test_trends_query_runner.py
@@ -67,6 +67,7 @@ from posthog.hogql.database.database import Database
from posthog.hogql.modifiers import create_default_modifiers_for_team
from posthog.hogql.query import execute_hogql_query
+from posthog.clickhouse.client.connection import ClickHouseUser
from posthog.clickhouse.client.execute import sync_execute
from posthog.hogql_queries.utils.breakdowns import (
BREAKDOWN_NULL_DISPLAY,
@@ -8239,9 +8240,15 @@ class TestTrendsQueryRunner(ClickhouseTestMixin, APIBaseTest):
class TestTrendsQueryRunnerParallelExecution(ClickhouseTestMixin, APIBaseTest):
"""Runs without override_settings(IN_UNIT_TESTING=True) so the queries go through the executor."""
- def _run(self, series: list[EventsNode], breakdown: Optional[BreakdownFilter] = None):
+ def _run(
+ self,
+ series: list[EventsNode],
+ breakdown: Optional[BreakdownFilter] = None,
+ ch_user: ClickHouseUser = ClickHouseUser.DEFAULT,
+ ) -> TrendsQueryResponse:
return TrendsQueryRunner(
team=self.team,
+ ch_user=ch_user,
query=TrendsQuery(
dateRange=DateRange(date_from="2020-01-09", date_to="2020-01-20"),
interval=IntervalType.DAY,
@@ -8250,13 +8257,20 @@ class TestTrendsQueryRunnerParallelExecution(ClickhouseTestMixin, APIBaseTest):
),
).calculate()
- def test_every_series_comes_back_when_the_queries_run_through_the_executor(self):
+ @parameterized.expand([(ClickHouseUser.DEFAULT,), (ClickHouseUser.SIGNALS,)])
+ def test_every_series_comes_back_when_the_queries_run_through_the_executor(self, ch_user: ClickHouseUser) -> None:
_create_person(distinct_ids=["p1"], team=self.team)
for event in ("a", "b", "c"):
_create_event(team=self.team, event=event, distinct_id="p1", timestamp="2020-01-11T12:00:00Z")
flush_persons_and_events()
- response = self._run([EventsNode(event=event) for event in ("a", "b", "c")])
+ with patch(
+ "products.product_analytics.backend.hogql_queries.trends.trends_query_runner.execute_hogql_query",
+ wraps=execute_hogql_query,
+ ) as query:
+ response = self._run([EventsNode(event=event) for event in ("a", "b", "c")], ch_user=ch_user)
+ assert len(query.call_args_list) == 3
+ assert all(call.kwargs["ch_user"] == ch_user for call in query.call_args_list)
self.assertEqual([result["label"] for result in response.results], ["a", "b", "c"])
self.assertEqual([sum(result["data"]) for result in response.results], [1, 1, 1])
diff --git a/products/product_analytics/backend/hogql_queries/trends/trends_query_runner.py b/products/product_analytics/backend/hogql_queries/trends/trends_query_runner.py
index 85a38dbede52..ec351fdae81a 100644
--- a/products/product_analytics/backend/hogql_queries/trends/trends_query_runner.py
+++ b/products/product_analytics/backend/hogql_queries/trends/trends_query_runner.py
@@ -52,7 +52,7 @@ from posthog.caching.insights_api import (
REAL_TIME_INSIGHT_REFRESH_INTERVAL,
REDUCED_MINIMUM_INSIGHT_REFRESH_INTERVAL,
)
-from posthog.clickhouse.client.connection import Workload
+from posthog.clickhouse.client.connection import ClickHouseUser, Workload
from posthog.hogql_queries.query_runner import AnalyticsQueryRunner, resolve_series_custom_name
from posthog.hogql_queries.utils.breakdowns import (
BREAKDOWN_NULL_DISPLAY,
@@ -115,6 +115,7 @@ class TrendsQueryRunner(AnalyticsQueryRunner[TrendsQueryResponse]):
*,
workload: Workload = Workload.DEFAULT,
hogql_settings: Optional[HogQLGlobalSettings] = None,
+ ch_user: ClickHouseUser | None = None,
):
from posthog.hogql_queries.utils.utils import convert_active_user_math_based_on_interval
@@ -161,6 +162,7 @@ class TrendsQueryRunner(AnalyticsQueryRunner[TrendsQueryResponse]):
limit_context=limit_context,
workload=workload,
user=user,
+ ch_user=ch_user or ClickHouseUser.DEFAULT,
)
def __post_init__(self):
@@ -169,9 +171,12 @@ class TrendsQueryRunner(AnalyticsQueryRunner[TrendsQueryResponse]):
def single_flight_variant(self) -> str:
# A caller can pass its own execution time, which does not reach the cache key.
- if self.hogql_settings is None:
- return super().single_flight_variant()
- return f"{super().single_flight_variant()}:max_execution_time={self.hogql_settings.max_execution_time}"
+ variant = super().single_flight_variant()
+ if self.ch_user != ClickHouseUser.DEFAULT:
+ variant = f"{variant}:ch_user={self.ch_user.value}"
+ if self.hogql_settings is not None:
+ variant = f"{variant}:max_execution_time={self.hogql_settings.max_execution_time}"
+ return variant
def validators(self) -> Sequence[QueryValidationRule[TrendsQuery]]:
return (
@@ -352,6 +357,7 @@ class TrendsQueryRunner(AnalyticsQueryRunner[TrendsQueryResponse]):
team=self.team,
user=self.user,
workload=self.workload,
+ ch_user=self.ch_user,
settings=self.hogql_settings,
# timings=timings,
# modifiers=modifiers,
@@ -426,6 +432,7 @@ class TrendsQueryRunner(AnalyticsQueryRunner[TrendsQueryResponse]):
team=self.team,
user=self.user,
workload=self.workload,
+ ch_user=self.ch_user,
settings=self.hogql_settings,
timings=timings,
modifiers=self.modifiers,
diff --git a/products/product_analytics/backend/tests/test_facade.py b/products/product_analytics/backend/tests/test_facade.py
index 73ec13a7098b..2419aab75f0a 100644
--- a/products/product_analytics/backend/tests/test_facade.py
+++ b/products/product_analytics/backend/tests/test_facade.py
@@ -11,6 +11,7 @@ from parameterized import parameterized
from posthog.schema import TrendsQueryResponse
+from posthog.clickhouse.client.connection import ClickHouseUser
from posthog.clickhouse.query_tagging import reset_query_tags, tag_queries
from posthog.models.team import Team
@@ -103,7 +104,11 @@ class TestRunCachedTrendsQuery(BaseTest):
)
result = run_cached_trends_query(
- query=query, team=self.team, max_execution_time_seconds=20, cache_age_seconds=900
+ query=query,
+ team=self.team,
+ max_execution_time_seconds=20,
+ cache_age_seconds=900,
+ ch_user=ClickHouseUser.SIGNALS,
)
assert result.results == [{"aggregated_value": 12}]
@@ -111,6 +116,7 @@ class TestRunCachedTrendsQuery(BaseTest):
assert runner_type.call_args.kwargs["query"] == query
assert runner_type.call_args.kwargs["team"] == self.team
assert runner_type.call_args.kwargs["hogql_settings"].max_execution_time == 20
+ assert runner_type.call_args.kwargs["ch_user"] == ClickHouseUser.SIGNALS
assert runner_type.return_value.is_query_service is True
assert runner_type.return_value.run.call_args.kwargs["cache_age_seconds"] == 900
assert (
diff --git a/products/signals/backend/auto_start.py b/products/signals/backend/auto_start.py
index edf5fcbc39ed..3d4e846d9dae 100644
--- a/products/signals/backend/auto_start.py
+++ b/products/signals/backend/auto_start.py
@@ -13,6 +13,7 @@ from django.utils.text import slugify
import structlog
import posthoganalytics
+from posthog.clickhouse.client.connection import ClickHouseUser
from posthog.dataclasses import frozen
from posthog.event_usage import groups
from posthog.models import Team, User
@@ -428,11 +429,16 @@ def _build_autostart_task_description(
)
-def _fetch_source_references(team_id: int, report_id: str) -> list[SignalSourceReference]:
+def _fetch_source_references(
+ team_id: int,
+ report_id: str,
+ *,
+ ch_user: ClickHouseUser = ClickHouseUser.DEFAULT,
+) -> list[SignalSourceReference]:
"""PR traceability is best-effort: a ClickHouse hiccup here must not block auto-start."""
try:
team = Team.objects.get(pk=team_id)
- return fetch_source_references_for_report(team, report_id)
+ return fetch_source_references_for_report(team, report_id, ch_user=ch_user)
except Exception:
logger.exception("signals auto-start source reference fetch failed", report_id=report_id, team_id=team_id)
return []
@@ -458,11 +464,18 @@ def _stamp_billing_exemption(report: SignalReport, declared_reason: str | None)
return report.billing_exempt_reason
-def _capture_billing_exempted(*, team: Team, report_id: str, reason: str, task_id: str) -> None:
+def _capture_billing_exempted(
+ *,
+ team: Team,
+ report_id: str,
+ reason: str,
+ task_id: str,
+ ch_user: ClickHouseUser = ClickHouseUser.DEFAULT,
+) -> None:
"""`signals_pr_billing_exempted` — exempt volume is a cost signal for the weekly refund review."""
try:
source_products: list[str] = []
- meta = fetch_source_products_for_reports(team, [report_id]).get(report_id)
+ meta = fetch_source_products_for_reports(team, [report_id], ch_user=ch_user).get(report_id)
if meta is not None:
source_products = meta.source_products
posthoganalytics.capture(
@@ -742,6 +755,7 @@ def _create_implementation_task_if_absent(
dispatch: ImplementationDispatch | None = None,
requested_after_run_count: int | None = None,
runner_source: str = "requested",
+ ch_user: ClickHouseUser = ClickHouseUser.DEFAULT,
) -> bool | AutostartSkip:
"""Create the implementation task and record it (gate row + work-log artefact), serialized per report.
@@ -928,7 +942,9 @@ def _create_implementation_task_if_absent(
if exempt_reason and task_id:
# After commit: the exempt report's implementation task exists — count it (includes a
# best-effort ClickHouse lookup, so it must not run under the lock).
- _capture_billing_exempted(team=team, report_id=report_id, reason=exempt_reason, task_id=task_id)
+ _capture_billing_exempted(
+ team=team, report_id=report_id, reason=exempt_reason, task_id=task_id, ch_user=ch_user
+ )
if task_id:
_capture_steering_attached(
team=team, report_id=report_id, task_id=task_id, steering=steering, runner_source=runner_source
@@ -936,7 +952,11 @@ def _create_implementation_task_if_absent(
return True
-def start_requested_implementation(request: RequestedImplementation) -> str:
+def start_requested_implementation(
+ request: RequestedImplementation,
+ *,
+ ch_user: ClickHouseUser = ClickHouseUser.DEFAULT,
+) -> str:
team = Team.objects.select_related("organization").get(id=request.team_id)
user = User.objects.filter(id=request.user_id, is_active=True, organization__id=team.organization_id).first()
if user is None or UserPermissions(user=user, team=team).current_team.effective_membership_level is None:
@@ -1031,7 +1051,7 @@ def start_requested_implementation(request: RequestedImplementation) -> str:
summary=report.summary,
repository=repo_selection.repository,
priority=priority,
- source_references=_fetch_source_references(request.team_id, request.report_id),
+ source_references=_fetch_source_references(request.team_id, request.report_id, ch_user=ch_user),
steering=steering,
),
expected_content=expected_content,
@@ -1041,6 +1061,7 @@ def start_requested_implementation(request: RequestedImplementation) -> str:
steering=steering,
free_trial_enabled=self_driving_free_trial_enabled(team),
requested_after_run_count=request.after_run_count,
+ ch_user=ch_user,
)
if not created:
raise RequestedImplementationUnavailable("Another implementation started before this request")
@@ -1326,6 +1347,7 @@ async def maybe_autostart_implementation_task(
implementation_decision: ImplementationDecision | None = None,
remaining_retries: int = 2,
dispatch: ImplementationDispatch | None = None,
+ ch_user: ClickHouseUser = ClickHouseUser.DEFAULT,
) -> AutostartOutcome:
"""Start an implementation Task for a SignalReport if autonomy + priority allow it.
@@ -1389,7 +1411,11 @@ async def maybe_autostart_implementation_task(
if title != expected_content.title or summary != expected_content.summary:
if remaining_retries:
return await maybe_autostart_from_report_artefacts(
- team_id=team_id, report_id=report_id, remaining_retries=remaining_retries - 1, dispatch=dispatch
+ team_id=team_id,
+ report_id=report_id,
+ remaining_retries=remaining_retries - 1,
+ dispatch=dispatch,
+ ch_user=ch_user,
)
raise ReportChangedDuringAutostart("Report kept changing during autostart")
supersede = await database_sync_to_async(_resolve_supersede, thread_sensitive=False)(
@@ -1559,7 +1585,7 @@ async def maybe_autostart_implementation_task(
base_branch = stack_base_branch
source_references = await database_sync_to_async(_fetch_source_references, thread_sensitive=False)(
- team_id, report_id
+ team_id, report_id, ch_user=ch_user
)
steering = await database_sync_to_async(load_report_steering, thread_sensitive=False)(
team_id, report_id, memory_writable=grants_scratchpad_write(IMPLEMENTATION_MCP_SCOPES)
@@ -1594,11 +1620,16 @@ async def maybe_autostart_implementation_task(
supersede=supersede,
dispatch=dispatch,
runner_source=runner_source,
+ ch_user=ch_user,
)
except ReportChangedDuringAutostart:
if remaining_retries:
return await maybe_autostart_from_report_artefacts(
- team_id=team_id, report_id=report_id, remaining_retries=remaining_retries - 1, dispatch=dispatch
+ team_id=team_id,
+ report_id=report_id,
+ remaining_retries=remaining_retries - 1,
+ dispatch=dispatch,
+ ch_user=ch_user,
)
raise ReportChangedDuringAutostart("Report kept changing during autostart")
if isinstance(created, AutostartSkip):
@@ -1657,7 +1688,12 @@ async def _latest_reviewers_content(report_id: str) -> tuple[list[ReviewerConten
async def maybe_autostart_from_report_artefacts(
- *, team_id: int, report_id: str, remaining_retries: int = 2, dispatch: ImplementationDispatch | None = None
+ *,
+ team_id: int,
+ report_id: str,
+ remaining_retries: int = 2,
+ dispatch: ImplementationDispatch | None = None,
+ ch_user: ClickHouseUser = ClickHouseUser.DEFAULT,
) -> AutostartOutcome:
"""Re-evaluate auto-start from a report's *current* artefacts.
@@ -1760,4 +1796,5 @@ async def maybe_autostart_from_report_artefacts(
implementation_decision=implementation_decision,
remaining_retries=remaining_retries,
dispatch=dispatch,
+ ch_user=ch_user,
)
diff --git a/products/signals/backend/emission/fetchers/data_warehouse.py b/products/signals/backend/emission/fetchers/data_warehouse.py
index c1b04a2ccec8..46597bcc7f96 100644
--- a/products/signals/backend/emission/fetchers/data_warehouse.py
+++ b/products/signals/backend/emission/fetchers/data_warehouse.py
@@ -8,6 +8,8 @@ from posthog.hogql.escape_sql import escape_hogql_identifier
from posthog.hogql.parser import parse_select
from posthog.hogql.query import execute_hogql_query
+from posthog.clickhouse.client.connection import ClickHouseUser
+from posthog.clickhouse.workload import Workload
from posthog.models import Team
from products.signals.backend.contracts import scope_ids_problem
@@ -109,7 +111,12 @@ def data_warehouse_record_fetcher(
# Internal data-import signal fetcher (no user); bypass warehouse HogQL access control so it
# can read the source warehouse table.
result = execute_hogql_query(
- query=parsed, team=team, query_type="EmitSignalsNewRecords", bypass_warehouse_access_control=True
+ query=parsed,
+ team=team,
+ query_type="EmitSignalsNewRecords",
+ bypass_warehouse_access_control=True,
+ ch_user=ClickHouseUser.SIGNALS,
+ workload=Workload.OFFLINE,
)
except Exception as e:
logger.exception(f"Error querying new records: {e}", **extra)
diff --git a/products/signals/backend/emission/google_search_console_opportunities.py b/products/signals/backend/emission/google_search_console_opportunities.py
index 457290130756..0b86976e47b9 100644
--- a/products/signals/backend/emission/google_search_console_opportunities.py
+++ b/products/signals/backend/emission/google_search_console_opportunities.py
@@ -31,6 +31,8 @@ import structlog
from posthog.hogql.parser import parse_select
from posthog.hogql.query import execute_hogql_query
+from posthog.clickhouse.client.connection import ClickHouseUser
+from posthog.clickhouse.workload import Workload
from posthog.models import Team
from products.signals.backend.emission.fetchers.data_warehouse import escape_table_name
@@ -176,6 +178,8 @@ def google_search_console_record_fetcher(
team=team,
query_type="EmitSignalsNewRecords",
bypass_warehouse_access_control=True,
+ ch_user=ClickHouseUser.SIGNALS,
+ workload=Workload.OFFLINE,
)
except Exception as e:
logger.exception(f"Error querying Google Search Console opportunities: {e}", **extra)
diff --git a/products/signals/backend/emission/pganalyze_issues.py b/products/signals/backend/emission/pganalyze_issues.py
index 811ed209bcda..137af734645d 100644
--- a/products/signals/backend/emission/pganalyze_issues.py
+++ b/products/signals/backend/emission/pganalyze_issues.py
@@ -8,6 +8,8 @@ from posthog.hogql import ast
from posthog.hogql.parser import parse_select
from posthog.hogql.query import execute_hogql_query
+from posthog.clickhouse.client.connection import ClickHouseUser
+from posthog.clickhouse.workload import Workload
from posthog.models import Team
from products.signals.backend.emission.fetchers.data_warehouse import escape_table_name
@@ -158,6 +160,8 @@ def _fetch_issue_page(
team=team,
query_type="EmitSignalsNewRecords",
bypass_warehouse_access_control=True,
+ ch_user=ClickHouseUser.SIGNALS,
+ workload=Workload.OFFLINE,
)
if not result.results or not result.columns:
return []
diff --git a/products/signals/backend/implementation_dispatch.py b/products/signals/backend/implementation_dispatch.py
index b630452f2a73..ab438f022d5c 100644
--- a/products/signals/backend/implementation_dispatch.py
+++ b/products/signals/backend/implementation_dispatch.py
@@ -11,6 +11,8 @@ import structlog
from asgiref.sync import async_to_sync
from pydantic import ValidationError
+from posthog.clickhouse.client.connection import ClickHouseUser
+
from products.signals.backend.artefact_attribution import ArtefactAttribution
from products.signals.backend.artefact_schemas import (
ImplementationDecision,
@@ -178,7 +180,7 @@ class ImplementationDispatcher:
team_id=team_id, report_id=str(decision.report_id), attribution=ArtefactAttribution.system()
)
outcome = async_to_sync(maybe_autostart_from_report_artefacts)(
- team_id=team_id, report_id=str(decision.report_id), dispatch=reservation
+ team_id=team_id, report_id=str(decision.report_id), dispatch=reservation, ch_user=ClickHouseUser.SIGNALS
)
except Exception:
logger.exception("signals_implementation_dispatch_failed", decision_id=decision_id)
diff --git a/products/signals/backend/ranking/sweep.py b/products/signals/backend/ranking/sweep.py
index 6ca20488cfa5..71736a14741c 100644
--- a/products/signals/backend/ranking/sweep.py
+++ b/products/signals/backend/ranking/sweep.py
@@ -26,8 +26,9 @@ from temporalio import activity, workflow
from temporalio.common import RetryPolicy
from posthog.clickhouse.client import sync_execute
-from posthog.clickhouse.client.connection import Workload
+from posthog.clickhouse.client.connection import ClickHouseUser
from posthog.clickhouse.query_tagging import Feature, Product, tag_queries
+from posthog.clickhouse.workload import Workload
from posthog.dataclasses import frozen
from posthog.ph_client import ph_scoped_capture
from posthog.sync import database_sync_to_async
@@ -154,6 +155,7 @@ def _live_vectors(since: datetime.datetime, rendering: str) -> dict[str, tuple[i
},
settings=CANDIDATE_VECTORS_QUERY_SETTINGS,
workload=Workload.OFFLINE,
+ ch_user=ClickHouseUser.SIGNALS,
)
or [],
)
diff --git a/products/signals/backend/report_check_execution.py b/products/signals/backend/report_check_execution.py
index 16ed5617aeee..ebd5208c495a 100644
--- a/products/signals/backend/report_check_execution.py
+++ b/products/signals/backend/report_check_execution.py
@@ -37,6 +37,7 @@ from posthog.schema import (
InsightThresholdType,
)
+from posthog.clickhouse.client.connection import ClickHouseUser
from posthog.clickhouse.query_tagging import tag_queries
from posthog.dataclasses import frozen
from posthog.models import Team
@@ -252,7 +253,9 @@ def measure_check(check: SignalReportCheck, *, deadline: float) -> CheckVerdict:
},
}
tag_queries(trigger="signals_report_check")
- measurement = measure_metric(query, check.report.team, deadline=deadline, include_series=False)
+ measurement = measure_metric(
+ query, check.report.team, deadline=deadline, include_series=False, ch_user=ClickHouseUser.SIGNALS
+ )
except Exception as error:
logger.exception("signals.report_check.measurement_failed", check_id=str(check.id), team_id=check.team_id)
return CheckVerdict(outcome="errored", explanation=_errored_explanation(error, subject=check.title))
diff --git a/products/signals/backend/report_embedding_reader.py b/products/signals/backend/report_embedding_reader.py
index 27b4a62d5027..1f2a443e5104 100644
--- a/products/signals/backend/report_embedding_reader.py
+++ b/products/signals/backend/report_embedding_reader.py
@@ -16,8 +16,9 @@ from django.conf import settings
import numpy as np
from posthog.clickhouse.client import sync_execute
-from posthog.clickhouse.client.connection import Workload
+from posthog.clickhouse.client.connection import ClickHouseUser
from posthog.clickhouse.query_tagging import Feature, Product, tag_queries
+from posthog.clickhouse.workload import Workload
from posthog.dataclasses import frozen
from products.signals.backend.ranking.features import EMBEDDING_DIMENSIONS
@@ -80,6 +81,7 @@ def latest_report_vectors(team_id: int, report_ids: Sequence[str], *, rendering:
},
workload=Workload.OFFLINE,
team_id=team_id,
+ ch_user=ClickHouseUser.SIGNALS,
)
or [],
)
diff --git a/products/signals/backend/report_metric_refresh.py b/products/signals/backend/report_metric_refresh.py
index 10a88cc311b4..2b4b8de2113d 100644
--- a/products/signals/backend/report_metric_refresh.py
+++ b/products/signals/backend/report_metric_refresh.py
@@ -29,6 +29,7 @@ import structlog
from posthog.schema import ChartDisplayType
+from posthog.clickhouse.client.connection import ClickHouseUser
from posthog.clickhouse.query_tagging import tag_queries
from posthog.dataclasses import frozen
from posthog.models import Team
@@ -101,7 +102,12 @@ def _source_count(query: dict[str, Any]) -> int:
def _run_metric_query(
- query: dict[str, Any], team: Team, display: ChartDisplayType, *, deadline: float
+ query: dict[str, Any],
+ team: Team,
+ display: ChartDisplayType,
+ *,
+ deadline: float,
+ ch_user: ClickHouseUser = ClickHouseUser.DEFAULT,
) -> tuple[dict, datetime | None]:
"""Run the stored query in one derived display shape and return its first series and response.
@@ -135,6 +141,7 @@ def _run_metric_query(
team=team,
max_execution_time_seconds=timeout,
cache_age_seconds=int(REPORT_METRIC_SNAPSHOT_FRESH_FOR.total_seconds()),
+ ch_user=ch_user,
)
results = response.results
if not isinstance(results, list) or not results or not isinstance(results[0], dict):
@@ -151,8 +158,16 @@ def _finite_number(raw: object, *, what: str) -> float:
return value
-def whole_window_value(query: dict[str, Any], team: Team, *, deadline: float = math.inf) -> tuple[float, datetime]:
- first_series, last_refresh = _run_metric_query(query, team, ChartDisplayType.BOLD_NUMBER, deadline=deadline)
+def whole_window_value(
+ query: dict[str, Any],
+ team: Team,
+ *,
+ deadline: float = math.inf,
+ ch_user: ClickHouseUser = ClickHouseUser.DEFAULT,
+) -> tuple[float, datetime]:
+ first_series, last_refresh = _run_metric_query(
+ query, team, ChartDisplayType.BOLD_NUMBER, deadline=deadline, ch_user=ch_user
+ )
value = _finite_number(first_series.get("aggregated_value"), what="aggregate")
measured_at = last_refresh if isinstance(last_refresh, datetime) else timezone.now()
if measured_at.tzinfo is None or measured_at.utcoffset() is None:
@@ -160,23 +175,36 @@ def whole_window_value(query: dict[str, Any], team: Team, *, deadline: float = m
return value, measured_at
-def longitudinal_values(query: dict[str, Any], team: Team, *, deadline: float = math.inf) -> list[float]:
+def longitudinal_values(
+ query: dict[str, Any],
+ team: Team,
+ *,
+ deadline: float = math.inf,
+ ch_user: ClickHouseUser = ClickHouseUser.DEFAULT,
+) -> list[float]:
"""The trailing per-bucket values, oldest first, for the row-sized trend strip."""
- first_series, _ = _run_metric_query(query, team, ChartDisplayType.ACTIONS_BAR, deadline=deadline)
+ first_series, _ = _run_metric_query(query, team, ChartDisplayType.ACTIONS_BAR, deadline=deadline, ch_user=ch_user)
raw_points = first_series.get("data")
if not isinstance(raw_points, list):
raise ValueError("metric query returned no buckets")
return [_finite_number(point, what="bucket") for point in raw_points[-MAX_METRIC_SERIES_POINTS:]]
-def measure_metric(query: dict[str, Any], team: Team, *, deadline: float, include_series: bool) -> MetricMeasurement:
- value, measured_at = whole_window_value(query, team, deadline=deadline)
+def measure_metric(
+ query: dict[str, Any],
+ team: Team,
+ *,
+ deadline: float,
+ include_series: bool,
+ ch_user: ClickHouseUser = ClickHouseUser.DEFAULT,
+) -> MetricMeasurement:
+ value, measured_at = whole_window_value(query, team, deadline=deadline, ch_user=ch_user)
series: list[float] | None
if not include_series or time.monotonic() >= deadline:
return MetricMeasurement(value=value, measured_at=measured_at, series=None)
try:
- series = longitudinal_values(query, team, deadline=deadline)
+ series = longitudinal_values(query, team, deadline=deadline, ch_user=ch_user)
except Exception:
# The buckets only decorate the row, so losing them must not cost the headline value.
logger.exception("signals.report_metric_refresh.series_query_failed", team_id=team.id)
diff --git a/products/signals/backend/signal_metadata.py b/products/signals/backend/signal_metadata.py
index 9ea263a2172d..e548a5eb1506 100644
--- a/products/signals/backend/signal_metadata.py
+++ b/products/signals/backend/signal_metadata.py
@@ -19,7 +19,9 @@ from posthog.hogql.context import HogQLContext
from posthog.hogql.database.database import Database
from posthog.hogql.query import execute_hogql_query
+from posthog.clickhouse.client.connection import ClickHouseUser
from posthog.clickhouse.query_tagging import Feature, Product, tag_queries
+from posthog.clickhouse.workload import Workload
from posthog.dataclasses import frozen
from posthog.models import Team
@@ -146,7 +148,12 @@ class ReportSignalMeta:
scout_name: str | None
-def fetch_source_products_for_reports(team: Team, report_ids: list[str]) -> dict[str, ReportSignalMeta]:
+def fetch_source_products_for_reports(
+ team: Team,
+ report_ids: list[str],
+ *,
+ ch_user: ClickHouseUser = ClickHouseUser.DEFAULT,
+) -> dict[str, ReportSignalMeta]:
"""Return a mapping of report_id -> `ReportSignalMeta` (distinct source_products + authoring scout).
Only includes non-deleted signals. Source products are returned in sorted order. `scout_name` is
@@ -202,6 +209,7 @@ def fetch_source_products_for_reports(team: Team, report_ids: list[str]) -> dict
tag_queries(product=Product.SIGNALS, feature=Feature.QUERY)
result = execute_hogql_query(
+ ch_user=ch_user,
query_type="SignalsFetchSourceProductsForReports",
query=ch_query,
team=team,
@@ -299,7 +307,12 @@ _LABEL_RE = re.compile(r"^[A-Za-z0-9#/_.-]{1,64}$")
_URL_RE = re.compile(r"^https?://[^\s()<>\[\]]{1,500}$")
-def fetch_source_references_for_report(team: Team, report_id: str) -> list[SignalSourceReference]:
+def fetch_source_references_for_report(
+ team: Team,
+ report_id: str,
+ *,
+ ch_user: ClickHouseUser = ClickHouseUser.DEFAULT,
+) -> list[SignalSourceReference]:
"""Return issue references (label + URL) for the report's non-deleted Linear/GitHub signals.
Sources whose signals don't carry a stable human-facing URL (e.g. Zendesk's `url` extra is the
@@ -345,6 +358,7 @@ def fetch_source_references_for_report(team: Team, report_id: str) -> list[Signa
tag_queries(product=Product.SIGNALS, feature=Feature.QUERY)
result = execute_hogql_query(
+ ch_user=ch_user,
query_type="SignalsFetchSourceReferencesForReport",
query=ch_query,
team=team,
@@ -476,13 +490,22 @@ def _report_placeholders(report_id: str) -> dict:
}
-def fetch_signals_for_report_sync(team: Team, report_id: str, newest: int | None = None) -> list[dict]:
+def fetch_signals_for_report_sync(
+ team: Team,
+ report_id: str,
+ newest: int | None = None,
+ *,
+ ch_user: ClickHouseUser = ClickHouseUser.DEFAULT,
+ workload: Workload = Workload.DEFAULT,
+) -> list[dict]:
"""Fetch the signals of a report from ClickHouse, including full metadata. Synchronous.
With `newest`, fetch only that many of the newest signals.
"""
tag_queries(product=Product.SIGNALS, feature=Feature.QUERY)
result = execute_hogql_query(
+ ch_user=ch_user,
+ workload=workload,
query_type="SignalsDebugFetchForReport",
query=_signals_for_report_query(limit=newest, newest_first=newest is not None),
team=team,
diff --git a/products/signals/backend/spend.py b/products/signals/backend/spend.py
index 5482fe8410db..ede997b3e69f 100644
--- a/products/signals/backend/spend.py
+++ b/products/signals/backend/spend.py
@@ -12,6 +12,7 @@ from django.utils import timezone
import structlog
from asgiref.sync import async_to_sync
+from posthog.clickhouse.client.connection import ClickHouseUser
from posthog.dataclasses import frozen
from posthog.sync import database_sync_to_async
@@ -41,7 +42,9 @@ def signal_id_for(*, team_id: int, source_product: str, source_type: str, idempo
)
-def report_triggering_signal(*, team_id: int, report_id: str) -> str | None:
+def report_triggering_signal(
+ *, team_id: int, report_id: str, ch_user: ClickHouseUser = ClickHouseUser.DEFAULT
+) -> str | None:
with transaction.atomic():
report = (
SignalReport.objects.filter(team_id=team_id, id=report_id).only("triggering_signal_id", "team_id").first()
@@ -55,7 +58,7 @@ def report_triggering_signal(*, team_id: int, report_id: str) -> str | None:
fetch_signals_for_report_sync, # noqa: PLC0415 — keeps HogQL off Django startup
)
- signals = fetch_signals_for_report_sync(report.team, report_id, newest=1)
+ signals = fetch_signals_for_report_sync(report.team, report_id, newest=1, ch_user=ch_user)
if not signals:
return None
with transaction.atomic():
diff --git a/products/signals/backend/spend_metadata.py b/products/signals/backend/spend_metadata.py
index 57375ff0ec7a..b564de23b840 100644
--- a/products/signals/backend/spend_metadata.py
+++ b/products/signals/backend/spend_metadata.py
@@ -11,6 +11,7 @@ from posthog.schema import EmbeddingModelName
from posthog.hogql import ast
from posthog.hogql.query import execute_hogql_query
+from posthog.clickhouse.client.connection import ClickHouseUser
from posthog.clickhouse.query_tagging import Feature, Product, tag_queries
from posthog.dataclasses import frozen
from posthog.kafka_client.routing import producer_scope
@@ -68,6 +69,7 @@ class _SignalEmbedding:
def _read_embedding(team: Team, signal_id: str, model: EmbeddingModelName) -> _SignalEmbedding | None:
result = execute_hogql_query(
query_type="SignalsRefreshSpendMetadata",
+ ch_user=ClickHouseUser.SIGNALS,
query="""
SELECT document_id, argMax(tuple(content, metadata, timestamp, embedding, inserted_at), inserted_at)
FROM document_embeddings
diff --git a/products/signals/backend/stack_plan.py b/products/signals/backend/stack_plan.py
index 6d2ef48efaa6..aee0514e2ddd 100644
--- a/products/signals/backend/stack_plan.py
+++ b/products/signals/backend/stack_plan.py
@@ -20,6 +20,7 @@ import structlog
from asgiref.sync import async_to_sync
from pydantic import ValidationError
+from posthog.clickhouse.client.connection import ClickHouseUser
from posthog.models.github_integration_base import GitHubIntegrationBase
from products.signals.backend.artefact_attribution import ArtefactAttribution
@@ -213,7 +214,12 @@ def _dependencies_have_pull_requests(*, team_id: int, report_id: str) -> bool:
return set(dependency_ids) <= has_open_or_merged_pull_request(team_id=team_id, report_ids=dependency_ids)
-def start_unblocked_layers(*, team_id: int, report_ids: list[str]) -> list[str]:
+def start_unblocked_layers(
+ *,
+ team_id: int,
+ report_ids: list[str],
+ ch_user: ClickHouseUser = ClickHouseUser.DEFAULT,
+) -> list[str]:
"""Run auto-start for each layer whose dependencies all have a pull request. Returns those ids.
A layer that is still blocked is not evaluated, so waiting writes no `autostart_skip` row on
@@ -242,7 +248,7 @@ def start_unblocked_layers(*, team_id: int, report_ids: list[str]) -> list[str]:
continue
evaluated.append(report_id)
try:
- async_to_sync(maybe_autostart_from_report_artefacts)(team_id=team_id, report_id=report_id)
+ async_to_sync(maybe_autostart_from_report_artefacts)(team_id=team_id, report_id=report_id, ch_user=ch_user)
except Exception:
# One layer that fails to start must not hold back its siblings.
logger.exception("signals.stack_plan.layer_autostart_failed", report_id=report_id, team_id=team_id)
@@ -254,13 +260,18 @@ def start_unblocked_layers(*, team_id: int, report_ids: list[str]) -> list[str]:
return evaluated
-def start_unblocked_layers_of_plan(*, team_id: int, parent_report_id: str) -> list[str]:
+def start_unblocked_layers_of_plan(
+ *,
+ team_id: int,
+ parent_report_id: str,
+ ch_user: ClickHouseUser = ClickHouseUser.DEFAULT,
+) -> list[str]:
"""Start the layers of a plan that can start now. A report with no layers starts nothing."""
child_ids = [
edge.source_id
for edge in incoming_links(team_id=team_id, report_id=parent_report_id, kinds=(ReportLinkKind.PART_OF,))
]
- return start_unblocked_layers(team_id=team_id, report_ids=child_ids)
+ return start_unblocked_layers(ch_user=ch_user, team_id=team_id, report_ids=child_ids)
def start_dependent_layers(*, team_id: int, report_id: str) -> list[str]:
@@ -269,7 +280,7 @@ def start_dependent_layers(*, team_id: int, report_id: str) -> list[str]:
edge.source_id
for edge in incoming_links(team_id=team_id, report_id=report_id, kinds=(ReportLinkKind.DEPENDS_ON,))
]
- return start_unblocked_layers(team_id=team_id, report_ids=dependent_ids)
+ return start_unblocked_layers(team_id=team_id, report_ids=dependent_ids, ch_user=ClickHouseUser.SIGNALS)
def schedule_dependent_layers(*, team_id: int, report_ids: list[str]) -> None:
diff --git a/products/signals/backend/tasks.py b/products/signals/backend/tasks.py
index 147a06327a22..c27c5ca7862e 100644
--- a/products/signals/backend/tasks.py
+++ b/products/signals/backend/tasks.py
@@ -12,6 +12,7 @@ from celery import shared_task
from celery.exceptions import SoftTimeLimitExceeded
from slack_sdk.errors import SlackApiError
+from posthog.clickhouse.client.connection import ClickHouseUser
from posthog.cloud_utils import get_cached_instance_license
from posthog.egress.github.transport import GitHubEgressBudgetExhausted, GitHubRateLimitError
from posthog.egress.limiter.policies import Priority
@@ -661,7 +662,7 @@ def send_reviewer_added_slack_notifications(
# them must not suppress the ping itself.
source_products: list[str] | None = None
try:
- meta = fetch_source_products_for_reports(team, [report_id]).get(report_id)
+ meta = fetch_source_products_for_reports(team, [report_id], ch_user=ClickHouseUser.SIGNALS).get(report_id)
source_products = meta.source_products if meta else None
except Exception:
logger.exception(
diff --git a/products/signals/backend/temporal/clickhouse.py b/products/signals/backend/temporal/clickhouse.py
index 1cb70d754f71..160a8f2272fa 100644
--- a/products/signals/backend/temporal/clickhouse.py
+++ b/products/signals/backend/temporal/clickhouse.py
@@ -8,6 +8,8 @@ from asgiref.sync import sync_to_async
from posthog.hogql.query import execute_hogql_query
+from posthog.clickhouse.client.connection import ClickHouseUser
+from posthog.clickhouse.workload import Workload
from posthog.errors import CH_TRANSIENT_ERRORS
from posthog.models import Team
@@ -54,6 +56,8 @@ async def execute_hogql_query_with_retry(
query=query,
team=team,
placeholders=placeholders or {},
+ ch_user=ClickHouseUser.SIGNALS,
+ workload=Workload.OFFLINE,
)
# Avoid retrying on unexpected errors, fail loudly
except RETRIABLE_ERRORS as e:
diff --git a/products/signals/backend/temporal/inbox_notification.py b/products/signals/backend/temporal/inbox_notification.py
index 6573d168b360..1e6399da5582 100644
--- a/products/signals/backend/temporal/inbox_notification.py
+++ b/products/signals/backend/temporal/inbox_notification.py
@@ -22,6 +22,8 @@ from temporalio import workflow
from temporalio.common import RetryPolicy
from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_random_exponential
+from posthog.clickhouse.client.connection import ClickHouseUser
+from posthog.clickhouse.workload import Workload
from posthog.dataclasses import frozen
from posthog.errors import CH_TRANSIENT_ERRORS
from posthog.models import Team
@@ -121,7 +123,7 @@ def _release_inbox_notification_claim(team_id: int, report_id: str) -> None:
reraise=True,
)
def _fetch_signals_for_notification(team: Team, report_id: str) -> list[dict]:
- return fetch_signals_for_report_sync(team, report_id)
+ return fetch_signals_for_report_sync(team, report_id, ch_user=ClickHouseUser.SIGNALS, workload=Workload.OFFLINE)
def _send_report_inbox_notifications(team_id: int, report_id: str) -> int:
diff --git a/products/signals/backend/temporal/report_safety_judge.py b/products/signals/backend/temporal/report_safety_judge.py
index cce8e65db872..1c3271720ca1 100644
--- a/products/signals/backend/temporal/report_safety_judge.py
+++ b/products/signals/backend/temporal/report_safety_judge.py
@@ -9,6 +9,7 @@ import temporalio
from pydantic import BaseModel, Field, model_validator
from temporalio.exceptions import ApplicationError
+from posthog.clickhouse.client.connection import ClickHouseUser
from posthog.clickhouse.query_tagging import get_query_tags
from posthog.sync import database_sync_to_async
from posthog.temporal.common.scoped import scoped_temporal
@@ -258,7 +259,7 @@ async def report_safety_judge_activity(input: SafetyJudgeInput) -> SafetyJudgeOu
"""Assess report for prompt injection attacks and store result as artefact."""
try:
signal_id = await database_sync_to_async(report_triggering_signal)(
- team_id=input.team_id, report_id=input.report_id
+ team_id=input.team_id, report_id=input.report_id, ch_user=ClickHouseUser.SIGNALS
)
except Exception:
logger.exception(
diff --git a/products/signals/backend/temporal/signal_queries.py b/products/signals/backend/temporal/signal_queries.py
index cf0fba39e9b1..64791f218d84 100644
--- a/products/signals/backend/temporal/signal_queries.py
+++ b/products/signals/backend/temporal/signal_queries.py
@@ -13,7 +13,9 @@ from posthog.hogql import ast
from posthog.hogql.query import execute_hogql_query
from posthog.api.embedding_worker import DocumentKey, async_get_recently_seen_documents
+from posthog.clickhouse.client.connection import ClickHouseUser
from posthog.clickhouse.query_tagging import Feature, Product, tag_queries
+from posthog.clickhouse.workload import Workload
from posthog.models import Team
from posthog.temporal.common.client import async_connect
from posthog.temporal.common.scoped import scoped_temporal
@@ -108,6 +110,8 @@ def soft_delete_report_signals(report_id: str, team_id: int, team: Team) -> None
query=_signals_for_report_query(include_deleted=True, limit=5000),
team=team,
placeholders=_report_placeholders(report_id),
+ ch_user=ClickHouseUser.SIGNALS,
+ workload=Workload.OFFLINE,
)
async_to_sync(submit_signal_metadata_updates)(
@@ -157,6 +161,8 @@ def retract_source_signals(*, team: Team, source_product: str, source_type: str,
"source_id": ast.Constant(value=source_id),
"after": ast.Constant(value=after),
},
+ ch_user=ClickHouseUser.SIGNALS,
+ workload=Workload.OFFLINE,
)
rows = result.results or []
async_to_sync(submit_signal_metadata_updates)(
@@ -183,6 +189,8 @@ def reassign_report_signals(*, source_report_id: str, survivor_report_id: str, t
query=_signals_for_report_query(limit=REASSIGN_SIGNAL_ROW_CAP),
team=team,
placeholders=_report_placeholders(source_report_id),
+ ch_user=ClickHouseUser.SIGNALS,
+ workload=Workload.OFFLINE,
)
signal_ids = [row[0] for row in result.results or []]
diff --git a/products/signals/backend/temporal/summary.py b/products/signals/backend/temporal/summary.py
index ce3340de2183..b425bf12f65a 100644
--- a/products/signals/backend/temporal/summary.py
+++ b/products/signals/backend/temporal/summary.py
@@ -19,6 +19,7 @@ from temporalio import workflow
from temporalio.common import RetryPolicy, WorkflowIDReusePolicy
from temporalio.workflow import ParentClosePolicy
+from posthog.clickhouse.client.connection import ClickHouseUser
from posthog.dataclasses import frozen
from posthog.event_usage import groups
from posthog.kafka_client.routing import get_producer
@@ -1170,13 +1171,16 @@ async def maybe_autostart_implementation_activity(input: MaybeAutostartImplement
user_id=input.requested_user_id,
task_id=input.requested_task_id,
after_run_count=input.requested_after_run_count,
- )
+ ),
+ ch_user=ClickHouseUser.SIGNALS,
)
else:
- await maybe_autostart_from_report_artefacts(team_id=input.team_id, report_id=input.report_id)
+ await maybe_autostart_from_report_artefacts(
+ team_id=input.team_id, report_id=input.report_id, ch_user=ClickHouseUser.SIGNALS
+ )
# A plan never starts its own run, so its first layers start here, at the same settle point.
await database_sync_to_async(start_unblocked_layers_of_plan, thread_sensitive=False)(
- team_id=input.team_id, parent_report_id=input.report_id
+ team_id=input.team_id, parent_report_id=input.report_id, ch_user=ClickHouseUser.SIGNALS
)
diff --git a/products/signals/backend/test/test_report_checks.py b/products/signals/backend/test/test_report_checks.py
index 5eafd11807fb..0b26a3735f1d 100644
--- a/products/signals/backend/test/test_report_checks.py
+++ b/products/signals/backend/test/test_report_checks.py
@@ -18,6 +18,7 @@ from rest_framework.request import Request
from rest_framework.test import APIRequestFactory, force_authenticate
from temporalio.exceptions import WorkflowAlreadyStartedError
+from posthog.clickhouse.client.connection import ClickHouseUser
from posthog.constants import AvailableFeature
from posthog.models import Organization, PropertyDefinition, Team
from posthog.models.personal_api_key import PersonalAPIKey
@@ -1446,6 +1447,7 @@ class TestReportCheckAPI(APIBaseTest):
_MEASURE, return_value=MetricMeasurement(value=0.0, measured_at=timezone.now(), series=None)
) as measure:
run_due_report_checks()
+ assert measure.call_args.kwargs["ch_user"] == ClickHouseUser.SIGNALS
measured_query = measure.call_args.args[0]
assert {key: value for key, value in measured_query["source"].items() if key != "dateRange"} == {
key: value for key, value in _PAGEVIEWS["source"].items() if key != "dateRange"
diff --git a/products/signals/backend/test/test_report_metric_refresh.py b/products/signals/backend/test/test_report_metric_refresh.py
index e28b153ddb17..ba171b96d917 100644
--- a/products/signals/backend/test/test_report_metric_refresh.py
+++ b/products/signals/backend/test/test_report_metric_refresh.py
@@ -11,6 +11,7 @@ from rest_framework import status
from posthog.schema import ChartDisplayType
+from posthog.clickhouse.client.connection import ClickHouseUser
from posthog.constants import AvailableFeature
from products.access_control.backend.facade.contracts import PropertyAccessLevel
@@ -334,14 +335,19 @@ class TestReportMetricQueryShapes(APIBaseTest):
assert run_query.call_args_list[0].kwargs["query"]["trendsFilter"]["display"] == ChartDisplayType.BOLD_NUMBER
assert run_query.call_args_list[1].kwargs["query"]["trendsFilter"]["display"] == ChartDisplayType.ACTIONS_BAR
- def test_passes_the_snapshot_cache_age_to_both_shapes(self) -> None:
+ @parameterized.expand([("web", None), ("pipeline", ClickHouseUser.SIGNALS)])
+ def test_passes_the_snapshot_cache_age_to_both_shapes(self, _name: str, ch_user: ClickHouseUser | None) -> None:
response = TrendsQueryRunResult(results=[{"aggregated_value": 2.0}], last_refresh=timezone.now())
with patch(
"products.signals.backend.report_metric_refresh.run_cached_trends_query", return_value=response
) as run_query:
- whole_window_value(_metric()["query"], self.team)
+ if ch_user is None:
+ whole_window_value(_metric()["query"], self.team)
+ else:
+ whole_window_value(_metric()["query"], self.team, ch_user=ch_user)
assert run_query.call_args.kwargs["cache_age_seconds"] == 15 * 60
+ assert run_query.call_args.kwargs["ch_user"] == (ch_user or ClickHouseUser.DEFAULT)
def test_skips_the_strip_when_the_deadline_passes_after_the_headline(self) -> None:
measured_at = timezone.now()
diff --git a/products/signals/backend/test/test_signal_queries.py b/products/signals/backend/test/test_signal_queries.py
index 929a3b4510c3..ee560f4383f6 100644
--- a/products/signals/backend/test/test_signal_queries.py
+++ b/products/signals/backend/test/test_signal_queries.py
@@ -14,7 +14,10 @@ from parameterized import parameterized
from posthog.schema import EmbeddingModelName
+from posthog.hogql.query import execute_hogql_query
+
from posthog.clickhouse.client import sync_execute
+from posthog.clickhouse.client.connection import ClickHouseUser
from products.signals.backend.facade.api import (
SignalSourceSlicePullRequest,
@@ -119,15 +122,25 @@ class TestFetchSourceProductsForReports(_SignalEmbeddingsTestBase):
# Guards the early return: an empty list would otherwise compile to `report_id IN ()` and raise.
assert fetch_source_products_for_reports(self.team, []) == {}
- def test_maps_each_report_to_its_sorted_distinct_source_products(self) -> None:
+ @parameterized.expand([("web", None), ("pipeline", ClickHouseUser.SIGNALS)])
+ def test_maps_each_report_to_its_sorted_distinct_source_products(
+ self, _name: str, ch_user: ClickHouseUser | None
+ ) -> None:
self._emit_version(document_id="d1", report_id="rA", source_product="errors", inserted_at=self.base)
self._emit_version(document_id="d2", report_id="rA", source_product="replay", inserted_at=self.base)
# duplicate source_product within a report collapses to one entry
self._emit_version(document_id="d3", report_id="rA", source_product="errors", inserted_at=self.base)
self._emit_version(document_id="d4", report_id="rB", source_product="surveys", inserted_at=self.base)
- with self.assertNumQueries(FuzzyInt(0, 1)):
- result = fetch_source_products_for_reports(self.team, ["rA", "rB"])
+ with (
+ self.assertNumQueries(FuzzyInt(0, 1)),
+ patch("products.signals.backend.signal_metadata.execute_hogql_query", wraps=execute_hogql_query) as query,
+ ):
+ result = fetch_source_products_for_reports(
+ self.team, ["rA", "rB"], **({"ch_user": ch_user} if ch_user else {})
+ )
+
+ assert query.call_args.kwargs["ch_user"] == (ch_user or ClickHouseUser.DEFAULT)
assert result == {
"rA": ReportSignalMeta(source_products=["errors", "replay"], scout_name=None),
diff --git a/products/signals/backend/test/test_spend.py b/products/signals/backend/test/test_spend.py
index f651fdf0d202..ffd57a71b6b7 100644
--- a/products/signals/backend/test/test_spend.py
+++ b/products/signals/backend/test/test_spend.py
@@ -23,6 +23,7 @@ from structlog.testing import capture_logs
from posthog.schema import EmbeddingModelName
+from posthog.clickhouse.client.connection import ClickHouseUser
from posthog.llm.gateway_client import AIGatewayConfig
from posthog.llm.gateway_usage import GatewayRequestCost
from posthog.models import Team
@@ -41,6 +42,7 @@ from products.signals.backend.spend import (
complete_task_spend,
publish_report_spend,
record_llm_spend,
+ report_triggering_signal,
settle_signal_task_spend,
signal_spend_summaries,
signal_spend_totals,
@@ -85,7 +87,7 @@ class TestSignalSpend(BaseTest):
patch(
"products.signals.backend.spend_metadata.execute_hogql_query",
side_effect=lambda **kwargs: Mock(results=rows.get(kwargs["placeholders"]["model_name"].value, [])),
- ),
+ ) as query,
):
with self.assertRaisesRegex(RuntimeError, "not yet visible"):
publish_signal_spend_metadata(signal)
@@ -157,6 +159,8 @@ class TestSignalSpend(BaseTest):
json.loads(call.kwargs["data"]["metadata"])["spend_accounting_failed_stages"] == ["summarization"]
for call in producer.produce.call_args_list
)
+ assert query.call_args_list
+ assert all(call.kwargs["ch_user"] == ClickHouseUser.SIGNALS for call in query.call_args_list)
with patch(
"products.signals.backend.signal_metadata.execute_hogql_query",
return_value=Mock(results=[(driver, "Example finding", json.dumps(metadata), now, now)]),
@@ -164,6 +168,27 @@ class TestSignalSpend(BaseTest):
signal_row = fetch_signals_for_report_sync(self.team, str(uuid4()))[0]
assert signal_row["total_spend"] == metadata["total_spend"]
+ @parameterized.expand([("web", None), ("pipeline", ClickHouseUser.SIGNALS)])
+ def test_triggering_signal_lookup_preserves_query_identity(
+ self, _name: str, ch_user: ClickHouseUser | None
+ ) -> None:
+ report = SignalReport.objects.create(team=self.team)
+ driver = str(uuid4())
+ now = timezone.now()
+ with patch(
+ "products.signals.backend.signal_metadata.execute_hogql_query",
+ return_value=Mock(results=[(driver, "Example finding", "{}", now, now)]),
+ ) as query:
+ assert (
+ report_triggering_signal(
+ team_id=self.team.id, report_id=str(report.id), **({"ch_user": ch_user} if ch_user else {})
+ )
+ == driver
+ )
+ assert query.call_args.kwargs["ch_user"] == (ch_user or ClickHouseUser.DEFAULT)
+ report.refresh_from_db()
+ assert str(report.triggering_signal_id) == driver
+
@parameterized.expand([("priced", 12_345), ("free", 0), ("cancelled", None), ("missing_id", None)])
async def test_generation_waits_for_spend_without_regenerating(self, outcome: str, cost: int | None) -> None:
driver = str(uuid4())
diff --git a/products/signals/backend/test/test_stack_plan.py b/products/signals/backend/test/test_stack_plan.py
index 3f87f4db2afb..a51e949d59a2 100644
--- a/products/signals/backend/test/test_stack_plan.py
+++ b/products/signals/backend/test/test_stack_plan.py
@@ -6,6 +6,8 @@ from django.test import SimpleTestCase
from parameterized import parameterized
+from posthog.clickhouse.client.connection import ClickHouseUser
+
from products.signals.backend.artefact_attribution import ArtefactAttribution
from products.signals.backend.artefact_schemas import (
PullRequestLink,
@@ -256,14 +258,17 @@ class TestStackPlan(BaseTest):
assert second == []
assert SignalReport.objects.filter(team=self.team).count() == 4
- def test_only_the_first_layer_starts_with_the_plan(self):
+ @parameterized.expand([("web", ClickHouseUser.DEFAULT), ("pipeline", ClickHouseUser.SIGNALS)])
+ def test_only_the_first_layer_starts_with_the_plan(self, _name: str, ch_user: ClickHouseUser):
child_ids = self._create_layers()
with patch(AUTOSTART, new_callable=AsyncMock) as autostart:
- started = start_unblocked_layers_of_plan(team_id=self.team.id, parent_report_id=str(self.parent.id))
+ started = start_unblocked_layers_of_plan(
+ team_id=self.team.id, parent_report_id=str(self.parent.id), ch_user=ch_user
+ )
assert started == [child_ids[0]]
- autostart.assert_awaited_once_with(team_id=self.team.id, report_id=child_ids[0])
+ autostart.assert_awaited_once_with(team_id=self.team.id, report_id=child_ids[0], ch_user=ch_user)
def test_a_failed_layer_start_raises_after_its_siblings_are_tried(self):
with transaction.atomic():
@@ -290,7 +295,7 @@ class TestStackPlan(BaseTest):
self._attach_pull_request(child_ids[0], 1, SignalReportPullRequest.State.OPEN)
assert start_dependent_layers(team_id=self.team.id, report_id=child_ids[0]) == [child_ids[1]]
- autostart.assert_awaited_once_with(team_id=self.team.id, report_id=child_ids[1])
+ autostart.assert_awaited_once_with(team_id=self.team.id, report_id=child_ids[1], ch_user=ClickHouseUser.SIGNALS)
def test_an_archived_layer_does_not_start_when_its_dependency_opens_a_pull_request(self):
child_ids = self._create_layers()
diff --git a/products/signals/dags/inbox_ranking/dataset/dag.py b/products/signals/dags/inbox_ranking/dataset/dag.py
index 3e84c2fa3ee9..28037f540fb6 100644
--- a/products/signals/dags/inbox_ranking/dataset/dag.py
+++ b/products/signals/dags/inbox_ranking/dataset/dag.py
@@ -70,6 +70,7 @@ import pyarrow.compute as pc
from posthog import settings
from posthog.clickhouse.client import sync_execute
+from posthog.clickhouse.client.connection import ClickHouseUser
from posthog.clickhouse.query_tagging import Feature, Product, get_query_tags, tag_queries
from posthog.dags.common import dagster_tags
from posthog.dataclasses import frozen
@@ -1048,6 +1049,7 @@ def _snapshot_report_embeddings(
},
settings=REPORT_EMBEDDINGS_QUERY_SETTINGS,
workload=etl_workload(),
+ ch_user=ClickHouseUser.SIGNALS,
)
or [],
)
@@ -1191,6 +1193,7 @@ def inbox_signal_embeddings(context: dagster.AssetExecutionContext) -> None:
},
settings=SIGNAL_EMBEDDINGS_QUERY_SETTINGS,
workload=etl_workload(),
+ ch_user=ClickHouseUser.SIGNALS,
)
or [],
)
diff --git a/products/signals/dags/inbox_ranking/dataset/queries.py b/products/signals/dags/inbox_ranking/dataset/queries.py
index 3475149b8c48..79de627344ed 100644
--- a/products/signals/dags/inbox_ranking/dataset/queries.py
+++ b/products/signals/dags/inbox_ranking/dataset/queries.py
@@ -20,7 +20,7 @@ from posthog.hogql.constants import HogQLGlobalSettings, LimitContext
from posthog.hogql.query import execute_hogql_query
from posthog import settings
-from posthog.clickhouse.client.connection import Workload
+from posthog.clickhouse.client.connection import ClickHouseUser, Workload
from posthog.cloud_utils import is_cloud
from posthog.models import Team
@@ -101,6 +101,7 @@ def hogql_rows(sql: str, *, team: Team, query_type: str, snapshot_end: datetime.
settings=HogQLGlobalSettings(max_execution_time=600),
# The dag runs without a user; the read is a trusted internal ETL over the dogfood project.
bypass_warehouse_access_control=True,
+ ch_user=ClickHouseUser.SIGNALS,
)
return [tuple(row) for row in response.results or []]
diff --git a/products/signals/dags/inbox_ranking/tests/test_dataset.py b/products/signals/dags/inbox_ranking/tests/test_dataset.py
index ceb90bbc9b23..a7ed4bebb658 100644
--- a/products/signals/dags/inbox_ranking/tests/test_dataset.py
+++ b/products/signals/dags/inbox_ranking/tests/test_dataset.py
@@ -10,6 +10,7 @@ import pyarrow as pa
from parameterized import parameterized
from social_django.models import UserSocialAuth
+from posthog.clickhouse.client.connection import ClickHouseUser
from posthog.models import Organization, Team, User
from products.event_definitions.backend.models.property_definition import PropertyDefinition
@@ -775,7 +776,11 @@ class TestImpressionsStream(ClickhouseTestMixin, BaseTest):
properties={"impressions": [{"report_id": UUID_A, "rank": 1, "source_products": ["error_tracking"]}]},
)
- rows = hogql_rows(sql, team=self.team, query_type="test", snapshot_end=SNAPSHOT_END)
+ with patch.object(queries, "execute_hogql_query", wraps=queries.execute_hogql_query) as query:
+ rows = hogql_rows(sql, team=self.team, query_type="test", snapshot_end=SNAPSHOT_END)
+ assert query.call_args.kwargs["ch_user"] == ClickHouseUser.SIGNALS
+ assert query.call_args.kwargs["settings"].max_execution_time == 600
+ assert query.call_args.kwargs["workload"] == queries.etl_workload()
assert [row[0] for row in rows] == [UUID_A]
@@ -1189,6 +1194,7 @@ def _run_embeddings_asset(monkeypatch, asset, rows, consent_team_ids=frozenset({
def fake_sync_execute(sql, params, **kwargs):
captured["params"] = params
+ captured["query_kwargs"] = kwargs
return rows
def fake_write_parquet(client, bucket, key, table, **kwargs):
@@ -1221,6 +1227,9 @@ def _run_embeddings_asset(monkeypatch, asset, rows, consent_team_ids=frozenset({
def test_each_embeddings_asset_snapshots_its_own_rendering(monkeypatch, asset, table, rendering):
written = _run_embeddings_asset(monkeypatch, asset, [(2, UUID_A, [0.5, 0.25], False, T1)])
+ assert written["query_kwargs"]["ch_user"] == ClickHouseUser.SIGNALS
+ assert written["query_kwargs"]["settings"] == dag.REPORT_EMBEDDINGS_QUERY_SETTINGS
+ assert written["query_kwargs"]["workload"] == queries.etl_workload()
assert written["params"]["rendering"] == rendering
assert written["key"] == common.partition_object_key("inbox_ranking", table, SNAPSHOT_DATE.isoformat())
assert written["table"].schema == EMBEDDINGS_SCHEMA
diff --git a/products/signals/dags/inbox_ranking/training/served.py b/products/signals/dags/inbox_ranking/training/served.py
index 5a1f8ec12ce5..382734b0bd79 100644
--- a/products/signals/dags/inbox_ranking/training/served.py
+++ b/products/signals/dags/inbox_ranking/training/served.py
@@ -34,6 +34,7 @@ from posthog.hogql.constants import HogQLGlobalSettings, LimitContext
from posthog.hogql.query import execute_hogql_query
from posthog import settings
+from posthog.clickhouse.client.connection import ClickHouseUser
from posthog.models import Team
from products.signals.backend.ranking.serving_manifest import SERVED_ROLE, model_key
@@ -102,6 +103,7 @@ def served_events(
settings=HogQLGlobalSettings(max_execution_time=600),
# The dag runs without a user; the read is a trusted internal ETL over the dogfood project.
bypass_warehouse_access_control=True,
+ ch_user=ClickHouseUser.SIGNALS,
)
return pd.DataFrame(
[(scored_at, _properties(properties)) for scored_at, properties in response.results or []],