:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(devex): pay down debt the isolation soundness checks grandfathered (#128229)
Julian Bezd1fd096555da
diff --git a/posthog/hogql/direct_sql/test/test_trino_adapter.py b/posthog/hogql/direct_sql/test/test_trino_adapter.pyindex 823bd1fb8670..dddf73789c1d 100644--- a/posthog/hogql/direct_sql/test/test_trino_adapter.py+++ b/posthog/hogql/direct_sql/test/test_trino_adapter.py@@ -175,7 +175,7 @@ def test_validate_source_config_accepts_the_teams_managed_warehouse() -> None:patch(f"{_MIXINS_MODULE}.socket.getaddrinfo", return_value=[(None, None, None, None, ("10.1.0.1", 0))]),patch(f"{_MIXINS_MODULE}._managed_warehouse_organization_id", return_value="org-1"),patch(- "products.managed_warehouse.backend.managed_ingress.get_managed_warehouse_target",+ "products.managed_warehouse.backend.facade.api.get_managed_warehouse_target",return_value=ManagedWarehouseTarget(host=host, ports=frozenset({443})),),):diff --git a/posthog/hogql/test/test_direct_duckgres_query.py b/posthog/hogql/test/test_direct_duckgres_query.pyindex 80df3eacd90e..36c462c38ddd 100644--- a/posthog/hogql/test/test_direct_duckgres_query.py+++ b/posthog/hogql/test/test_direct_duckgres_query.py@@ -1,7 +1,6 @@import socketimport threadingfrom collections.abc import Generator-from contextlib import contextmanager, nullcontextfrom uuid import UUIDfrom posthog.test.base import APIBaseTest@@ -26,10 +25,9 @@ from posthog.models import OrganizationMembership, Teamfrom products.access_control.backend.models.access_control import AccessControl-# The adapter imports these inside a method, and the CI test selector does not follow such imports.-# Module-level imports here keep an edit to the facade selecting this file.+# The adapter imports this inside a method, and the CI test selector does not follow such imports.+# A module-level import here keeps an edit to the facade selecting this file.from products.managed_warehouse.backend.facade import sql_editor as managed_warehouse_sql_editor-from products.managed_warehouse.backend.facade.contracts import ManagedWarehouseTrinoConnectionUnavailablefrom products.warehouse_sources.backend.facade.models import (MANAGED_WAREHOUSE_SOURCE_PREFIX,DataWarehouseTable,@@ -219,100 +217,6 @@ class TestDirectDuckgresQuery(APIBaseTest):self.resolve_hostaddr = resolver_patcher.start()self.addCleanup(resolver_patcher.stop)- @parameterized.expand([(True, False), (False, True)])- @patch("posthoganalytics.feature_enabled", return_value=True)- def test_managed_source_switches_raw_and_hogql_execution_to_trino(- self, raw: bool, canary: bool, _flag: MagicMock- ) -> None:- _flag.side_effect = lambda flag, *_args, **_kwargs: flag == "managed-warehouse-trino-query"- source = self._dynamic_managed_source()- source.job_inputs = {}- source.save(update_fields=["job_inputs"])- DataWarehouseTable.objects.create(- team=self.team,- external_data_source=source,- name="posthog.events",- format=DataWarehouseTable.TableFormat.Parquet,- url_pattern="direct://trino",- columns={"uuid": {"hogql": "StringDatabaseField", "clickhouse": "String", "valid": True}},- options={- "direct_trino_catalog": "org_example",- "direct_trino_schema": "posthog",- "direct_trino_table": "events",- },- )- cursor = MagicMock()- cursor.fetchmany.return_value = [("row-uuid",)]- cursor.description = [("uuid", "varchar")]- connection = MagicMock()- connection.cursor.return_value = cursor- query = 'SELECT uuid FROM "posthog"."events"' if raw else "SELECT uuid FROM posthog.events"- with (- patch(- "products.managed_warehouse.backend.logic.sql_editor_trino.connect_managed_warehouse_trino",- return_value=nullcontext(connection),- ) as connect,- patch("posthog.hogql.direct_sql.duckgres_adapter.psycopg.connect") as duckgres,- ):- response = HogQLQueryExecutor(- query=query,- team=self.team,- user=self.user,- connection_id=str(source.id),- send_raw_query=raw,- trino_canary=canary,- ).execute()- with self.assertRaisesMessage(ExposedHogQLError, "warehouse engine changed"):- HogQLQueryExecutor(- query=query,- team=self.team,- user=self.user,- connection_id=str(source.id),- send_raw_query=raw,- expected_direct_engine="postgres",- ).execute()- self.assertEqual(response.results, [("row-uuid",)])- connect.assert_called_once_with(- str(self.team.organization_id),- principal=f"posthog:sql-editor:team:{self.team.pk}:user:{self.user.pk}",- canary=canary,- )- duckgres.assert_not_called()- self.assertIn('"posthog"."events"', cursor.execute.call_args.args[0])- if not raw:- self.assertIn('"org_example"', cursor.execute.call_args.args[0])- options = self.client.get(f"/api/projects/{self.team.pk}/external_data_sources/connections/").json()- self.assertEqual(options[0]["id"], str(source.pk))- self.assertEqual(options[0]["engine"], "trino")- self.assertTrue(options[0]["is_builtin_managed_warehouse"])-- @parameterized.expand([("open",), ("enter",)])- @patch("posthoganalytics.feature_enabled", return_value=True)- def test_managed_trino_unavailable_does_not_fall_back_to_duckgres(self, stage: str, _flag: MagicMock) -> None:- source = self._dynamic_managed_source()- source.job_inputs = {}- source.save(update_fields=["job_inputs"])-- @contextmanager- def unavailable() -> Generator[None]:- if stage == "enter":- raise ManagedWarehouseTrinoConnectionUnavailable("not ready")- yield None-- with (- patch(- "products.managed_warehouse.backend.logic.sql_editor_trino.connect_managed_warehouse_trino",- side_effect=ManagedWarehouseTrinoConnectionUnavailable("not ready") if stage == "open" else None,- return_value=unavailable(),- ),- patch("posthog.hogql.direct_sql.duckgres_adapter.psycopg.connect") as duckgres,- self.assertRaisesMessage(ExposedHogQLError, "Trino is not ready"),- ):- HogQLQueryExecutor(- query="SELECT 1", team=self.team, user=self.user, connection_id=str(source.id), send_raw_query=True- ).execute()- duckgres.assert_not_called()-def _managed_source(self,*,diff --git a/posthog/hogql/test/test_direct_postgres_query.py b/posthog/hogql/test/test_direct_postgres_query.pyindex 148289eeb88c..04ca56340e49 100644--- a/posthog/hogql/test/test_direct_postgres_query.py+++ b/posthog/hogql/test/test_direct_postgres_query.py@@ -2206,7 +2206,7 @@ class TestDirectPostgresQuery(APIBaseTest):),patch(f"{_MIXINS_MODULE}._managed_warehouse_organization_id", return_value="org-1"),patch(- "products.managed_warehouse.backend.managed_ingress.get_managed_warehouse_target",+ "products.managed_warehouse.backend.facade.api.get_managed_warehouse_target",return_value=ManagedWarehouseTarget(host=host, ports=frozenset({5432})),),):diff --git a/products/alerts_platform/turbo.json b/products/alerts_platform/turbo.jsonindex 1ff7e1cd9b3f..2ed03ebce7eb 100644--- a/products/alerts_platform/turbo.json+++ b/products/alerts_platform/turbo.json@@ -6,9 +6,6 @@"backend/facade/**","backend/presentation/**","backend/routes.py",- "backend/delivery/email.py",- "backend/delivery/pagerduty.py",- "backend/delivery/team_flags.py","backend/models/**","backend/temporal/**","backend/migrations/**"diff --git a/products/dashboards/backend/api/test/test_metrics_table_widget.py b/products/dashboards/backend/api/test/test_metrics_table_widget.pyindex 27701c76dc92..41909e01f775 100644--- a/products/dashboards/backend/api/test/test_metrics_table_widget.py+++ b/products/dashboards/backend/api/test/test_metrics_table_widget.py@@ -1,4 +1,6 @@import datetime as dt+from collections.abc import Iterator+from contextlib import contextmanagerfrom typing import Anyimport pytest@@ -15,11 +17,13 @@ from rest_framework.exceptions import ValidationErrorfrom posthog.api.test.dashboards import DashboardAPIfrom posthog.clickhouse.query_tagging import get_query_tag_valuefrom posthog.models.personal_api_key import PersonalAPIKey+from posthog.models.team import Teamfrom posthog.models.utils import generate_random_token_personal, hash_key_valuefrom products.dashboards.backend.constants import METRICS_TABLE_SPARKLINE_POINTSfrom products.dashboards.backend.widget_registry import METRICS_TABLE_WIDGET_TYPE, validate_widget_config-from products.metrics.backend.facade.contracts import MetricQueryTooManyRowsError+from products.metrics.backend.facade.api import run_metric_query+from products.metrics.backend.facade.contracts import MetricQueryRequest, MetricQueryTooManyRowsError, MetricSeriesfrom products.metrics.backend.facade.testing import seed_metric, truncate_metrics_tablesQUEUE_COLUMN = {"label": "Queue depth", "metricName": "queue_depth", "aggregation": "avg"}@@ -31,6 +35,19 @@ ERRORS_COLUMN = {}+@contextmanager+def metrics_row_budget(rows: int) -> Iterator[None]:+ # A small budget at the widget's facade call lets a few seeded series go past it.+ def run_within_budget(*, team: Team, request: MetricQueryRequest) -> list[MetricSeries]:+ series = run_metric_query(team=team, request=request)+ if sum(len(line.points) for line in series) > rows:+ raise MetricQueryTooManyRowsError("query produced too many (time bucket, series) rows")+ return series++ with patch("products.dashboards.backend.widgets.metrics_table.run_metric_query", side_effect=run_within_budget):+ yield++class TestMetricsTableWidgetConfig(SimpleTestCase):def test_config_defaults_to_unconfigured(self) -> None:validated = validate_widget_config(METRICS_TABLE_WIDGET_TYPE, {})@@ -188,7 +205,6 @@ class TestMetricsTableWidgetRun(ClickhouseTestMixin, APIBaseTest):assert response.status_code == status.HTTP_400_BAD_REQUEST, response.content- @patch("products.metrics.backend.metric_query_runner._ROW_LIMIT", 100)def test_many_label_values_over_a_long_range_stay_within_the_row_budget(self) -> None:# At the default 6-hour interval, 5 routes over 14 days make 280 bucket rows, above the patched budget.for route in ("/a", "/b", "/c", "/d", "/e"):@@ -200,7 +216,8 @@ class TestMetricsTableWidgetRun(ClickhouseTestMixin, APIBaseTest):)column = {"label": "Load", "metricName": "worker_load", "aggregation": "avg"}- result = self._run({"groupBy": "route", "columns": [column], "dateRange": {"date_from": "-14d"}})+ with metrics_row_budget(100):+ result = self._run({"groupBy": "route", "columns": [column], "dateRange": {"date_from": "-14d"}})assert result["totalCount"] == 5assert all(len(row["sparklines"][0]) <= METRICS_TABLE_SPARKLINE_POINTS for row in result["rows"])@@ -254,7 +271,6 @@ class TestMetricsTableWidgetRun(ClickhouseTestMixin, APIBaseTest):assert [row["values"] for row in result["rows"]] == [[10.0, 4.0]]- @patch("products.metrics.backend.metric_query_runner._ROW_LIMIT", 60)def test_too_many_label_values_retry_with_fewer_buckets(self) -> None:for route in ("/r1", "/r2", "/r3", "/r4", "/r5"):seed_metric(@@ -265,7 +281,8 @@ class TestMetricsTableWidgetRun(ClickhouseTestMixin, APIBaseTest):)column = {"label": "Busy", "metricName": "busy_workers", "aggregation": "avg"}- result = self._run({"groupBy": "route", "columns": [column], "dateRange": {"date_from": "-1h"}})+ with metrics_row_budget(50):+ result = self._run({"groupBy": "route", "columns": [column], "dateRange": {"date_from": "-1h"}})assert result.get("queryError") is Noneassert result["totalCount"] == 5@@ -301,7 +318,7 @@ class TestMetricsTableWidgetRun(ClickhouseTestMixin, APIBaseTest):}full_buckets = self._run(config)- with patch("products.metrics.backend.metric_query_runner._ROW_LIMIT", 60):+ with metrics_row_budget(50):retried = self._run(config)assert [row["values"][0] for row in retried["rows"]] == pytest.approx(diff --git a/products/data_warehouse/backend/tests/api/test_warehouse_table_file_upload.py b/products/data_warehouse/backend/tests/api/test_warehouse_table_file_upload.pyindex 7d92b5d12a52..e54e69538196 100644--- a/products/data_warehouse/backend/tests/api/test_warehouse_table_file_upload.py+++ b/products/data_warehouse/backend/tests/api/test_warehouse_table_file_upload.py@@ -1,33 +1,22 @@import iofrom posthog.test.base import APIBaseTest-from unittest.mock import MagicMock, patch+from unittest.mock import patch-from django.conf import settingsfrom django.core.files.uploadedfile import SimpleUploadedFilefrom django.test import override_settingsfrom parameterized import parameterizedfrom rest_framework import status-from posthog.models.team import Teamfrom posthog.test.anti_ssrf import mock_anti_ssrf_dns_lookupfrom products.warehouse_sources.backend.facade.api import FILE_FORMAT_READ_HINTS-from products.warehouse_sources.backend.facade.testing import (- find_replaced_self_managed_table,- migrate_team_self_managed_tables,-)from products.warehouse_sources.backend.models.credential import DataWarehouseCredential-from products.warehouse_sources.backend.models.external_data_job import ExternalDataJob-from products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchema-from products.warehouse_sources.backend.models.external_data_source import ExternalDataSourcefrom products.warehouse_sources.backend.models.table import DataWarehouseTableVIEW_MODULE = "products.data_warehouse.backend.presentation.views.table"MODEL_MODULE = "products.warehouse_sources.backend.models.table"-SERVICE_MODULE = "products.warehouse_sources.backend.file_upload_sources"-STORAGE_MODULE = "products.warehouse_sources.backend.temporal.data_imports.sources.file_upload.storage"UPLOAD_ID = "11111111-2222-4333-8444-555555555555"NEW_UPLOAD_ID = "22222222-2222-4333-8444-555555555555"@@ -200,16 +189,10 @@ class TestCreateTableFromUpload(APIBaseTest):super().setUp()self.upload_url = f"/api/environments/{self.team.pk}/warehouse_tables/upload_file/"self.create_url = f"/api/environments/{self.team.pk}/warehouse_tables/create_from_upload/"- # Each create also makes a file upload source and starts its first sync.- patches = [- patch(f"{STORAGE_MODULE}.get_s3_client", return_value=_FakeS3()),- patch(f"{SERVICE_MODULE}.sync_connect"),- patch(f"{SERVICE_MODULE}.sync_external_data_job_workflow"),- patch(f"{SERVICE_MODULE}.trigger_ad_hoc_sync"),- ]- for p in patches:- p.start()- self.addCleanup(p.stop)+ # The file upload source behind each table has its own tests in warehouse_sources.+ source = patch(f"{VIEW_MODULE}.create_file_upload_source")+ source.start()+ self.addCleanup(source.stop)def _upload(self, filename: str = "orders.csv", file_format: str = "csv") -> str:with patch(f"{VIEW_MODULE}.get_s3_client", return_value=_FakeS3()):@@ -470,479 +453,3 @@ class TestWarehouseTableDeleteRemovesHostedFile(APIBaseTest):status_code, removed = self._delete(second)assert status_code == status.HTTP_204_NO_CONTENTassert removed == [f"test-bucket/file_uploads/team_{self.team.pk}/{upload_id}/orders.csv"]---SOURCE_VIEWS = "products.warehouse_sources.backend.presentation.views.external_data_source.base"-REPLACEMENT_COLUMNS = {- "id": {"hogql": "IntegerDatabaseField", "clickhouse": "Nullable(Int64)"},- "placed_at": {"hogql": "DateTimeDatabaseField", "clickhouse": "Nullable(DateTime64(6))"},-}---def _warehouse_api_flag(flag_key: str, *args: object, **kwargs: object) -> bool:- return flag_key == "warehouse-api"---@override_settings(DATAWAREHOUSE_BUCKET="test-bucket", DATAWAREHOUSE_BUCKET_DOMAIN="warehouse.posthog.test")-class TestFileUploadSources(APIBaseTest):- def setUp(self) -> None:- super().setUp()- self.base_url = f"/api/environments/{self.team.pk}/warehouse_tables"- self.s3 = _FakeS3()- self.trigger_count = 0- patches = [- patch("posthoganalytics.feature_enabled", return_value=False),- patch(f"{VIEW_MODULE}.get_s3_client", return_value=self.s3),- patch(f"{STORAGE_MODULE}.get_s3_client", return_value=self.s3),- patch(f"{VIEW_MODULE}.validate_data_warehouse_table_columns.delay"),- patch(f"{MODEL_MODULE}.DataWarehouseTable.detect_csv_double_quotes_setting", return_value=False),- patch(f"{SERVICE_MODULE}.sync_connect"),- patch(f"{SERVICE_MODULE}.sync_external_data_job_workflow"),- patch(f"{SERVICE_MODULE}.trigger_ad_hoc_sync", side_effect=self._trigger),- ]- mocks = [p.start() for p in patches]- for p in patches:- self.addCleanup(p.stop)- self.flag, self.schedule, self.trigger = mocks[0], mocks[6], mocks[7]- self.columns = patch(f"{MODEL_MODULE}.DataWarehouseTable.get_columns", return_value=dict(FAKE_COLUMNS))- self.get_columns = self.columns.start()- self.addCleanup(self.columns.stop)-- def _trigger(self, *_args: object, **_kwargs: object) -> MagicMock:- self.trigger_count += 1- return MagicMock(workflow_id=f"workflow-{self.trigger_count}", schedule_paused_now=False)-- def _create(self, upload_id: str = UPLOAD_ID, **payload: str):- return self.client.post(- f"{self.base_url}/create_from_upload/?include_columns=false",- data={"upload_id": upload_id, "filename": "orders.csv", "file_format": "csv", "table_name": "orders"}- | payload,- format="json",- )-- def _table_with_source(self) -> tuple[DataWarehouseTable, ExternalDataSchema]:- response = self._create()- assert response.status_code == status.HTTP_201_CREATED, response.json()- table = DataWarehouseTable.objects.get(id=response.json()["id"])- schema = ExternalDataSchema.objects.select_related("source").get(table=table)- return table, schema-- def _replace(self, table: DataWarehouseTable, upload_id: str = NEW_UPLOAD_ID, **payload: str):- return self.client.post(- f"{self.base_url}/{table.id}/replace_file/?include_columns=false",- data={"upload_id": upload_id, "filename": "orders_v2.csv", "file_format": "csv"} | payload,- format="json",- )-- def test_an_upload_becomes_a_file_upload_source_and_one_billable_sync(self) -> None:- response = self._create()-- assert response.status_code == status.HTTP_201_CREATED, response.json()- table = DataWarehouseTable.objects.get(id=response.json()["id"])- # Until the first sync imports it, the table reads the file in place.- assert (table.name, table.external_data_source_id, table.credential_id) == ("orders", None, None)- assert (- table.url_pattern- == f"https://warehouse.posthog.test/file_uploads/team_{self.team.pk}/{UPLOAD_ID}/orders.csv"- )- assert response.json()["external_schema"] is not None-- schema = ExternalDataSchema.objects.select_related("source").get(table=table)- assert schema.source.source_type == "FileUpload"- assert schema.source.job_inputs == {- "upload_id": UPLOAD_ID,- "filename": "orders.csv",- "file_format": "csv",- "schema_name": "orders",- }- assert (schema.name, schema.should_sync, schema.sync_type) == ("orders", False, "full_refresh")- assert schema.schema_metadata is not None- assert schema.schema_metadata["csv"] == {- "has_header": True,- "allow_double_quotes": False,- "column_types": {"id": "integer", "total": "integer"},- }- self.schedule.assert_called_once_with(schema, create=True, should_sync=False, trigger_immediately=False)- assert self.trigger.call_args.kwargs == {- "billable": True,- "reset_pipeline": False,- "workflow_id_prefix": "file-upload",- }-- def test_a_sync_that_fails_to_start_still_leaves_a_working_table(self) -> None:- self.trigger.side_effect = RuntimeError("temporal is down")-- response = self._create()-- assert response.status_code == status.HTTP_201_CREATED, response.json()- table = DataWarehouseTable.objects.get(id=response.json()["id"])- assert ExternalDataSchema.objects.filter(table=table, source__source_type="FileUpload").exists()-- @parameterized.expand(- [- ("traversal", "../orders.csv"),- ("other_folder", "team_1/orders.csv"),- ("hidden", ".orders.csv"),- ]- )- def test_a_filename_upload_file_could_not_have_returned_is_refused(self, _name: str, filename: str) -> None:- response = self._create(filename=filename)-- assert response.status_code == status.HTTP_400_BAD_REQUEST- assert not DataWarehouseTable.objects.filter(team=self.team).exists()-- def test_replacing_the_file_keeps_the_table_and_starts_a_billable_full_refresh(self) -> None:- table, schema = self._table_with_source()- self.get_columns.return_value = dict(REPLACEMENT_COLUMNS)-- response = self._replace(table)-- assert response.status_code == status.HTTP_202_ACCEPTED, response.json()- assert response.json()["id"] == str(table.id)- schema.refresh_from_db()- schema.source.refresh_from_db()- assert schema.source.job_inputs["upload_id"] == NEW_UPLOAD_ID- assert schema.source.job_inputs["filename"] == "orders_v2.csv"- assert schema.schema_metadata is not None- assert schema.schema_metadata["csv"]["column_types"] == {"id": "integer", "placed_at": "datetime"}- [superseded] = (schema.source.connection_metadata or {})["file_upload"]["superseded"]- assert (superseded["upload_id"], superseded["filename"]) == (UPLOAD_ID, "orders.csv")- assert self.trigger_count == 2- assert self.trigger.call_args.kwargs["billable"] is True- assert schema.sync_type == "full_refresh"- # The old file stays until a sync of the new one succeeds.- assert self.s3.removed == []- table.refresh_from_db()- assert table.name == "orders"-- def test_replacing_the_file_of_an_append_schema_switches_it_to_full_refresh(self) -> None:- table, schema = self._table_with_source()- ExternalDataSchema.objects.filter(pk=schema.pk).update(sync_type=ExternalDataSchema.SyncType.APPEND)- self.get_columns.return_value = dict(REPLACEMENT_COLUMNS)-- response = self._replace(table)-- assert response.status_code == status.HTTP_202_ACCEPTED, response.json()- schema.refresh_from_db()- assert schema.sync_type == ExternalDataSchema.SyncType.FULL_REFRESH-- @parameterized.expand(["missing_file", "unreadable_file", "sync_running", "not_a_file_upload_table"])- def test_a_replacement_that_cannot_go_ahead_changes_nothing(self, case: str) -> None:- table, schema = self._table_with_source()- if case == "missing_file":- self.s3._exists_result = False- elif case == "unreadable_file":- self.get_columns.side_effect = Exception("DB::Exception: Cannot extract table structure")- elif case == "sync_running":- ExternalDataJob.objects.create(- team=self.team, pipeline=schema.source, schema=schema, status=ExternalDataJob.Status.RUNNING- )- else:- table = DataWarehouseTable.objects.create(- team=self.team,- name="linked",- format="CSVWithNames",- url_pattern=f"https://warehouse.posthog.test/file_uploads/team_{self.team.pk}/{UPLOAD_ID}/orders.csv",- columns={},- )-- response = self._replace(table)-- assert response.status_code == status.HTTP_400_BAD_REQUEST, response.json()- schema.source.refresh_from_db()- assert schema.source.job_inputs["upload_id"] == UPLOAD_ID- assert "file_upload" not in (schema.source.connection_metadata or {})- assert self.trigger_count == 1-- def test_the_source_update_endpoint_cannot_repoint_the_upload(self) -> None:- _, schema = self._table_with_source()-- response = self.client.patch(- f"/api/environments/{self.team.pk}/external_data_sources/{schema.source_id}/",- {- "job_inputs": {- "upload_id": NEW_UPLOAD_ID,- "filename": "../../team_1/secret.csv",- "file_format": "parquet",- "schema_name": "other",- }- },- format="json",- )-- assert response.status_code == status.HTTP_200_OK, response.json()- schema.source.refresh_from_db()- assert schema.source.job_inputs == {- "upload_id": UPLOAD_ID,- "filename": "orders.csv",- "file_format": "csv",- "schema_name": "orders",- }-- def test_the_generic_source_endpoint_cannot_create_one(self) -> None:- response = self.client.post(- f"/api/environments/{self.team.pk}/external_data_sources/",- {- "source_type": "FileUpload",- "payload": {- "upload_id": UPLOAD_ID,- "filename": "orders.csv",- "file_format": "csv",- "schema_name": "orders",- },- },- format="json",- )-- assert response.status_code == status.HTTP_400_BAD_REQUEST- assert "can't be created here" in response.json()["message"]- assert not ExternalDataSource.objects.filter(team=self.team).exists()-- def test_its_schedule_cannot_be_turned_on(self) -> None:- _, schema = self._table_with_source()-- response = self.client.patch(- f"/api/environments/{self.team.pk}/external_data_schemas/{schema.id}/",- {"should_sync": True, "sync_frequency": "24hour"},- format="json",- )-- assert response.status_code == status.HTTP_400_BAD_REQUEST- assert "can't be turned on" in str(response.json())- schema.refresh_from_db()- assert schema.should_sync is False-- def test_its_table_is_deleted_through_the_source(self) -> None:- table, _ = self._table_with_source()-- response = self.client.delete(f"{self.base_url}/{table.id}")-- assert response.status_code == status.HTTP_400_BAD_REQUEST- table.refresh_from_db()- assert table.deleted is False- assert self.s3.removed == []-- @patch(f"{SOURCE_VIEWS}.delete_discover_schemas_schedule")- @patch(f"{SOURCE_VIEWS}.delete_external_data_schedule")- @patch(f"{SOURCE_VIEWS}.bulk_delete_external_data_schedules", return_value=[])- def test_deleting_the_source_deletes_the_table_and_its_files(self, *_mocks: object) -> None:- table, schema = self._table_with_source()- self.get_columns.return_value = dict(REPLACEMENT_COLUMNS)- assert self._replace(table).status_code == status.HTTP_202_ACCEPTED-- with self.captureOnCommitCallbacks(execute=True):- response = self.client.delete(f"/api/environments/{self.team.pk}/external_data_sources/{schema.source_id}")-- assert response.status_code == status.HTTP_204_NO_CONTENT- table.refresh_from_db()- assert table.deleted is True- prefix = f"{settings.DATAWAREHOUSE_BUCKET}/file_uploads/team_{self.team.pk}/"- assert sorted(self.s3.removed) == sorted(- [f"{prefix}{NEW_UPLOAD_ID}/orders_v2.csv", f"{prefix}{UPLOAD_ID}/orders.csv"]- )-- def _migrated_table(self, key: str) -> tuple[DataWarehouseTable, ExternalDataSchema]:- table = DataWarehouseTable.objects.create(- team=self.team,- name="orders",- format="CSVWithNames",- url_pattern=f"https://warehouse.posthog.test/{key.format(team=self.team.pk, upload=UPLOAD_ID)}",- columns=dict(FAKE_COLUMNS),- options={"csv_allow_double_quotes": False},- )- [outcome] = migrate_team_self_managed_tables(self.team.pk)- assert outcome.action == "linked", outcome- assert outcome.status.schema_id is not None- schema = ExternalDataSchema.objects.select_related("source").get(pk=outcome.status.schema_id)- return table, schema-- def _legacy_file_action(self, filename: str = "orders_v2.csv"):- return self.client.post(- f"{self.base_url}/file/",- {"file": SimpleUploadedFile(filename, b"id,total\n1,2\n"), "name": "orders", "format": "CSVWithNames"},- format="multipart",- )-- @parameterized.expand(- [- ("upload_folder_replace_file", "file_uploads/team_{team}/{upload}/orders.csv", "replace_file"),- ("legacy_folder_replace_file", "managed/team_{team}/orders.csv", "replace_file"),- ("legacy_folder_file_action", "managed/team_{team}/orders.csv", "file"),- ]- )- def test_a_new_file_for_a_migrated_table_starts_the_sync_that_adopts_it(- self, _name: str, key: str, path: str- ) -> None:- self.flag.side_effect = _warehouse_api_flag- table, schema = self._migrated_table(key)- old_url = table.url_pattern- self.get_columns.return_value = dict(REPLACEMENT_COLUMNS)-- with patch(f"{VIEW_MODULE}.boto3.client") as legacy_s3:- if path == "replace_file":- response = self._replace(table)- else:- response = self._legacy_file_action()-- expected_status = status.HTTP_202_ACCEPTED if path == "replace_file" else status.HTTP_201_CREATED- assert response.status_code == expected_status, response.json()- assert response.json()["id"] == str(table.id)- # The legacy action no longer writes over the file the table reads.- legacy_s3.assert_not_called()- schema.refresh_from_db()- schema.source.refresh_from_db()- job_inputs = schema.source.job_inputs- if path == "replace_file":- assert job_inputs["upload_id"] == NEW_UPLOAD_ID- else:- [written] = self.s3.written- assert written == (f"test-bucket/file_uploads/team_{self.team.pk}/{job_inputs['upload_id']}/orders_v2.csv")- assert (job_inputs["filename"], job_inputs["file_format"]) == ("orders_v2.csv", "csv")- [superseded] = (schema.source.connection_metadata or {})["file_upload"]["superseded"]- if key.startswith("managed/"):- assert superseded["legacy_filename"] == "orders.csv"- else:- assert (superseded["upload_id"], superseded["filename"]) == (UPLOAD_ID, "orders.csv")- assert schema.schema_metadata is not None- assert schema.schema_metadata["csv"]["column_types"] == {"id": "integer", "placed_at": "datetime"}- assert schema.schema_metadata["self_managed_table"]["table_id"] == str(table.id)- self.schedule.assert_called_once_with(schema, create=True, should_sync=False, trigger_immediately=False)- assert self.trigger_count == 1- assert self.trigger.call_args.kwargs["billable"] is True- # Until that sync publishes, the table reads its old file, and the sync still adopts the same row.- table.refresh_from_db()- assert (table.name, table.url_pattern, table.external_data_source_id) == ("orders", old_url, None)- assert find_replaced_self_managed_table(schema, schema.source) == table- assert self.s3.removed == []-- @parameterized.expand(["sync_running", "table_changed_since_migration"])- def test_a_migrated_table_refuses_a_new_file_it_cant_take(self, case: str) -> None:- table, schema = self._migrated_table("managed/team_{team}/orders.csv")- if case == "sync_running":- ExternalDataJob.objects.create(- team=self.team, pipeline=schema.source, schema=schema, status=ExternalDataJob.Status.RUNNING- )- else:- DataWarehouseTable.objects.filter(pk=table.pk).update(- url_pattern=f"https://warehouse.posthog.test/managed/team_{self.team.pk}/other.csv"- )-- response = self._replace(table)-- assert response.status_code == status.HTTP_400_BAD_REQUEST, response.json()- schema.source.refresh_from_db()- assert schema.source.job_inputs["upload_id"] == ""- assert "file_upload" not in (schema.source.connection_metadata or {})- self.trigger.assert_not_called()-- def test_a_migrated_source_can_be_updated_before_its_next_upload(self) -> None:- _, schema = self._migrated_table("managed/team_{team}/orders.csv")-- response = self.client.patch(- f"/api/environments/{self.team.pk}/external_data_sources/{schema.source_id}/",- {"description": "Orders", "job_inputs": {"upload_id": NEW_UPLOAD_ID, "filename": "orders_v2.csv"}},- format="json",- )-- assert response.status_code == status.HTTP_200_OK, response.json()- schema.source.refresh_from_db()- assert schema.source.description == "Orders"- assert (schema.source.job_inputs["upload_id"], schema.source.job_inputs["filename"]) == ("", "")-- def test_a_refused_legacy_file_action_deletes_the_file_it_stored(self) -> None:- self.flag.side_effect = _warehouse_api_flag- _, schema = self._migrated_table("managed/team_{team}/orders.csv")- ExternalDataJob.objects.create(- team=self.team, pipeline=schema.source, schema=schema, status=ExternalDataJob.Status.RUNNING- )- self.get_columns.return_value = dict(REPLACEMENT_COLUMNS)-- with patch(f"{VIEW_MODULE}.boto3.client"):- response = self._legacy_file_action()-- assert response.status_code == status.HTTP_400_BAD_REQUEST, response.json()- [written] = self.s3.written- assert self.s3.removed == [written]- schema.source.refresh_from_db()- assert schema.source.job_inputs["upload_id"] == ""- self.trigger.assert_not_called()-- @parameterized.expand(["reload", "resync"])- def test_a_migrated_table_cant_sync_before_its_next_upload(self, action: str) -> None:- _, schema = self._migrated_table("managed/team_{team}/orders.csv")-- response = self.client.post(f"/api/environments/{self.team.pk}/external_data_schemas/{schema.id}/{action}/")-- assert response.status_code == status.HTTP_400_BAD_REQUEST- assert "Upload the file" in response.json()["message"]- assert not ExternalDataJob.objects.filter(team=self.team).exists()- self.trigger.assert_not_called()-- @parameterized.expand(["reload", "resync"])- def test_a_table_that_imported_its_file_points_to_replace_file(self, action: str) -> None:- _, schema = self._table_with_source()- ExternalDataJob.objects.create(- team=self.team, pipeline=schema.source, schema=schema, status=ExternalDataJob.Status.COMPLETED- )- self.trigger.reset_mock()-- response = self.client.post(f"/api/environments/{self.team.pk}/external_data_schemas/{schema.id}/{action}/")-- assert response.status_code == status.HTTP_400_BAD_REQUEST- assert "Replace file" in response.json()["message"]- self.trigger.assert_not_called()-- @parameterized.expand(- [- ("linked", None),- ("pinned", "managed/team_{team}/orders.csv"),- ("pinned_in_upload_folder", "file_uploads/team_{team}/{upload}/orders.csv"),- ]- )- def test_the_schema_and_source_name_the_table_and_the_file(self, _name: str, migrated_key: str | None) -> None:- if migrated_key is None:- table, schema = self._table_with_source()- else:- table, schema = self._migrated_table(migrated_key)- assert schema.table_id is None-- schema_body = self.client.get(f"/api/environments/{self.team.pk}/external_data_schemas/{schema.id}/").json()- source_body = self.client.get(- f"/api/environments/{self.team.pk}/external_data_sources/{schema.source_id}/"- ).json()-- assert schema_body["table_id"] == str(table.id)- assert source_body["schemas"][0]["table_id"] == str(table.id)- assert source_body["file_name"] == "orders.csv"- assert schema_body["source"]["file_name"] == "orders.csv"- assert source_body["job_inputs"] == {}-- @parameterized.expand(["other_team", "deleted"])- def test_a_pinned_table_that_is_not_live_in_the_team_is_not_returned(self, case: str) -> None:- table, schema = self._migrated_table("managed/team_{team}/orders.csv")- if case == "deleted":- DataWarehouseTable.objects.filter(pk=table.pk).update(deleted=True)- else:- other_team = Team.objects.create(organization=self.organization, name="Other")- DataWarehouseTable.objects.filter(pk=table.pk).update(team=other_team)-- body = self.client.get(f"/api/environments/{self.team.pk}/external_data_schemas/{schema.id}/").json()-- assert body["table_id"] is None-- @patch(f"{SOURCE_VIEWS}.delete_discover_schemas_schedule")- @patch(f"{SOURCE_VIEWS}.delete_external_data_schedule")- @patch(f"{SOURCE_VIEWS}.bulk_delete_external_data_schedules", return_value=[])- def test_deleting_a_migrated_source_never_deletes_the_file_its_table_reads(self, *_mocks: object) -> None:- table, schema = self._migrated_table("managed/team_{team}/orders.csv")- self.get_columns.return_value = dict(REPLACEMENT_COLUMNS)- # The sync of the new file never finishes, so the table still reads the old one.- assert self._replace(table).status_code == status.HTTP_202_ACCEPTED-- with self.captureOnCommitCallbacks(execute=True):- response = self.client.delete(f"/api/environments/{self.team.pk}/external_data_sources/{schema.source_id}")-- assert response.status_code == status.HTTP_204_NO_CONTENT- table.refresh_from_db()- assert table.deleted is False- assert self.s3.removed == [f"test-bucket/file_uploads/team_{self.team.pk}/{NEW_UPLOAD_ID}/orders_v2.csv"]diff --git a/products/logs/backend/test/test_alerts_api.py b/products/logs/backend/test/test_alerts_api.pyindex ce47a85e0605..574271bd9eec 100644--- a/products/logs/backend/test/test_alerts_api.py+++ b/products/logs/backend/test/test_alerts_api.py@@ -24,6 +24,7 @@ from posthog.models.activity_logging.activity_log import ActivityLogfrom posthog.models.scoping import team_scopefrom posthog.models.team.team import Teamfrom posthog.models.user import User+from posthog.security.url_validation import PinnedUrlVerdictfrom products.alerts_platform.backend.facade import testing as platform_testingfrom products.alerts_platform.backend.facade.api import get_configuration, list_configurations, upsert_configuration@@ -76,7 +77,9 @@ def _make_log_row(*, team_id: int, service: str, uuid: str, ts: datetime, body:# Where a request-side delivery meets Temporal and the transports, which the tests stand in for._WEB_START = "products.alerts_platform.backend.temporal.web_start"_ALERTS_API = "products.logs.backend.presentation.views.alerts_api"-_NATIVE_PAGERDUTY_DELIVER = "products.alerts_platform.backend.delivery.pagerduty.PagerDutyTransport.deliver"+# A native send leaves through the pinned adapter, so a stub there counts native sends with no network.+_PINNED_URL_CHECK = "posthog.security.pinned_requests.validate_url_and_pin_ips"+_PINNED_SEND = "posthog.security.pinned_requests.PinnedIPAdapter.send"class TestLogsAlertAPI(APIBaseTest):@@ -2906,7 +2909,8 @@ class TestLogsAlertAPI(APIBaseTest):patch("posthog.permissions.posthog_feature_flag_enabled", return_value=True),# Sends the resolve inline, so the test sees which path delivered it.patch(f"{_WEB_START}.async_connect", AsyncMock(side_effect=RuntimeError("temporal unreachable"))),- patch(_NATIVE_PAGERDUTY_DELIVER, return_value=None) as native,+ patch(_PINNED_URL_CHECK, return_value=PinnedUrlVerdict(allowed=True, reason=None, pinned_ips=set())),+ patch(_PINNED_SEND, return_value=MagicMock(status_code=202)) as native,patch("products.logs.backend.platform_alert_legacy_events.emit_alert_internal_events") as emit,self.captureOnCommitCallbacks(execute=True),):diff --git a/products/logs/backend/test/test_platform_cutover.py b/products/logs/backend/test/test_platform_cutover.pyindex bcbe720512cf..71f3bf38e4da 100644--- a/products/logs/backend/test/test_platform_cutover.py+++ b/products/logs/backend/test/test_platform_cutover.py@@ -35,7 +35,7 @@ from products.logs.backend.platform_cutover import CutoverSkip, RollbackSkip, cuNOW = datetime(2026, 10, 13, 12, 0, tzinfo=UTC)State = LogsAlertConfiguration.State-LEGACY_BODY_FLAG = "products.alerts_platform.backend.delivery.team_flags.posthoganalytics.feature_enabled"+LEGACY_BODY_FLAG = "posthoganalytics.feature_enabled"STOCK_HOG = "return event"DESTINATION_DATA: dict[DestinationType, dict[str, Any]] = {DestinationType.SLACK: {"slack_workspace_id": 1, "slack_channel_id": "C-1"},diff --git a/products/managed_warehouse/backend/tests/test_managed_ingress.py b/products/managed_warehouse/backend/tests/test_managed_ingress.pynew file mode 100644index 000000000000..61f8e7f6aa45--- /dev/null+++ b/products/managed_warehouse/backend/tests/test_managed_ingress.py@@ -0,0 +1,188 @@+from unittest.mock import patch++from django.test import SimpleTestCase, override_settings++from parameterized import parameterized++from products.managed_warehouse.backend.facade.api import get_managed_warehouse_target+from products.managed_warehouse.backend.facade.contracts import ManagedWarehouseTarget++_INGRESS_MODULE = "products.managed_warehouse.backend.managed_ingress"++_NLB_A = "10.1.0.1"+_NLB_B = "10.1.0.2"+_TENANT_HOST = "acme.dw.us.postwh.com"+++def _target(host: str = _TENANT_HOST, ports: frozenset[int] = frozenset({5432})) -> ManagedWarehouseTarget:+ return ManagedWarehouseTarget(host=host, ports=ports)+++class TestManagedWarehouseTargetRead(SimpleTestCase):+ """The control-plane read: what it accepts as a ready target, and what it refuses."""++ @override_settings(MANAGED_WAREHOUSE_INGRESS_IPS=f"{_NLB_A},{_NLB_B}")+ def test_a_ready_warehouse_yields_its_host(self) -> None:+ with (+ patch(f"{_INGRESS_MODULE}.hogtower") as hogtower,+ patch(f"{_INGRESS_MODULE}.cache"),+ ):+ hogtower.is_configured.return_value = True+ hogtower.request.return_value = _response(+ {"org_id": "org-1", "state": "ready", "connection": {"host": _TENANT_HOST}}+ )+ target = get_managed_warehouse_target("org-1", source_type="postgres")++ assert target is not None+ assert target.host == _TENANT_HOST+ assert target.ports == frozenset({5432})++ @parameterized.expand(+ [+ ("provisioning", {"state": "provisioning", "connection": {"host": _TENANT_HOST}}),+ ("deleting", {"state": "deleting", "connection": {"host": _TENANT_HOST}}),+ ("failed", {"state": "failed", "connection": {"host": _TENANT_HOST}}),+ ("ready_without_connection", {"state": "ready"}),+ ("ready_with_blank_host", {"state": "ready", "connection": {"host": " "}}),+ ]+ )+ def test_a_warehouse_that_is_not_ready_yields_no_target(self, _name: str, body: dict) -> None:+ with (+ patch(f"{_INGRESS_MODULE}.hogtower") as hogtower,+ patch(f"{_INGRESS_MODULE}.cache"),+ ):+ hogtower.is_configured.return_value = True+ hogtower.request.return_value = _response({"org_id": "org-1", **body})+ assert get_managed_warehouse_target("org-1", source_type="postgres") is None++ def test_a_response_naming_another_organization_is_refused(self) -> None:+ # A control-plane bug or a misrouted proxy must never let one tenant's host answer for+ # another, which would be a cross-tenant read path.+ with (+ patch(f"{_INGRESS_MODULE}.hogtower") as hogtower,+ patch(f"{_INGRESS_MODULE}.cache"),+ patch(f"{_INGRESS_MODULE}.logger"),+ ):+ hogtower.is_configured.return_value = True+ hogtower.request.return_value = _response(+ {"org_id": "org-2", "state": "ready", "connection": {"host": "other.dw.us.postwh.com"}}+ )+ assert get_managed_warehouse_target("org-1", source_type="postgres") is None++ def test_a_source_type_the_warehouse_does_not_serve_is_not_read(self) -> None:+ # No control-plane call at all: the mapping decides before anything leaves the process.+ with (+ patch(f"{_INGRESS_MODULE}.hogtower") as hogtower,+ patch(f"{_INGRESS_MODULE}.cache"),+ ):+ assert get_managed_warehouse_target("org-1", source_type="mysql") is None+ hogtower.request.assert_not_called()++ def test_an_unconfigured_control_plane_yields_no_target(self) -> None:+ with (+ patch(f"{_INGRESS_MODULE}.hogtower") as hogtower,+ patch(f"{_INGRESS_MODULE}.cache"),+ ):+ hogtower.is_configured.return_value = False+ assert get_managed_warehouse_target("org-1", source_type="postgres") is None++ def test_a_control_plane_timeout_is_denied_and_cached_briefly(self) -> None:+ # An outage must not turn every connect into a full-timeout call to an already-sick+ # service, so a denial is cached for a short window.+ import requests++ with (+ patch(f"{_INGRESS_MODULE}.hogtower") as hogtower,+ patch(f"{_INGRESS_MODULE}.cache") as cache,+ patch(f"{_INGRESS_MODULE}.logger"),+ ):+ hogtower.is_configured.return_value = True+ hogtower.request.side_effect = requests.Timeout()+ assert get_managed_warehouse_target("org-1", source_type="postgres") is None++ assert cache.set.called++ def test_a_cached_target_is_returned_without_a_control_plane_call(self) -> None:+ cached = _target()+ with (+ patch(f"{_INGRESS_MODULE}.hogtower") as hogtower,+ patch(f"{_INGRESS_MODULE}.cache") as cache,+ ):+ cache.get.return_value = cached+ assert get_managed_warehouse_target("org-1", source_type="postgres") is cached+ hogtower.request.assert_not_called()+++def _response(body: dict):+ # The control-plane read only checks status_code and body, so a small local object stands in+ # for the translator's response type without importing the presentation layer.+ return type("Response", (), {"status_code": 200, "body": body})()+++class TestDenialCacheSurvivesSerialization(SimpleTestCase):+ """A denial must be recognised after a round trip through the cache backend.++ The cache pickles its values, so an `object()` sentinel comes back as a different object and+ an identity comparison never matches. A denial that never hits is worse than no cache: every+ denied dial, and every dial during a control-plane outage, pays the full control-plane+ timeout against an already-sick service.+ """++ def test_a_denial_is_recognised_after_a_pickle_round_trip(self) -> None:+ import pickle++ from products.managed_warehouse.backend.facade.testing import NO_MANAGED_WAREHOUSE_TARGET++ serialized = pickle.dumps(NO_MANAGED_WAREHOUSE_TARGET) # nosemgrep+ restored = pickle.loads(serialized) # nosemgrep+ assert restored == NO_MANAGED_WAREHOUSE_TARGET++ def test_a_cached_denial_is_returned_without_a_control_plane_call(self) -> None:+ from products.managed_warehouse.backend.facade.testing import NO_MANAGED_WAREHOUSE_TARGET++ with (+ patch(f"{_INGRESS_MODULE}.hogtower") as hogtower,+ patch(f"{_INGRESS_MODULE}.cache") as cache,+ ):+ cache.get.return_value = NO_MANAGED_WAREHOUSE_TARGET+ assert get_managed_warehouse_target("org-1", source_type="postgres") is None+ hogtower.request.assert_not_called()++ def test_a_cache_read_fault_denies_instead_of_raising(self) -> None:+ # The guard runs inside a connect. An exception from the cache would replace a clear+ # refusal with an opaque connection error.+ with (+ patch(f"{_INGRESS_MODULE}.cache") as cache,+ patch(f"{_INGRESS_MODULE}.hogtower") as hogtower,+ patch(f"{_INGRESS_MODULE}.logger"),+ ):+ cache.get.side_effect = RuntimeError("redis down")+ hogtower.is_configured.return_value = False+ assert get_managed_warehouse_target("org-1", source_type="postgres") is None++ def test_a_cache_write_fault_still_answers_the_dial(self) -> None:+ with (+ patch(f"{_INGRESS_MODULE}.cache") as cache,+ patch(f"{_INGRESS_MODULE}.hogtower") as hogtower,+ patch(f"{_INGRESS_MODULE}.logger"),+ ):+ cache.get.return_value = None+ cache.set.side_effect = RuntimeError("redis down")+ hogtower.is_configured.return_value = True+ hogtower.request.return_value = _response(+ {"org_id": "org-1", "state": "ready", "connection": {"host": _TENANT_HOST}}+ )+ target = get_managed_warehouse_target("org-1", source_type="postgres")++ assert target is not None+ assert target.host == _TENANT_HOST++ def test_a_control_plane_error_the_adapter_cannot_translate_denies(self) -> None:+ with (+ patch(f"{_INGRESS_MODULE}.cache"),+ patch(f"{_INGRESS_MODULE}.hogtower") as hogtower,+ patch(f"{_INGRESS_MODULE}.logger"),+ ):+ hogtower.is_configured.return_value = True+ hogtower.request.side_effect = KeyError("unmapped route")+ assert get_managed_warehouse_target("org-1", source_type="postgres") is Nonediff --git a/products/managed_warehouse/backend/tests/test_sql_editor_trino.py b/products/managed_warehouse/backend/tests/test_sql_editor_trino.pynew file mode 100644index 000000000000..d6f4e6edd139--- /dev/null+++ b/products/managed_warehouse/backend/tests/test_sql_editor_trino.py@@ -0,0 +1,139 @@+from collections.abc import Generator+from contextlib import contextmanager, nullcontext++from posthog.test.base import APIBaseTest+from unittest.mock import MagicMock, patch++from parameterized import parameterized++from posthog.hogql.errors import ExposedHogQLError+from posthog.hogql.query import HogQLQueryExecutor++from products.managed_warehouse.backend.facade.contracts import ManagedWarehouseTrinoConnectionUnavailable+from products.warehouse_sources.backend.facade.models import (+ MANAGED_WAREHOUSE_SOURCE_PREFIX,+ DataWarehouseTable,+ ExternalDataSource,+)+from products.warehouse_sources.backend.facade.types import ExternalDataSourceType+++class TestManagedTrinoSqlEditor(APIBaseTest):+ def setUp(self) -> None:+ super().setUp()+ resolver_patcher = patch(+ "posthog.hogql.direct_sql.duckgres_adapter.resolve_psycopg_hostaddr_with_timeout",+ return_value=["203.0.113.10"],+ )+ resolver_patcher.start()+ self.addCleanup(resolver_patcher.stop)++ @parameterized.expand([(True, False), (False, True)])+ @patch("posthoganalytics.feature_enabled", return_value=True)+ def test_managed_source_switches_raw_and_hogql_execution_to_trino(+ self, raw: bool, canary: bool, _flag: MagicMock+ ) -> None:+ _flag.side_effect = lambda flag, *_args, **_kwargs: flag == "managed-warehouse-trino-query"+ source = self._dynamic_managed_source()+ DataWarehouseTable.objects.create(+ team=self.team,+ external_data_source=source,+ name="posthog.events",+ format=DataWarehouseTable.TableFormat.Parquet,+ url_pattern="direct://trino",+ columns={"uuid": {"hogql": "StringDatabaseField", "clickhouse": "String", "valid": True}},+ options={+ "direct_trino_catalog": "org_example",+ "direct_trino_schema": "posthog",+ "direct_trino_table": "events",+ },+ )+ cursor = MagicMock()+ cursor.fetchmany.return_value = [("row-uuid",)]+ cursor.description = [("uuid", "varchar")]+ connection = MagicMock()+ connection.cursor.return_value = cursor+ query = 'SELECT uuid FROM "posthog"."events"' if raw else "SELECT uuid FROM posthog.events"+ with (+ patch(+ "products.managed_warehouse.backend.logic.sql_editor_trino.connect_managed_warehouse_trino",+ return_value=nullcontext(connection),+ ) as connect,+ patch("posthog.hogql.direct_sql.duckgres_adapter.psycopg.connect") as duckgres,+ ):+ response = HogQLQueryExecutor(+ query=query,+ team=self.team,+ user=self.user,+ connection_id=str(source.id),+ send_raw_query=raw,+ trino_canary=canary,+ ).execute()+ with self.assertRaisesMessage(ExposedHogQLError, "warehouse engine changed"):+ HogQLQueryExecutor(+ query=query,+ team=self.team,+ user=self.user,+ connection_id=str(source.id),+ send_raw_query=raw,+ expected_direct_engine="postgres",+ ).execute()+ self.assertEqual(response.results, [("row-uuid",)])+ connect.assert_called_once_with(+ str(self.team.organization_id),+ principal=f"posthog:sql-editor:team:{self.team.pk}:user:{self.user.pk}",+ canary=canary,+ )+ duckgres.assert_not_called()+ self.assertIn('"posthog"."events"', cursor.execute.call_args.args[0])+ if not raw:+ self.assertIn('"org_example"', cursor.execute.call_args.args[0])+ options = self.client.get(f"/api/projects/{self.team.pk}/external_data_sources/connections/").json()+ self.assertEqual(options[0]["id"], str(source.pk))+ self.assertEqual(options[0]["engine"], "trino")+ self.assertTrue(options[0]["is_builtin_managed_warehouse"])++ @parameterized.expand([("open",), ("enter",)])+ @patch("posthoganalytics.feature_enabled", return_value=True)+ def test_managed_trino_unavailable_does_not_fall_back_to_duckgres(self, stage: str, _flag: MagicMock) -> None:+ source = self._dynamic_managed_source()++ @contextmanager+ def unavailable() -> Generator[None]:+ if stage == "enter":+ raise ManagedWarehouseTrinoConnectionUnavailable("not ready")+ yield None++ with (+ patch(+ "products.managed_warehouse.backend.logic.sql_editor_trino.connect_managed_warehouse_trino",+ side_effect=ManagedWarehouseTrinoConnectionUnavailable("not ready") if stage == "open" else None,+ return_value=unavailable(),+ ),+ patch("posthog.hogql.direct_sql.duckgres_adapter.psycopg.connect") as duckgres,+ self.assertRaisesMessage(ExposedHogQLError, "Trino is not ready"),+ ):+ HogQLQueryExecutor(+ query="SELECT 1", team=self.team, user=self.user, connection_id=str(source.id), send_raw_query=True+ ).execute()+ duckgres.assert_not_called()++ def _dynamic_managed_source(self) -> ExternalDataSource:+ source_id = "managed-service-source"+ return ExternalDataSource.objects.create(+ team=self.team,+ source_id=source_id,+ connection_id=f"{source_id}-connection",+ status=ExternalDataSource.Status.COMPLETED,+ source_type=ExternalDataSourceType.POSTGRES,+ access_method=ExternalDataSource.AccessMethod.DIRECT,+ direct_query_enabled=True,+ prefix=MANAGED_WAREHOUSE_SOURCE_PREFIX,+ job_inputs={},+ connection_metadata={+ "engine": "duckdb",+ "system_managed": True,+ "credential_kind": "duckgres_service",+ "lifecycle_generation": 1,+ },+ )diff --git a/products/managed_warehouse/turbo.json b/products/managed_warehouse/turbo.jsonindex 558938f600d4..519ba6d61e4d 100644--- a/products/managed_warehouse/turbo.json+++ b/products/managed_warehouse/turbo.json@@ -17,8 +17,6 @@"backend/presentation/**","backend/temporal/**","backend/tasks/**",- "backend/logic/sql_editor_trino.py",- "backend/managed_ingress.py","backend/models.py","backend/migrations/**","backend/signals.py"diff --git a/products/metrics/turbo.json b/products/metrics/turbo.jsonindex 8bf8dcd66e50..a83fb5d3579e 100644--- a/products/metrics/turbo.json+++ b/products/metrics/turbo.json@@ -9,7 +9,6 @@"backend/hogql_queries/**","backend/tasks/**","backend/temporal/**",- "backend/metric_query_runner.py","backend/dashboard_import/receivers.py","backend/models.py","backend/migrations/**"diff --git a/products/signals/backend/facade/api.py b/products/signals/backend/facade/api.pyindex 321652a0a7ea..59e6e3c7974f 100644--- a/products/signals/backend/facade/api.py+++ b/products/signals/backend/facade/api.py@@ -31,15 +31,23 @@ from posthog.sync import database_sync_to_asyncfrom posthog.temporal.common.client import async_connectfrom products.access_control.backend.facade.user_access_control import UserAccessControl-from products.signals.backend.activity_summary import read as activity_summary_read+from products.signals.backend.activity_summary import (+ read as activity_summary_read,+ render as activity_summary_render,+ search as activity_summary_search,+)+from products.signals.backend.activity_summary.prompt import ActivitySummaryMode as ActivitySummaryModefrom products.signals.backend.activity_summary.read import (+ ActivitySummaryStatus as ActivitySummaryStatus,ActivitySummaryTheme as ActivitySummaryTheme,ActivitySummaryVersionView as ActivitySummaryVersionView,ActivitySummaryView as ActivitySummaryView,)from products.signals.backend.activity_summary.search import (ActivitySummarySearchResult as ActivitySummarySearchResult,- SearchMode,+ MatchKind as MatchKind,+ SearchMode as SearchMode,+ SemanticSearchUnavailable as SemanticSearchUnavailable,search_summaries,)from products.signals.backend.anomaly_scores import search as anomaly_search@@ -144,6 +152,11 @@ logger = structlog.get_logger(__name__)MAX_SIGNAL_DESCRIPTION_TOKENS = 8000MAX_SIGNAL_REMEDIATION_TOKENS = 16000+ActivitySummaryProductKey = activity_summary_render.Product+ACTIVITY_SUMMARY_VERSIONS_DEFAULT_LIMIT = activity_summary_read.DEFAULT_VERSIONS_LIMIT+ACTIVITY_SUMMARY_SEARCH_DEFAULT_LIMIT = activity_summary_search.DEFAULT_LIMIT+ACTIVITY_SUMMARY_SEARCH_MAX_LIMIT = activity_summary_search.MAX_LIMIT+@frozenclass InboxSummaryPullRequest:diff --git a/products/signals/backend/presentation/activity_summary.py b/products/signals/backend/presentation/activity_summary.pyindex 95b17da8b88e..b5a0dd12dc42 100644--- a/products/signals/backend/presentation/activity_summary.py+++ b/products/signals/backend/presentation/activity_summary.py@@ -24,25 +24,26 @@ from posthog.event_usage import report_user_actionfrom posthog.models import Team, Userfrom posthog.permissions import APIScopePermission, is_service_auth-from products.signals.backend.activity_summary.prompt import ActivitySummaryMode-from products.signals.backend.activity_summary.read import DEFAULT_VERSIONS_LIMIT, ActivitySummaryStatus-from products.signals.backend.activity_summary.render import Product-from products.signals.backend.activity_summary.search import (- DEFAULT_LIMIT as SEARCH_DEFAULT_LIMIT,- MAX_LIMIT as SEARCH_MAX_LIMIT,+from products.signals.backend.facade import api as signals_facade+from products.signals.backend.facade.api import (+ ACTIVITY_SUMMARY_SEARCH_DEFAULT_LIMIT,+ ACTIVITY_SUMMARY_SEARCH_MAX_LIMIT,+ ACTIVITY_SUMMARY_VERSIONS_DEFAULT_LIMIT,+ ActivitySummaryMode,+ ActivitySummaryProductKey,+ ActivitySummaryStatus,MatchKind,SearchMode,SemanticSearchUnavailable,)-from products.signals.backend.facade import api as signals_facadefrom products.signals.backend.presentation.scout_permissions import ScoutCanonicalTeamAccessPermissionREAD_EVENT = "activity summary read"-# Built from `Product`, so the API enum cannot drift from the keys that `computed.py` writes.+# Built from `ActivitySummaryProductKey`, so the API enum cannot drift from the keys that `computed.py` writes.ActivitySummaryProduct = models.TextChoices( # type: ignore[call-overload] # the stubs lack the functional API- "ActivitySummaryProduct", [(product.name, product.value) for product in Product]+ "ActivitySummaryProduct", [(product.name, product.value) for product in ActivitySummaryProductKey])@@ -208,7 +209,7 @@ class ActivitySummaryVersionSerializer(serializers.Serializer):class ActivitySummaryVersionsQuerySerializer(serializers.Serializer):limit = serializers.IntegerField(min_value=1,- default=DEFAULT_VERSIONS_LIMIT,+ default=ACTIVITY_SUMMARY_VERSIONS_DEFAULT_LIMIT,help_text="The most versions to return, newest first. Capped at the number of versions kept.",)version = serializers.IntegerField(@@ -323,8 +324,8 @@ class ActivitySummarySearchQuerySerializer(serializers.Serializer):)limit = serializers.IntegerField(min_value=1,- max_value=SEARCH_MAX_LIMIT,- default=SEARCH_DEFAULT_LIMIT,+ max_value=ACTIVITY_SUMMARY_SEARCH_MAX_LIMIT,+ default=ACTIVITY_SUMMARY_SEARCH_DEFAULT_LIMIT,help_text="The most results to return, best match first.",)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/test_managed_warehouse_exemption.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/test_managed_warehouse_exemption.pyindex 75e4d738a992..eea1f543dc6b 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/test_managed_warehouse_exemption.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/test_managed_warehouse_exemption.py@@ -20,7 +20,6 @@ from django.test import SimpleTestCase, override_settingsfrom parameterized import parameterized-from products.managed_warehouse.backend.facade.api import get_managed_warehouse_targetfrom products.managed_warehouse.backend.facade.contracts import ManagedWarehouseTargetfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.mixins import (HostNotAllowedError,@@ -31,7 +30,8 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.mix)_MIXINS_MODULE = "products.warehouse_sources.backend.temporal.data_imports.sources.common.mixins"-_INGRESS_MODULE = "products.managed_warehouse.backend.managed_ingress"+# The guard imports this facade function at the call, so a patch on the facade reaches it.+_TARGET_READ = "products.managed_warehouse.backend.facade.api.get_managed_warehouse_target"# Two addresses, because the incident observed the shared load balancer answering on both._NLB_A = "10.1.0.1"@@ -161,7 +161,7 @@ class TestManagedWarehouseExemptionInTheGuard(SimpleTestCase):patch(f"{_MIXINS_MODULE}._managed_warehouse_organization_id", return_value="org-1"),patch(f"{_MIXINS_MODULE}.logger"),patch(- f"{_INGRESS_MODULE}.get_managed_warehouse_target",+ _TARGET_READ,return_value=_target(),),):@@ -218,7 +218,7 @@ class TestManagedWarehouseExemptionInTheGuard(SimpleTestCase):patch(f"{_MIXINS_MODULE}._managed_warehouse_organization_id", return_value="org-1"),patch(f"{_MIXINS_MODULE}.logger"),patch(- f"{_INGRESS_MODULE}.get_managed_warehouse_target",+ _TARGET_READ,return_value=_target(),),):@@ -236,7 +236,7 @@ class TestManagedWarehouseExemptionInTheGuard(SimpleTestCase):patch(f"{_MIXINS_MODULE}._managed_warehouse_organization_id", return_value="org-1"),patch(f"{_MIXINS_MODULE}.logger"),patch(- f"{_INGRESS_MODULE}.get_managed_warehouse_target",+ _TARGET_READ,return_value=_target(),),):@@ -253,7 +253,7 @@ class TestManagedWarehouseExemptionInTheGuard(SimpleTestCase):patch(f"{_MIXINS_MODULE}._managed_warehouse_organization_id", return_value="org-1"),patch(f"{_MIXINS_MODULE}.logger"),patch(- f"{_INGRESS_MODULE}.get_managed_warehouse_target",+ _TARGET_READ,return_value=_target(),),):@@ -271,7 +271,7 @@ class TestManagedWarehouseExemptionInTheGuard(SimpleTestCase):patch(f"{_MIXINS_MODULE}.socket.getaddrinfo", return_value=[(None, None, None, None, (_NLB_A, 0))]),patch(f"{_MIXINS_MODULE}._managed_warehouse_organization_id", return_value="org-1"),patch(f"{_MIXINS_MODULE}.logger"),- patch(f"{_INGRESS_MODULE}.get_managed_warehouse_target", return_value=None),+ patch(_TARGET_READ, return_value=None),):resolution = resolve_safe_host(_TENANT_HOST, 999, port=5432, source_type="postgres")@@ -294,7 +294,7 @@ class TestManagedWarehouseExemptionInTheGuard(SimpleTestCase):patch(f"{_MIXINS_MODULE}.socket.getaddrinfo", return_value=[(None, None, None, None, (_NLB_A, 0))]),patch(f"{_MIXINS_MODULE}._managed_warehouse_organization_id", return_value="org-1"),patch(f"{_MIXINS_MODULE}.logger"),- patch(f"{_INGRESS_MODULE}.get_managed_warehouse_target", return_value=None),+ patch(_TARGET_READ, return_value=None),):resolution = resolve_safe_host(_TENANT_HOST, 999, port=5432, source_type="postgres")@@ -316,7 +316,7 @@ class TestManagedWarehouseExemptionInTheGuard(SimpleTestCase):patch(f"{_MIXINS_MODULE}.socket.getaddrinfo", return_value=[(None, None, None, None, (_NLB_A, 0))]),patch(f"{_MIXINS_MODULE}._managed_warehouse_organization_id", return_value="org-1"),patch(f"{_MIXINS_MODULE}.logger"),- patch(f"{_INGRESS_MODULE}.get_managed_warehouse_target", return_value=_target()),+ patch(_TARGET_READ, return_value=_target()),override_settings(MANAGED_WAREHOUSE_INGRESS_IPS=""),):resolution = resolve_safe_host(_TENANT_HOST, 999, port=5432, source_type="postgres")@@ -324,115 +324,14 @@ class TestManagedWarehouseExemptionInTheGuard(SimpleTestCase):assert resolution.connect_host is None-class TestManagedWarehouseTargetRead(SimpleTestCase):- """The control-plane read: what it accepts as a ready target, and what it refuses."""-- @override_settings(MANAGED_WAREHOUSE_INGRESS_IPS=f"{_NLB_A},{_NLB_B}")- def test_a_ready_warehouse_yields_its_host(self) -> None:- with (- patch(f"{_INGRESS_MODULE}.hogtower") as hogtower,- patch(f"{_INGRESS_MODULE}.cache"),- ):- hogtower.is_configured.return_value = True- hogtower.request.return_value = _response(- {"org_id": "org-1", "state": "ready", "connection": {"host": _TENANT_HOST}}- )- target = get_managed_warehouse_target("org-1", source_type="postgres")-- assert target is not None- assert target.host == _TENANT_HOST- assert target.ports == frozenset({5432})-- @parameterized.expand(- [- ("provisioning", {"state": "provisioning", "connection": {"host": _TENANT_HOST}}),- ("deleting", {"state": "deleting", "connection": {"host": _TENANT_HOST}}),- ("failed", {"state": "failed", "connection": {"host": _TENANT_HOST}}),- ("ready_without_connection", {"state": "ready"}),- ("ready_with_blank_host", {"state": "ready", "connection": {"host": " "}}),- ]- )- def test_a_warehouse_that_is_not_ready_yields_no_target(self, _name: str, body: dict) -> None:- with (- patch(f"{_INGRESS_MODULE}.hogtower") as hogtower,- patch(f"{_INGRESS_MODULE}.cache"),- ):- hogtower.is_configured.return_value = True- hogtower.request.return_value = _response({"org_id": "org-1", **body})- assert get_managed_warehouse_target("org-1", source_type="postgres") is None-- def test_a_response_naming_another_organization_is_refused(self) -> None:- # A control-plane bug or a misrouted proxy must never let one tenant's host answer for- # another, which would be a cross-tenant read path.- with (- patch(f"{_INGRESS_MODULE}.hogtower") as hogtower,- patch(f"{_INGRESS_MODULE}.cache"),- patch(f"{_INGRESS_MODULE}.logger"),- ):- hogtower.is_configured.return_value = True- hogtower.request.return_value = _response(- {"org_id": "org-2", "state": "ready", "connection": {"host": "other.dw.us.postwh.com"}}- )- assert get_managed_warehouse_target("org-1", source_type="postgres") is None-- def test_a_source_type_the_warehouse_does_not_serve_is_not_read(self) -> None:- # No control-plane call at all: the mapping decides before anything leaves the process.- with (- patch(f"{_INGRESS_MODULE}.hogtower") as hogtower,- patch(f"{_INGRESS_MODULE}.cache"),- ):- assert get_managed_warehouse_target("org-1", source_type="mysql") is None- hogtower.request.assert_not_called()-- def test_an_unconfigured_control_plane_yields_no_target(self) -> None:- with (- patch(f"{_INGRESS_MODULE}.hogtower") as hogtower,- patch(f"{_INGRESS_MODULE}.cache"),- ):- hogtower.is_configured.return_value = False- assert get_managed_warehouse_target("org-1", source_type="postgres") is None-- def test_a_control_plane_timeout_is_denied_and_cached_briefly(self) -> None:- # An outage must not turn every connect into a full-timeout call to an already-sick- # service, so a denial is cached for a short window.- import requests-- with (- patch(f"{_INGRESS_MODULE}.hogtower") as hogtower,- patch(f"{_INGRESS_MODULE}.cache") as cache,- patch(f"{_INGRESS_MODULE}.logger"),- ):- hogtower.is_configured.return_value = True- hogtower.request.side_effect = requests.Timeout()- assert get_managed_warehouse_target("org-1", source_type="postgres") is None-- assert cache.set.called-- def test_a_cached_target_is_returned_without_a_control_plane_call(self) -> None:- cached = _target()- with (- patch(f"{_INGRESS_MODULE}.hogtower") as hogtower,- patch(f"{_INGRESS_MODULE}.cache") as cache,- ):- cache.get.return_value = cached- assert get_managed_warehouse_target("org-1", source_type="postgres") is cached- hogtower.request.assert_not_called()---def _response(body: dict):- # The control-plane read only checks status_code and body, so a small local object stands in- # for the translator's response type without importing the presentation layer.- return type("Response", (), {"status_code": 200, "body": body})()--class TestNoImportOfTheManagedWarehouseProductAtModuleLoad(SimpleTestCase):"""The guard must not pull the control-plane read into every worker that loads it.`mixins` is imported when the Temporal worker starts. The product's Django app registry- (models, apps) loads with `django.setup()` whatever this file does, so the assertion names- the modules the guard itself could add: the presentation layer and the control-plane read.- A module-level import of those would cost every worker and would fail at startup rather- than at the one call site that needs them.+ (models, apps) loads with `django.setup()` whatever this file does, so any managed warehouse+ module that appears after that comes from the guard. The guard reaches the product only at+ the call: a module-level import of its presentation layer or its control-plane read would+ cost every worker and would fail at startup rather than at the one call site that needs them."""def test_importing_the_guard_does_not_load_the_control_plane_path(self) -> None:@@ -446,11 +345,7 @@ class TestNoImportOfTheManagedWarehouseProductAtModuleLoad(SimpleTestCase):"before = set(sys.modules)\n""from products.warehouse_sources.backend.temporal.data_imports.sources.common import mixins\n""added = set(sys.modules) - before\n"- "forbidden = sorted(m for m in added if m in {\n"- " 'products.managed_warehouse.backend.presentation.views',\n"- " 'products.managed_warehouse.backend.presentation.hogtower',\n"- " 'products.managed_warehouse.backend.managed_ingress',\n"- "})\n"+ "forbidden = sorted(m for m in added if m.startswith('products.managed_warehouse.'))\n""assert not forbidden, forbidden\n""print('clean')\n")@@ -481,7 +376,7 @@ class TestPinnedHostKwargsForAManagedWarehouse(SimpleTestCase):with (patch(f"{_MIXINS_MODULE}._managed_warehouse_organization_id", return_value="org-1"),patch(f"{_MIXINS_MODULE}.logger"),- patch(f"{_INGRESS_MODULE}.get_managed_warehouse_target", return_value=_target()),+ patch(_TARGET_READ, return_value=_target()),patch(f"{_MIXINS_MODULE}.host_lookup_is_skipped", return_value=False),patch(f"{_MIXINS_MODULE}._resolve_hostaddr_with_timeout",@@ -502,7 +397,7 @@ class TestPinnedHostKwargsForAManagedWarehouse(SimpleTestCase):with (patch(f"{_MIXINS_MODULE}._managed_warehouse_organization_id", return_value="org-1"),patch(f"{_MIXINS_MODULE}.logger"),- patch(f"{_INGRESS_MODULE}.get_managed_warehouse_target", return_value=_target()),+ patch(_TARGET_READ, return_value=_target()),patch(f"{_MIXINS_MODULE}.host_lookup_is_skipped", return_value=False),patch(f"{_MIXINS_MODULE}._resolve_hostaddr_with_timeout", return_value=[]),):@@ -510,75 +405,6 @@ class TestPinnedHostKwargsForAManagedWarehouse(SimpleTestCase):pinned_host_kwargs(_TENANT_HOST, port=5432, connect_timeout=5, team_id=999, source_type="postgres")-class TestDenialCacheSurvivesSerialization(SimpleTestCase):- """A denial must be recognised after a round trip through the cache backend.-- The cache pickles its values, so an `object()` sentinel comes back as a different object and- an identity comparison never matches. A denial that never hits is worse than no cache: every- denied dial, and every dial during a control-plane outage, pays the full control-plane- timeout against an already-sick service.- """-- def test_a_denial_is_recognised_after_a_pickle_round_trip(self) -> None:- import pickle-- from products.managed_warehouse.backend.facade.testing import NO_MANAGED_WAREHOUSE_TARGET-- serialized = pickle.dumps(NO_MANAGED_WAREHOUSE_TARGET) # nosemgrep- restored = pickle.loads(serialized) # nosemgrep- assert restored == NO_MANAGED_WAREHOUSE_TARGET-- def test_a_cached_denial_is_returned_without_a_control_plane_call(self) -> None:- from products.managed_warehouse.backend.facade.testing import NO_MANAGED_WAREHOUSE_TARGET-- with (- patch(f"{_INGRESS_MODULE}.hogtower") as hogtower,- patch(f"{_INGRESS_MODULE}.cache") as cache,- ):- cache.get.return_value = NO_MANAGED_WAREHOUSE_TARGET- assert get_managed_warehouse_target("org-1", source_type="postgres") is None- hogtower.request.assert_not_called()-- def test_a_cache_read_fault_denies_instead_of_raising(self) -> None:- # The guard runs inside a connect. An exception from the cache would replace a clear- # refusal with an opaque connection error.- with (- patch(f"{_INGRESS_MODULE}.cache") as cache,- patch(f"{_INGRESS_MODULE}.hogtower") as hogtower,- patch(f"{_INGRESS_MODULE}.logger"),- ):- cache.get.side_effect = RuntimeError("redis down")- hogtower.is_configured.return_value = False- assert get_managed_warehouse_target("org-1", source_type="postgres") is None-- def test_a_cache_write_fault_still_answers_the_dial(self) -> None:- with (- patch(f"{_INGRESS_MODULE}.cache") as cache,- patch(f"{_INGRESS_MODULE}.hogtower") as hogtower,- patch(f"{_INGRESS_MODULE}.logger"),- ):- cache.get.return_value = None- cache.set.side_effect = RuntimeError("redis down")- hogtower.is_configured.return_value = True- hogtower.request.return_value = _response(- {"org_id": "org-1", "state": "ready", "connection": {"host": _TENANT_HOST}}- )- target = get_managed_warehouse_target("org-1", source_type="postgres")-- assert target is not None- assert target.host == _TENANT_HOST-- def test_a_control_plane_error_the_adapter_cannot_translate_denies(self) -> None:- with (- patch(f"{_INGRESS_MODULE}.cache"),- patch(f"{_INGRESS_MODULE}.hogtower") as hogtower,- patch(f"{_INGRESS_MODULE}.logger"),- ):- hogtower.is_configured.return_value = True- hogtower.request.side_effect = KeyError("unmapped route")- assert get_managed_warehouse_target("org-1", source_type="postgres") is None--class TestHostComparisonIsNormalized(SimpleTestCase):"""A control-plane host with a capital or a trailing dot must still match.@@ -656,7 +482,7 @@ class TestPostgresConnectPathsAgreeWithValidation(SimpleTestCase):patch(f"{_MIXINS_MODULE}.socket.getaddrinfo", return_value=[(None, None, None, None, (_NLB_A, 0))]),patch(f"{_MIXINS_MODULE}._managed_warehouse_organization_id", return_value="org-1"),patch(f"{_MIXINS_MODULE}.logger"),- patch(f"{_INGRESS_MODULE}.get_managed_warehouse_target", return_value=_target()),+ patch(_TARGET_READ, return_value=_target()),patch("products.warehouse_sources.backend.temporal.data_imports.sources.postgres.source.get_postgres_schemas",side_effect=_no_connection,diff --git a/posthog/test/activity_logging/test_external_data_schema_activity_logging.py b/products/warehouse_sources/backend/tests/test_external_data_schema_activity_logging.pysimilarity index 100%rename from posthog/test/activity_logging/test_external_data_schema_activity_logging.pyrename to products/warehouse_sources/backend/tests/test_external_data_schema_activity_logging.pydiff --git a/products/warehouse_sources/backend/tests/test_file_upload_sources.py b/products/warehouse_sources/backend/tests/test_file_upload_sources.pynew file mode 100644index 000000000000..3ad2da187ef9--- /dev/null+++ b/products/warehouse_sources/backend/tests/test_file_upload_sources.py@@ -0,0 +1,544 @@+import io++from posthog.test.base import APIBaseTest+from unittest.mock import MagicMock, patch++from django.conf import settings+from django.core.files.uploadedfile import SimpleUploadedFile+from django.test import override_settings++from parameterized import parameterized+from rest_framework import status++from posthog.models.team import Team++from products.warehouse_sources.backend.facade.testing import (+ find_replaced_self_managed_table,+ migrate_team_self_managed_tables,+)+from products.warehouse_sources.backend.models.external_data_job import ExternalDataJob+from products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchema+from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource+from products.warehouse_sources.backend.models.table import DataWarehouseTable++VIEW_MODULE = "products.data_warehouse.backend.presentation.views.table"+MODEL_MODULE = "products.warehouse_sources.backend.models.table"+UPLOAD_ID = "11111111-2222-4333-8444-555555555555"+NEW_UPLOAD_ID = "22222222-2222-4333-8444-555555555555"++# Stand-in for what `DataWarehouseTable.get_columns()` reads off the object in place — patched so the+# create tests never reach chdb/ClickHouse, which can't introspect a fake S3 object.+FAKE_COLUMNS = {+ "id": {"hogql": "IntegerDatabaseField", "clickhouse": "Nullable(Int64)"},+ "total": {"hogql": "IntegerDatabaseField", "clickhouse": "Nullable(Int64)"},+}+++class _CapturingBuffer(io.BytesIO):+ """A BytesIO that publishes its contents to a sink on close, so the endpoint's+ context-managed write is captured once the file handle is closed."""++ def __init__(self, sink: dict[str, bytes], path: str) -> None:+ super().__init__()+ self._sink = sink+ self._path = path++ def close(self) -> None:+ self._sink[self._path] = self.getvalue()+ super().close()+++class _FakeS3:+ """Captures what the endpoint writes, standing in for the object store boundary."""++ def __init__(self, *, exists_result: bool = True) -> None:+ self.written: dict[str, bytes] = {}+ self.removed: list[str] = []+ self._exists_result = exists_result++ def open(self, path: str, mode: str) -> io.BytesIO:+ return _CapturingBuffer(self.written, path)++ def exists(self, path: str) -> bool:+ return self._exists_result++ def rm(self, path: str) -> None:+ self.removed.append(path)+++SERVICE_MODULE = "products.warehouse_sources.backend.file_upload_sources"+STORAGE_MODULE = "products.warehouse_sources.backend.temporal.data_imports.sources.file_upload.storage"+SOURCE_VIEWS = "products.warehouse_sources.backend.presentation.views.external_data_source.base"+REPLACEMENT_COLUMNS = {+ "id": {"hogql": "IntegerDatabaseField", "clickhouse": "Nullable(Int64)"},+ "placed_at": {"hogql": "DateTimeDatabaseField", "clickhouse": "Nullable(DateTime64(6))"},+}+++def _warehouse_api_flag(flag_key: str, *args: object, **kwargs: object) -> bool:+ return flag_key == "warehouse-api"+++@override_settings(DATAWAREHOUSE_BUCKET="test-bucket", DATAWAREHOUSE_BUCKET_DOMAIN="warehouse.posthog.test")+class TestFileUploadSources(APIBaseTest):+ def setUp(self) -> None:+ super().setUp()+ self.base_url = f"/api/environments/{self.team.pk}/warehouse_tables"+ self.s3 = _FakeS3()+ self.trigger_count = 0+ patches = [+ patch("posthoganalytics.feature_enabled", return_value=False),+ patch(f"{VIEW_MODULE}.get_s3_client", return_value=self.s3),+ patch(f"{STORAGE_MODULE}.get_s3_client", return_value=self.s3),+ patch(f"{VIEW_MODULE}.validate_data_warehouse_table_columns.delay"),+ patch(f"{MODEL_MODULE}.DataWarehouseTable.detect_csv_double_quotes_setting", return_value=False),+ patch(f"{SERVICE_MODULE}.sync_connect"),+ patch(f"{SERVICE_MODULE}.sync_external_data_job_workflow"),+ patch(f"{SERVICE_MODULE}.trigger_ad_hoc_sync", side_effect=self._trigger),+ ]+ mocks = [p.start() for p in patches]+ for p in patches:+ self.addCleanup(p.stop)+ self.flag, self.schedule, self.trigger = mocks[0], mocks[6], mocks[7]+ self.columns = patch(f"{MODEL_MODULE}.DataWarehouseTable.get_columns", return_value=dict(FAKE_COLUMNS))+ self.get_columns = self.columns.start()+ self.addCleanup(self.columns.stop)++ def _trigger(self, *_args: object, **_kwargs: object) -> MagicMock:+ self.trigger_count += 1+ return MagicMock(workflow_id=f"workflow-{self.trigger_count}", schedule_paused_now=False)++ def _create(self, upload_id: str = UPLOAD_ID, **payload: str):+ return self.client.post(+ f"{self.base_url}/create_from_upload/?include_columns=false",+ data={"upload_id": upload_id, "filename": "orders.csv", "file_format": "csv", "table_name": "orders"}+ | payload,+ format="json",+ )++ def _table_with_source(self) -> tuple[DataWarehouseTable, ExternalDataSchema]:+ response = self._create()+ assert response.status_code == status.HTTP_201_CREATED, response.json()+ table = DataWarehouseTable.objects.get(id=response.json()["id"])+ schema = ExternalDataSchema.objects.select_related("source").get(table=table)+ return table, schema++ def _replace(self, table: DataWarehouseTable, upload_id: str = NEW_UPLOAD_ID, **payload: str):+ return self.client.post(+ f"{self.base_url}/{table.id}/replace_file/?include_columns=false",+ data={"upload_id": upload_id, "filename": "orders_v2.csv", "file_format": "csv"} | payload,+ format="json",+ )++ def test_an_upload_becomes_a_file_upload_source_and_one_billable_sync(self) -> None:+ response = self._create()++ assert response.status_code == status.HTTP_201_CREATED, response.json()+ table = DataWarehouseTable.objects.get(id=response.json()["id"])+ # Until the first sync imports it, the table reads the file in place.+ assert (table.name, table.external_data_source_id, table.credential_id) == ("orders", None, None)+ assert (+ table.url_pattern+ == f"https://warehouse.posthog.test/file_uploads/team_{self.team.pk}/{UPLOAD_ID}/orders.csv"+ )+ assert response.json()["external_schema"] is not None++ schema = ExternalDataSchema.objects.select_related("source").get(table=table)+ assert schema.source.source_type == "FileUpload"+ assert schema.source.job_inputs == {+ "upload_id": UPLOAD_ID,+ "filename": "orders.csv",+ "file_format": "csv",+ "schema_name": "orders",+ }+ assert (schema.name, schema.should_sync, schema.sync_type) == ("orders", False, "full_refresh")+ assert schema.schema_metadata is not None+ assert schema.schema_metadata["csv"] == {+ "has_header": True,+ "allow_double_quotes": False,+ "column_types": {"id": "integer", "total": "integer"},+ }+ self.schedule.assert_called_once_with(schema, create=True, should_sync=False, trigger_immediately=False)+ assert self.trigger.call_args.kwargs == {+ "billable": True,+ "reset_pipeline": False,+ "workflow_id_prefix": "file-upload",+ }++ def test_a_sync_that_fails_to_start_still_leaves_a_working_table(self) -> None:+ self.trigger.side_effect = RuntimeError("temporal is down")++ response = self._create()++ assert response.status_code == status.HTTP_201_CREATED, response.json()+ table = DataWarehouseTable.objects.get(id=response.json()["id"])+ assert ExternalDataSchema.objects.filter(table=table, source__source_type="FileUpload").exists()++ @parameterized.expand(+ [+ ("traversal", "../orders.csv"),+ ("other_folder", "team_1/orders.csv"),+ ("hidden", ".orders.csv"),+ ]+ )+ def test_a_filename_upload_file_could_not_have_returned_is_refused(self, _name: str, filename: str) -> None:+ response = self._create(filename=filename)++ assert response.status_code == status.HTTP_400_BAD_REQUEST+ assert not DataWarehouseTable.objects.filter(team=self.team).exists()++ def test_replacing_the_file_keeps_the_table_and_starts_a_billable_full_refresh(self) -> None:+ table, schema = self._table_with_source()+ self.get_columns.return_value = dict(REPLACEMENT_COLUMNS)++ response = self._replace(table)++ assert response.status_code == status.HTTP_202_ACCEPTED, response.json()+ assert response.json()["id"] == str(table.id)+ schema.refresh_from_db()+ schema.source.refresh_from_db()+ assert schema.source.job_inputs["upload_id"] == NEW_UPLOAD_ID+ assert schema.source.job_inputs["filename"] == "orders_v2.csv"+ assert schema.schema_metadata is not None+ assert schema.schema_metadata["csv"]["column_types"] == {"id": "integer", "placed_at": "datetime"}+ [superseded] = (schema.source.connection_metadata or {})["file_upload"]["superseded"]+ assert (superseded["upload_id"], superseded["filename"]) == (UPLOAD_ID, "orders.csv")+ assert self.trigger_count == 2+ assert self.trigger.call_args.kwargs["billable"] is True+ assert schema.sync_type == "full_refresh"+ # The old file stays until a sync of the new one succeeds.+ assert self.s3.removed == []+ table.refresh_from_db()+ assert table.name == "orders"++ def test_replacing_the_file_of_an_append_schema_switches_it_to_full_refresh(self) -> None:+ table, schema = self._table_with_source()+ ExternalDataSchema.objects.filter(pk=schema.pk).update(sync_type=ExternalDataSchema.SyncType.APPEND)+ self.get_columns.return_value = dict(REPLACEMENT_COLUMNS)++ response = self._replace(table)++ assert response.status_code == status.HTTP_202_ACCEPTED, response.json()+ schema.refresh_from_db()+ assert schema.sync_type == ExternalDataSchema.SyncType.FULL_REFRESH++ @parameterized.expand(["missing_file", "unreadable_file", "sync_running", "not_a_file_upload_table"])+ def test_a_replacement_that_cannot_go_ahead_changes_nothing(self, case: str) -> None:+ table, schema = self._table_with_source()+ if case == "missing_file":+ self.s3._exists_result = False+ elif case == "unreadable_file":+ self.get_columns.side_effect = Exception("DB::Exception: Cannot extract table structure")+ elif case == "sync_running":+ ExternalDataJob.objects.create(+ team=self.team, pipeline=schema.source, schema=schema, status=ExternalDataJob.Status.RUNNING+ )+ else:+ table = DataWarehouseTable.objects.create(+ team=self.team,+ name="linked",+ format="CSVWithNames",+ url_pattern=f"https://warehouse.posthog.test/file_uploads/team_{self.team.pk}/{UPLOAD_ID}/orders.csv",+ columns={},+ )++ response = self._replace(table)++ assert response.status_code == status.HTTP_400_BAD_REQUEST, response.json()+ schema.source.refresh_from_db()+ assert schema.source.job_inputs["upload_id"] == UPLOAD_ID+ assert "file_upload" not in (schema.source.connection_metadata or {})+ assert self.trigger_count == 1++ def test_the_source_update_endpoint_cannot_repoint_the_upload(self) -> None:+ _, schema = self._table_with_source()++ response = self.client.patch(+ f"/api/environments/{self.team.pk}/external_data_sources/{schema.source_id}/",+ {+ "job_inputs": {+ "upload_id": NEW_UPLOAD_ID,+ "filename": "../../team_1/secret.csv",+ "file_format": "parquet",+ "schema_name": "other",+ }+ },+ format="json",+ )++ assert response.status_code == status.HTTP_200_OK, response.json()+ schema.source.refresh_from_db()+ assert schema.source.job_inputs == {+ "upload_id": UPLOAD_ID,+ "filename": "orders.csv",+ "file_format": "csv",+ "schema_name": "orders",+ }++ def test_the_generic_source_endpoint_cannot_create_one(self) -> None:+ response = self.client.post(+ f"/api/environments/{self.team.pk}/external_data_sources/",+ {+ "source_type": "FileUpload",+ "payload": {+ "upload_id": UPLOAD_ID,+ "filename": "orders.csv",+ "file_format": "csv",+ "schema_name": "orders",+ },+ },+ format="json",+ )++ assert response.status_code == status.HTTP_400_BAD_REQUEST+ assert "can't be created here" in response.json()["message"]+ assert not ExternalDataSource.objects.filter(team=self.team).exists()++ def test_its_schedule_cannot_be_turned_on(self) -> None:+ _, schema = self._table_with_source()++ response = self.client.patch(+ f"/api/environments/{self.team.pk}/external_data_schemas/{schema.id}/",+ {"should_sync": True, "sync_frequency": "24hour"},+ format="json",+ )++ assert response.status_code == status.HTTP_400_BAD_REQUEST+ assert "can't be turned on" in str(response.json())+ schema.refresh_from_db()+ assert schema.should_sync is False++ def test_its_table_is_deleted_through_the_source(self) -> None:+ table, _ = self._table_with_source()++ response = self.client.delete(f"{self.base_url}/{table.id}")++ assert response.status_code == status.HTTP_400_BAD_REQUEST+ table.refresh_from_db()+ assert table.deleted is False+ assert self.s3.removed == []++ @patch(f"{SOURCE_VIEWS}.delete_discover_schemas_schedule")+ @patch(f"{SOURCE_VIEWS}.delete_external_data_schedule")+ @patch(f"{SOURCE_VIEWS}.bulk_delete_external_data_schedules", return_value=[])+ def test_deleting_the_source_deletes_the_table_and_its_files(self, *_mocks: object) -> None:+ table, schema = self._table_with_source()+ self.get_columns.return_value = dict(REPLACEMENT_COLUMNS)+ assert self._replace(table).status_code == status.HTTP_202_ACCEPTED++ with self.captureOnCommitCallbacks(execute=True):+ response = self.client.delete(f"/api/environments/{self.team.pk}/external_data_sources/{schema.source_id}")++ assert response.status_code == status.HTTP_204_NO_CONTENT+ table.refresh_from_db()+ assert table.deleted is True+ prefix = f"{settings.DATAWAREHOUSE_BUCKET}/file_uploads/team_{self.team.pk}/"+ assert sorted(self.s3.removed) == sorted(+ [f"{prefix}{NEW_UPLOAD_ID}/orders_v2.csv", f"{prefix}{UPLOAD_ID}/orders.csv"]+ )++ def _migrated_table(self, key: str) -> tuple[DataWarehouseTable, ExternalDataSchema]:+ table = DataWarehouseTable.objects.create(+ team=self.team,+ name="orders",+ format="CSVWithNames",+ url_pattern=f"https://warehouse.posthog.test/{key.format(team=self.team.pk, upload=UPLOAD_ID)}",+ columns=dict(FAKE_COLUMNS),+ options={"csv_allow_double_quotes": False},+ )+ [outcome] = migrate_team_self_managed_tables(self.team.pk)+ assert outcome.action == "linked", outcome+ assert outcome.status.schema_id is not None+ schema = ExternalDataSchema.objects.select_related("source").get(pk=outcome.status.schema_id)+ return table, schema++ def _legacy_file_action(self, filename: str = "orders_v2.csv"):+ return self.client.post(+ f"{self.base_url}/file/",+ {"file": SimpleUploadedFile(filename, b"id,total\n1,2\n"), "name": "orders", "format": "CSVWithNames"},+ format="multipart",+ )++ @parameterized.expand(+ [+ ("upload_folder_replace_file", "file_uploads/team_{team}/{upload}/orders.csv", "replace_file"),+ ("legacy_folder_replace_file", "managed/team_{team}/orders.csv", "replace_file"),+ ("legacy_folder_file_action", "managed/team_{team}/orders.csv", "file"),+ ]+ )+ def test_a_new_file_for_a_migrated_table_starts_the_sync_that_adopts_it(+ self, _name: str, key: str, path: str+ ) -> None:+ self.flag.side_effect = _warehouse_api_flag+ table, schema = self._migrated_table(key)+ old_url = table.url_pattern+ self.get_columns.return_value = dict(REPLACEMENT_COLUMNS)++ with patch(f"{VIEW_MODULE}.boto3.client") as legacy_s3:+ if path == "replace_file":+ response = self._replace(table)+ else:+ response = self._legacy_file_action()++ expected_status = status.HTTP_202_ACCEPTED if path == "replace_file" else status.HTTP_201_CREATED+ assert response.status_code == expected_status, response.json()+ assert response.json()["id"] == str(table.id)+ # The legacy action no longer writes over the file the table reads.+ legacy_s3.assert_not_called()+ schema.refresh_from_db()+ schema.source.refresh_from_db()+ job_inputs = schema.source.job_inputs+ if path == "replace_file":+ assert job_inputs["upload_id"] == NEW_UPLOAD_ID+ else:+ [written] = self.s3.written+ assert written == (f"test-bucket/file_uploads/team_{self.team.pk}/{job_inputs['upload_id']}/orders_v2.csv")+ assert (job_inputs["filename"], job_inputs["file_format"]) == ("orders_v2.csv", "csv")+ [superseded] = (schema.source.connection_metadata or {})["file_upload"]["superseded"]+ if key.startswith("managed/"):+ assert superseded["legacy_filename"] == "orders.csv"+ else:+ assert (superseded["upload_id"], superseded["filename"]) == (UPLOAD_ID, "orders.csv")+ assert schema.schema_metadata is not None+ assert schema.schema_metadata["csv"]["column_types"] == {"id": "integer", "placed_at": "datetime"}+ assert schema.schema_metadata["self_managed_table"]["table_id"] == str(table.id)+ self.schedule.assert_called_once_with(schema, create=True, should_sync=False, trigger_immediately=False)+ assert self.trigger_count == 1+ assert self.trigger.call_args.kwargs["billable"] is True+ # Until that sync publishes, the table reads its old file, and the sync still adopts the same row.+ table.refresh_from_db()+ assert (table.name, table.url_pattern, table.external_data_source_id) == ("orders", old_url, None)+ assert find_replaced_self_managed_table(schema, schema.source) == table+ assert self.s3.removed == []++ @parameterized.expand(["sync_running", "table_changed_since_migration"])+ def test_a_migrated_table_refuses_a_new_file_it_cant_take(self, case: str) -> None:+ table, schema = self._migrated_table("managed/team_{team}/orders.csv")+ if case == "sync_running":+ ExternalDataJob.objects.create(+ team=self.team, pipeline=schema.source, schema=schema, status=ExternalDataJob.Status.RUNNING+ )+ else:+ DataWarehouseTable.objects.filter(pk=table.pk).update(+ url_pattern=f"https://warehouse.posthog.test/managed/team_{self.team.pk}/other.csv"+ )++ response = self._replace(table)++ assert response.status_code == status.HTTP_400_BAD_REQUEST, response.json()+ schema.source.refresh_from_db()+ assert schema.source.job_inputs["upload_id"] == ""+ assert "file_upload" not in (schema.source.connection_metadata or {})+ self.trigger.assert_not_called()++ def test_a_migrated_source_can_be_updated_before_its_next_upload(self) -> None:+ _, schema = self._migrated_table("managed/team_{team}/orders.csv")++ response = self.client.patch(+ f"/api/environments/{self.team.pk}/external_data_sources/{schema.source_id}/",+ {"description": "Orders", "job_inputs": {"upload_id": NEW_UPLOAD_ID, "filename": "orders_v2.csv"}},+ format="json",+ )++ assert response.status_code == status.HTTP_200_OK, response.json()+ schema.source.refresh_from_db()+ assert schema.source.description == "Orders"+ assert (schema.source.job_inputs["upload_id"], schema.source.job_inputs["filename"]) == ("", "")++ def test_a_refused_legacy_file_action_deletes_the_file_it_stored(self) -> None:+ self.flag.side_effect = _warehouse_api_flag+ _, schema = self._migrated_table("managed/team_{team}/orders.csv")+ ExternalDataJob.objects.create(+ team=self.team, pipeline=schema.source, schema=schema, status=ExternalDataJob.Status.RUNNING+ )+ self.get_columns.return_value = dict(REPLACEMENT_COLUMNS)++ with patch(f"{VIEW_MODULE}.boto3.client"):+ response = self._legacy_file_action()++ assert response.status_code == status.HTTP_400_BAD_REQUEST, response.json()+ [written] = self.s3.written+ assert self.s3.removed == [written]+ schema.source.refresh_from_db()+ assert schema.source.job_inputs["upload_id"] == ""+ self.trigger.assert_not_called()++ @parameterized.expand(["reload", "resync"])+ def test_a_migrated_table_cant_sync_before_its_next_upload(self, action: str) -> None:+ _, schema = self._migrated_table("managed/team_{team}/orders.csv")++ response = self.client.post(f"/api/environments/{self.team.pk}/external_data_schemas/{schema.id}/{action}/")++ assert response.status_code == status.HTTP_400_BAD_REQUEST+ assert "Upload the file" in response.json()["message"]+ assert not ExternalDataJob.objects.filter(team=self.team).exists()+ self.trigger.assert_not_called()++ @parameterized.expand(["reload", "resync"])+ def test_a_table_that_imported_its_file_points_to_replace_file(self, action: str) -> None:+ _, schema = self._table_with_source()+ ExternalDataJob.objects.create(+ team=self.team, pipeline=schema.source, schema=schema, status=ExternalDataJob.Status.COMPLETED+ )+ self.trigger.reset_mock()++ response = self.client.post(f"/api/environments/{self.team.pk}/external_data_schemas/{schema.id}/{action}/")++ assert response.status_code == status.HTTP_400_BAD_REQUEST+ assert "Replace file" in response.json()["message"]+ self.trigger.assert_not_called()++ @parameterized.expand(+ [+ ("linked", None),+ ("pinned", "managed/team_{team}/orders.csv"),+ ("pinned_in_upload_folder", "file_uploads/team_{team}/{upload}/orders.csv"),+ ]+ )+ def test_the_schema_and_source_name_the_table_and_the_file(self, _name: str, migrated_key: str | None) -> None:+ if migrated_key is None:+ table, schema = self._table_with_source()+ else:+ table, schema = self._migrated_table(migrated_key)+ assert schema.table_id is None++ schema_body = self.client.get(f"/api/environments/{self.team.pk}/external_data_schemas/{schema.id}/").json()+ source_body = self.client.get(+ f"/api/environments/{self.team.pk}/external_data_sources/{schema.source_id}/"+ ).json()++ assert schema_body["table_id"] == str(table.id)+ assert source_body["schemas"][0]["table_id"] == str(table.id)+ assert source_body["file_name"] == "orders.csv"+ assert schema_body["source"]["file_name"] == "orders.csv"+ assert source_body["job_inputs"] == {}++ @parameterized.expand(["other_team", "deleted"])+ def test_a_pinned_table_that_is_not_live_in_the_team_is_not_returned(self, case: str) -> None:+ table, schema = self._migrated_table("managed/team_{team}/orders.csv")+ if case == "deleted":+ DataWarehouseTable.objects.filter(pk=table.pk).update(deleted=True)+ else:+ other_team = Team.objects.create(organization=self.organization, name="Other")+ DataWarehouseTable.objects.filter(pk=table.pk).update(team=other_team)++ body = self.client.get(f"/api/environments/{self.team.pk}/external_data_schemas/{schema.id}/").json()++ assert body["table_id"] is None++ @patch(f"{SOURCE_VIEWS}.delete_discover_schemas_schedule")+ @patch(f"{SOURCE_VIEWS}.delete_external_data_schedule")+ @patch(f"{SOURCE_VIEWS}.bulk_delete_external_data_schedules", return_value=[])+ def test_deleting_a_migrated_source_never_deletes_the_file_its_table_reads(self, *_mocks: object) -> None:+ table, schema = self._migrated_table("managed/team_{team}/orders.csv")+ self.get_columns.return_value = dict(REPLACEMENT_COLUMNS)+ # The sync of the new file never finishes, so the table still reads the old one.+ assert self._replace(table).status_code == status.HTTP_202_ACCEPTED++ with self.captureOnCommitCallbacks(execute=True):+ response = self.client.delete(f"/api/environments/{self.team.pk}/external_data_sources/{schema.source_id}")++ assert response.status_code == status.HTTP_204_NO_CONTENT+ table.refresh_from_db()+ assert table.deleted is False+ assert self.s3.removed == [f"test-bucket/file_uploads/team_{self.team.pk}/{NEW_UPLOAD_ID}/orders_v2.csv"]diff --git a/products/warehouse_sources/backend/tests/test_source_credentials.py b/products/warehouse_sources/backend/tests/test_source_credentials.pyindex 69c543f41e96..d3c9a69f7f1a 100644--- a/products/warehouse_sources/backend/tests/test_source_credentials.py+++ b/products/warehouse_sources/backend/tests/test_source_credentials.py@@ -41,7 +41,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.job)from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistryfrom products.warehouse_sources.backend.types import ExternalDataSourceType-from products.workflows.backend.facade.contracts import WorkflowSummary+from products.workflows.backend.facade.testing import create_workflow_for_testPG_FIELDS: dict[str, Any] = {"host": "db.example.com",@@ -374,25 +374,27 @@ class TestWarehouseSourceCredentialsStore(BaseTest):pk = self._create(source, PG_FIELDS)if other_user == "source":self._referencing(pk)- workflows = (- [WorkflowSummary(id="wf1", name="Nightly sync", status="active")] if other_user == "workflow" else []- )+ if other_user == "workflow":+ create_workflow_for_test(+ team_id=self.team.pk,+ name="Nightly sync",+ status="active",+ actions=[+ {"id": "query", "type": "function", "config": {"inputs": {"db": {"value": {"integrationId": pk}}}}}+ ],+ )- with patch(- "products.workflows.backend.integration_usages.get_active_workflows_using_integration",- return_value=workflows,- ):- if outcome == "conflict":- with self.assertRaises(SharedCredentialsError) as raised:- WarehouseSourceCredentialsStore.update(source, new_fields, allow_shared=allow_shared)- assert raised.exception.integration_id == pk- assert [usage.type for usage in raised.exception.usages] == [- "workflow" if other_user == "workflow" else "warehouse_source"- ]- assert source.credentials_integration_id == str(pk)- assert WarehouseSourceCredentialsStore.get(source) == PG_FIELDS- return- reference = WarehouseSourceCredentialsStore.update(source, new_fields, allow_shared=allow_shared)+ if outcome == "conflict":+ with self.assertRaises(SharedCredentialsError) as raised:+ WarehouseSourceCredentialsStore.update(source, new_fields, allow_shared=allow_shared)+ assert raised.exception.integration_id == pk+ assert [usage.type for usage in raised.exception.usages] == [+ "workflow" if other_user == "workflow" else "warehouse_source"+ ]+ assert source.credentials_integration_id == str(pk)+ assert WarehouseSourceCredentialsStore.get(source) == PG_FIELDS+ return+ reference = WarehouseSourceCredentialsStore.update(source, new_fields, allow_shared=allow_shared)new_pk = reference.legacy_idassert new_pk is not Nonediff --git a/products/warehouse_sources/backend/tests/test_sync_alerts.py b/products/warehouse_sources/backend/tests/test_sync_alerts.pyindex 40eb0ec97dcb..aed3ab76d478 100644--- a/products/warehouse_sources/backend/tests/test_sync_alerts.py+++ b/products/warehouse_sources/backend/tests/test_sync_alerts.py@@ -5,12 +5,14 @@ from uuid import UUIDfrom posthog.test.base import APIBaseTestfrom unittest.mock import patch+from django.core import mailfrom django.utils import timezonefrom parameterized import parameterizedfrom rest_framework import statusfrom posthog.constants import AvailableFeature+from posthog.models.instance_setting import set_instance_settingfrom posthog.models.organization import Organization, OrganizationMembershipfrom posthog.models.scoping import team_scopefrom posthog.models.team.team import Team@@ -720,13 +722,13 @@ class TestWarehouseAlerts(APIBaseTest):record_outcomes(self.team.id, list(evaluation.outcomes), self.cutoff)(delivery,) = evaluation.deliveries- with patch("products.alerts_platform.backend.delivery.email.EmailMessage") as email_message:+ set_instance_setting("EMAIL_HOST", "localhost")+ with self.settings(EMAIL_ENABLED=True):sent = platform_testing.deliver(delivery)assert sent == 1assert delivery.configuration_id == str(configuration_id)- assert "Sync failures" in email_message.call_args.kwargs["subject"]- assert [call.kwargs["email"] for call in email_message.return_value.add_recipient.call_args_list] == sorted(+ assert all("Sync failures" in message.subject for message in mail.outbox)+ assert sorted(address for message in mail.outbox for address in message.to) == sorted([self.user.email, "[email protected]"])- email_message.return_value.send.assert_called_once_with()diff --git a/products/warehouse_sources/turbo.json b/products/warehouse_sources/turbo.jsonindex fe1a9ba0cf2a..42edd16156d5 100644--- a/products/warehouse_sources/turbo.json+++ b/products/warehouse_sources/turbo.json@@ -11,8 +11,6 @@"backend/models/**","backend/migrations/**","backend/tasks/**",- "backend/activity_logging.py",- "backend/file_upload_sources.py","backend/temporal/data_imports/naming_convention.py","backend/temporal/data_imports/pipelines/core/account_property_row_sink.py","backend/temporal/data_imports/pipelines/core/consts.py",@@ -35,7 +33,6 @@"backend/temporal/data_imports/sources/bigquery/**","backend/temporal/data_imports/sources/clickhouse/**","backend/temporal/data_imports/sources/common/**",- "backend/temporal/data_imports/sources/file_upload/storage.py","backend/temporal/data_imports/sources/google_search_console/**","backend/temporal/data_imports/sources/motherduck/**","backend/temporal/data_imports/sources/mssql/**",diff --git a/pyproject.toml b/pyproject.tomlindex 489107af1f00..e897f3eae303 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -820,14 +820,6 @@ ignore_imports = ["products.actions.backend.presentation.views.action -> products.actions.backend.models.selector_match_change","products.actions.backend.presentation.views.action -> products.cdp.backend.models.hog_functions.hog_function","products.actions.backend.presentation.views.action -> products.cohorts.backend.models.cohort",- # TODO: signals presentation wave. The activity summary viewset reads its modes, limits and- # render types from backend/activity_summary/ directly. That directory has no __init__.py, so- # grimp cannot see these edges and only the AST surface check in `hogli product:lint` holds- # them. Expose what the view needs through the facade and delete each entry.- "products.signals.backend.presentation.activity_summary -> products.signals.backend.activity_summary.prompt",- "products.signals.backend.presentation.activity_summary -> products.signals.backend.activity_summary.read",- "products.signals.backend.presentation.activity_summary -> products.signals.backend.activity_summary.render",- "products.signals.backend.presentation.activity_summary -> products.signals.backend.activity_summary.search",]# routes.py is the one module core reads to assemble a product's API router, so it is the