Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(devex): pay down debt the isolation soundness checks grandfathered (#128229)

Julian Bezd1fd096555da
diff --git a/posthog/hogql/direct_sql/test/test_trino_adapter.py b/posthog/hogql/direct_sql/test/test_trino_adapter.py
index 823bd1fb8670..dddf73789c1d 100644
--- a/posthog/hogql/direct_sql/test/test_trino_adapter.py
+++ b/posthog/hogql/direct_sql/test/test_trino_adapter.py
@@ -175,7 +175,7 @@ def test_validate_source_config_accepts_the_teams_managed_warehouse() -> None:
patch(f"{_MIXINS_MODULE}.socket.getaddrinfo", return_value=[(None, None, None, None, ("10.1.0.1", 0))]),
patch(f"{_MIXINS_MODULE}._managed_warehouse_organization_id", return_value="org-1"),
patch(
- "products.managed_warehouse.backend.managed_ingress.get_managed_warehouse_target",
+ "products.managed_warehouse.backend.facade.api.get_managed_warehouse_target",
return_value=ManagedWarehouseTarget(host=host, ports=frozenset({443})),
),
):
diff --git a/posthog/hogql/test/test_direct_duckgres_query.py b/posthog/hogql/test/test_direct_duckgres_query.py
index 80df3eacd90e..36c462c38ddd 100644
--- a/posthog/hogql/test/test_direct_duckgres_query.py
+++ b/posthog/hogql/test/test_direct_duckgres_query.py
@@ -1,7 +1,6 @@
import socket
import threading
from collections.abc import Generator
-from contextlib import contextmanager, nullcontext
from uuid import UUID
from posthog.test.base import APIBaseTest
@@ -26,10 +25,9 @@ from posthog.models import OrganizationMembership, Team
from products.access_control.backend.models.access_control import AccessControl
-# The adapter imports these inside a method, and the CI test selector does not follow such imports.
-# Module-level imports here keep an edit to the facade selecting this file.
+# The adapter imports this inside a method, and the CI test selector does not follow such imports.
+# A module-level import here keeps an edit to the facade selecting this file.
from products.managed_warehouse.backend.facade import sql_editor as managed_warehouse_sql_editor
-from products.managed_warehouse.backend.facade.contracts import ManagedWarehouseTrinoConnectionUnavailable
from products.warehouse_sources.backend.facade.models import (
MANAGED_WAREHOUSE_SOURCE_PREFIX,
DataWarehouseTable,
@@ -219,100 +217,6 @@ class TestDirectDuckgresQuery(APIBaseTest):
self.resolve_hostaddr = resolver_patcher.start()
self.addCleanup(resolver_patcher.stop)
- @parameterized.expand([(True, False), (False, True)])
- @patch("posthoganalytics.feature_enabled", return_value=True)
- def test_managed_source_switches_raw_and_hogql_execution_to_trino(
- self, raw: bool, canary: bool, _flag: MagicMock
- ) -> None:
- _flag.side_effect = lambda flag, *_args, **_kwargs: flag == "managed-warehouse-trino-query"
- source = self._dynamic_managed_source()
- source.job_inputs = {}
- source.save(update_fields=["job_inputs"])
- DataWarehouseTable.objects.create(
- team=self.team,
- external_data_source=source,
- name="posthog.events",
- format=DataWarehouseTable.TableFormat.Parquet,
- url_pattern="direct://trino",
- columns={"uuid": {"hogql": "StringDatabaseField", "clickhouse": "String", "valid": True}},
- options={
- "direct_trino_catalog": "org_example",
- "direct_trino_schema": "posthog",
- "direct_trino_table": "events",
- },
- )
- cursor = MagicMock()
- cursor.fetchmany.return_value = [("row-uuid",)]
- cursor.description = [("uuid", "varchar")]
- connection = MagicMock()
- connection.cursor.return_value = cursor
- query = 'SELECT uuid FROM "posthog"."events"' if raw else "SELECT uuid FROM posthog.events"
- with (
- patch(
- "products.managed_warehouse.backend.logic.sql_editor_trino.connect_managed_warehouse_trino",
- return_value=nullcontext(connection),
- ) as connect,
- patch("posthog.hogql.direct_sql.duckgres_adapter.psycopg.connect") as duckgres,
- ):
- response = HogQLQueryExecutor(
- query=query,
- team=self.team,
- user=self.user,
- connection_id=str(source.id),
- send_raw_query=raw,
- trino_canary=canary,
- ).execute()
- with self.assertRaisesMessage(ExposedHogQLError, "warehouse engine changed"):
- HogQLQueryExecutor(
- query=query,
- team=self.team,
- user=self.user,
- connection_id=str(source.id),
- send_raw_query=raw,
- expected_direct_engine="postgres",
- ).execute()
- self.assertEqual(response.results, [("row-uuid",)])
- connect.assert_called_once_with(
- str(self.team.organization_id),
- principal=f"posthog:sql-editor:team:{self.team.pk}:user:{self.user.pk}",
- canary=canary,
- )
- duckgres.assert_not_called()
- self.assertIn('"posthog"."events"', cursor.execute.call_args.args[0])
- if not raw:
- self.assertIn('"org_example"', cursor.execute.call_args.args[0])
- options = self.client.get(f"/api/projects/{self.team.pk}/external_data_sources/connections/").json()
- self.assertEqual(options[0]["id"], str(source.pk))
- self.assertEqual(options[0]["engine"], "trino")
- self.assertTrue(options[0]["is_builtin_managed_warehouse"])
-
- @parameterized.expand([("open",), ("enter",)])
- @patch("posthoganalytics.feature_enabled", return_value=True)
- def test_managed_trino_unavailable_does_not_fall_back_to_duckgres(self, stage: str, _flag: MagicMock) -> None:
- source = self._dynamic_managed_source()
- source.job_inputs = {}
- source.save(update_fields=["job_inputs"])
-
- @contextmanager
- def unavailable() -> Generator[None]:
- if stage == "enter":
- raise ManagedWarehouseTrinoConnectionUnavailable("not ready")
- yield None
-
- with (
- patch(
- "products.managed_warehouse.backend.logic.sql_editor_trino.connect_managed_warehouse_trino",
- side_effect=ManagedWarehouseTrinoConnectionUnavailable("not ready") if stage == "open" else None,
- return_value=unavailable(),
- ),
- patch("posthog.hogql.direct_sql.duckgres_adapter.psycopg.connect") as duckgres,
- self.assertRaisesMessage(ExposedHogQLError, "Trino is not ready"),
- ):
- HogQLQueryExecutor(
- query="SELECT 1", team=self.team, user=self.user, connection_id=str(source.id), send_raw_query=True
- ).execute()
- duckgres.assert_not_called()
-
def _managed_source(
self,
*,
diff --git a/posthog/hogql/test/test_direct_postgres_query.py b/posthog/hogql/test/test_direct_postgres_query.py
index 148289eeb88c..04ca56340e49 100644
--- a/posthog/hogql/test/test_direct_postgres_query.py
+++ b/posthog/hogql/test/test_direct_postgres_query.py
@@ -2206,7 +2206,7 @@ class TestDirectPostgresQuery(APIBaseTest):
),
patch(f"{_MIXINS_MODULE}._managed_warehouse_organization_id", return_value="org-1"),
patch(
- "products.managed_warehouse.backend.managed_ingress.get_managed_warehouse_target",
+ "products.managed_warehouse.backend.facade.api.get_managed_warehouse_target",
return_value=ManagedWarehouseTarget(host=host, ports=frozenset({5432})),
),
):
diff --git a/products/alerts_platform/turbo.json b/products/alerts_platform/turbo.json
index 1ff7e1cd9b3f..2ed03ebce7eb 100644
--- a/products/alerts_platform/turbo.json
+++ b/products/alerts_platform/turbo.json
@@ -6,9 +6,6 @@
"backend/facade/**",
"backend/presentation/**",
"backend/routes.py",
- "backend/delivery/email.py",
- "backend/delivery/pagerduty.py",
- "backend/delivery/team_flags.py",
"backend/models/**",
"backend/temporal/**",
"backend/migrations/**"
diff --git a/products/dashboards/backend/api/test/test_metrics_table_widget.py b/products/dashboards/backend/api/test/test_metrics_table_widget.py
index 27701c76dc92..41909e01f775 100644
--- a/products/dashboards/backend/api/test/test_metrics_table_widget.py
+++ b/products/dashboards/backend/api/test/test_metrics_table_widget.py
@@ -1,4 +1,6 @@
import datetime as dt
+from collections.abc import Iterator
+from contextlib import contextmanager
from typing import Any
import pytest
@@ -15,11 +17,13 @@ from rest_framework.exceptions import ValidationError
from posthog.api.test.dashboards import DashboardAPI
from posthog.clickhouse.query_tagging import get_query_tag_value
from posthog.models.personal_api_key import PersonalAPIKey
+from posthog.models.team import Team
from posthog.models.utils import generate_random_token_personal, hash_key_value
from products.dashboards.backend.constants import METRICS_TABLE_SPARKLINE_POINTS
from products.dashboards.backend.widget_registry import METRICS_TABLE_WIDGET_TYPE, validate_widget_config
-from products.metrics.backend.facade.contracts import MetricQueryTooManyRowsError
+from products.metrics.backend.facade.api import run_metric_query
+from products.metrics.backend.facade.contracts import MetricQueryRequest, MetricQueryTooManyRowsError, MetricSeries
from products.metrics.backend.facade.testing import seed_metric, truncate_metrics_tables
QUEUE_COLUMN = {"label": "Queue depth", "metricName": "queue_depth", "aggregation": "avg"}
@@ -31,6 +35,19 @@ ERRORS_COLUMN = {
}
+@contextmanager
+def metrics_row_budget(rows: int) -> Iterator[None]:
+ # A small budget at the widget's facade call lets a few seeded series go past it.
+ def run_within_budget(*, team: Team, request: MetricQueryRequest) -> list[MetricSeries]:
+ series = run_metric_query(team=team, request=request)
+ if sum(len(line.points) for line in series) > rows:
+ raise MetricQueryTooManyRowsError("query produced too many (time bucket, series) rows")
+ return series
+
+ with patch("products.dashboards.backend.widgets.metrics_table.run_metric_query", side_effect=run_within_budget):
+ yield
+
+
class TestMetricsTableWidgetConfig(SimpleTestCase):
def test_config_defaults_to_unconfigured(self) -> None:
validated = validate_widget_config(METRICS_TABLE_WIDGET_TYPE, {})
@@ -188,7 +205,6 @@ class TestMetricsTableWidgetRun(ClickhouseTestMixin, APIBaseTest):
assert response.status_code == status.HTTP_400_BAD_REQUEST, response.content
- @patch("products.metrics.backend.metric_query_runner._ROW_LIMIT", 100)
def test_many_label_values_over_a_long_range_stay_within_the_row_budget(self) -> None:
# At the default 6-hour interval, 5 routes over 14 days make 280 bucket rows, above the patched budget.
for route in ("/a", "/b", "/c", "/d", "/e"):
@@ -200,7 +216,8 @@ class TestMetricsTableWidgetRun(ClickhouseTestMixin, APIBaseTest):
)
column = {"label": "Load", "metricName": "worker_load", "aggregation": "avg"}
- result = self._run({"groupBy": "route", "columns": [column], "dateRange": {"date_from": "-14d"}})
+ with metrics_row_budget(100):
+ result = self._run({"groupBy": "route", "columns": [column], "dateRange": {"date_from": "-14d"}})
assert result["totalCount"] == 5
assert all(len(row["sparklines"][0]) <= METRICS_TABLE_SPARKLINE_POINTS for row in result["rows"])
@@ -254,7 +271,6 @@ class TestMetricsTableWidgetRun(ClickhouseTestMixin, APIBaseTest):
assert [row["values"] for row in result["rows"]] == [[10.0, 4.0]]
- @patch("products.metrics.backend.metric_query_runner._ROW_LIMIT", 60)
def test_too_many_label_values_retry_with_fewer_buckets(self) -> None:
for route in ("/r1", "/r2", "/r3", "/r4", "/r5"):
seed_metric(
@@ -265,7 +281,8 @@ class TestMetricsTableWidgetRun(ClickhouseTestMixin, APIBaseTest):
)
column = {"label": "Busy", "metricName": "busy_workers", "aggregation": "avg"}
- result = self._run({"groupBy": "route", "columns": [column], "dateRange": {"date_from": "-1h"}})
+ with metrics_row_budget(50):
+ result = self._run({"groupBy": "route", "columns": [column], "dateRange": {"date_from": "-1h"}})
assert result.get("queryError") is None
assert result["totalCount"] == 5
@@ -301,7 +318,7 @@ class TestMetricsTableWidgetRun(ClickhouseTestMixin, APIBaseTest):
}
full_buckets = self._run(config)
- with patch("products.metrics.backend.metric_query_runner._ROW_LIMIT", 60):
+ with metrics_row_budget(50):
retried = self._run(config)
assert [row["values"][0] for row in retried["rows"]] == pytest.approx(
diff --git a/products/data_warehouse/backend/tests/api/test_warehouse_table_file_upload.py b/products/data_warehouse/backend/tests/api/test_warehouse_table_file_upload.py
index 7d92b5d12a52..e54e69538196 100644
--- a/products/data_warehouse/backend/tests/api/test_warehouse_table_file_upload.py
+++ b/products/data_warehouse/backend/tests/api/test_warehouse_table_file_upload.py
@@ -1,33 +1,22 @@
import io
from posthog.test.base import APIBaseTest
-from unittest.mock import MagicMock, patch
+from unittest.mock import patch
-from django.conf import settings
from django.core.files.uploadedfile import SimpleUploadedFile
from django.test import override_settings
from parameterized import parameterized
from rest_framework import status
-from posthog.models.team import Team
from posthog.test.anti_ssrf import mock_anti_ssrf_dns_lookup
from products.warehouse_sources.backend.facade.api import FILE_FORMAT_READ_HINTS
-from products.warehouse_sources.backend.facade.testing import (
- find_replaced_self_managed_table,
- migrate_team_self_managed_tables,
-)
from products.warehouse_sources.backend.models.credential import DataWarehouseCredential
-from products.warehouse_sources.backend.models.external_data_job import ExternalDataJob
-from products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchema
-from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource
from products.warehouse_sources.backend.models.table import DataWarehouseTable
VIEW_MODULE = "products.data_warehouse.backend.presentation.views.table"
MODEL_MODULE = "products.warehouse_sources.backend.models.table"
-SERVICE_MODULE = "products.warehouse_sources.backend.file_upload_sources"
-STORAGE_MODULE = "products.warehouse_sources.backend.temporal.data_imports.sources.file_upload.storage"
UPLOAD_ID = "11111111-2222-4333-8444-555555555555"
NEW_UPLOAD_ID = "22222222-2222-4333-8444-555555555555"
@@ -200,16 +189,10 @@ class TestCreateTableFromUpload(APIBaseTest):
super().setUp()
self.upload_url = f"/api/environments/{self.team.pk}/warehouse_tables/upload_file/"
self.create_url = f"/api/environments/{self.team.pk}/warehouse_tables/create_from_upload/"
- # Each create also makes a file upload source and starts its first sync.
- patches = [
- patch(f"{STORAGE_MODULE}.get_s3_client", return_value=_FakeS3()),
- patch(f"{SERVICE_MODULE}.sync_connect"),
- patch(f"{SERVICE_MODULE}.sync_external_data_job_workflow"),
- patch(f"{SERVICE_MODULE}.trigger_ad_hoc_sync"),
- ]
- for p in patches:
- p.start()
- self.addCleanup(p.stop)
+ # The file upload source behind each table has its own tests in warehouse_sources.
+ source = patch(f"{VIEW_MODULE}.create_file_upload_source")
+ source.start()
+ self.addCleanup(source.stop)
def _upload(self, filename: str = "orders.csv", file_format: str = "csv") -> str:
with patch(f"{VIEW_MODULE}.get_s3_client", return_value=_FakeS3()):
@@ -470,479 +453,3 @@ class TestWarehouseTableDeleteRemovesHostedFile(APIBaseTest):
status_code, removed = self._delete(second)
assert status_code == status.HTTP_204_NO_CONTENT
assert removed == [f"test-bucket/file_uploads/team_{self.team.pk}/{upload_id}/orders.csv"]
-
-
-SOURCE_VIEWS = "products.warehouse_sources.backend.presentation.views.external_data_source.base"
-REPLACEMENT_COLUMNS = {
- "id": {"hogql": "IntegerDatabaseField", "clickhouse": "Nullable(Int64)"},
- "placed_at": {"hogql": "DateTimeDatabaseField", "clickhouse": "Nullable(DateTime64(6))"},
-}
-
-
-def _warehouse_api_flag(flag_key: str, *args: object, **kwargs: object) -> bool:
- return flag_key == "warehouse-api"
-
-
-@override_settings(DATAWAREHOUSE_BUCKET="test-bucket", DATAWAREHOUSE_BUCKET_DOMAIN="warehouse.posthog.test")
-class TestFileUploadSources(APIBaseTest):
- def setUp(self) -> None:
- super().setUp()
- self.base_url = f"/api/environments/{self.team.pk}/warehouse_tables"
- self.s3 = _FakeS3()
- self.trigger_count = 0
- patches = [
- patch("posthoganalytics.feature_enabled", return_value=False),
- patch(f"{VIEW_MODULE}.get_s3_client", return_value=self.s3),
- patch(f"{STORAGE_MODULE}.get_s3_client", return_value=self.s3),
- patch(f"{VIEW_MODULE}.validate_data_warehouse_table_columns.delay"),
- patch(f"{MODEL_MODULE}.DataWarehouseTable.detect_csv_double_quotes_setting", return_value=False),
- patch(f"{SERVICE_MODULE}.sync_connect"),
- patch(f"{SERVICE_MODULE}.sync_external_data_job_workflow"),
- patch(f"{SERVICE_MODULE}.trigger_ad_hoc_sync", side_effect=self._trigger),
- ]
- mocks = [p.start() for p in patches]
- for p in patches:
- self.addCleanup(p.stop)
- self.flag, self.schedule, self.trigger = mocks[0], mocks[6], mocks[7]
- self.columns = patch(f"{MODEL_MODULE}.DataWarehouseTable.get_columns", return_value=dict(FAKE_COLUMNS))
- self.get_columns = self.columns.start()
- self.addCleanup(self.columns.stop)
-
- def _trigger(self, *_args: object, **_kwargs: object) -> MagicMock:
- self.trigger_count += 1
- return MagicMock(workflow_id=f"workflow-{self.trigger_count}", schedule_paused_now=False)
-
- def _create(self, upload_id: str = UPLOAD_ID, **payload: str):
- return self.client.post(
- f"{self.base_url}/create_from_upload/?include_columns=false",
- data={"upload_id": upload_id, "filename": "orders.csv", "file_format": "csv", "table_name": "orders"}
- | payload,
- format="json",
- )
-
- def _table_with_source(self) -> tuple[DataWarehouseTable, ExternalDataSchema]:
- response = self._create()
- assert response.status_code == status.HTTP_201_CREATED, response.json()
- table = DataWarehouseTable.objects.get(id=response.json()["id"])
- schema = ExternalDataSchema.objects.select_related("source").get(table=table)
- return table, schema
-
- def _replace(self, table: DataWarehouseTable, upload_id: str = NEW_UPLOAD_ID, **payload: str):
- return self.client.post(
- f"{self.base_url}/{table.id}/replace_file/?include_columns=false",
- data={"upload_id": upload_id, "filename": "orders_v2.csv", "file_format": "csv"} | payload,
- format="json",
- )
-
- def test_an_upload_becomes_a_file_upload_source_and_one_billable_sync(self) -> None:
- response = self._create()
-
- assert response.status_code == status.HTTP_201_CREATED, response.json()
- table = DataWarehouseTable.objects.get(id=response.json()["id"])
- # Until the first sync imports it, the table reads the file in place.
- assert (table.name, table.external_data_source_id, table.credential_id) == ("orders", None, None)
- assert (
- table.url_pattern
- == f"https://warehouse.posthog.test/file_uploads/team_{self.team.pk}/{UPLOAD_ID}/orders.csv"
- )
- assert response.json()["external_schema"] is not None
-
- schema = ExternalDataSchema.objects.select_related("source").get(table=table)
- assert schema.source.source_type == "FileUpload"
- assert schema.source.job_inputs == {
- "upload_id": UPLOAD_ID,
- "filename": "orders.csv",
- "file_format": "csv",
- "schema_name": "orders",
- }
- assert (schema.name, schema.should_sync, schema.sync_type) == ("orders", False, "full_refresh")
- assert schema.schema_metadata is not None
- assert schema.schema_metadata["csv"] == {
- "has_header": True,
- "allow_double_quotes": False,
- "column_types": {"id": "integer", "total": "integer"},
- }
- self.schedule.assert_called_once_with(schema, create=True, should_sync=False, trigger_immediately=False)
- assert self.trigger.call_args.kwargs == {
- "billable": True,
- "reset_pipeline": False,
- "workflow_id_prefix": "file-upload",
- }
-
- def test_a_sync_that_fails_to_start_still_leaves_a_working_table(self) -> None:
- self.trigger.side_effect = RuntimeError("temporal is down")
-
- response = self._create()
-
- assert response.status_code == status.HTTP_201_CREATED, response.json()
- table = DataWarehouseTable.objects.get(id=response.json()["id"])
- assert ExternalDataSchema.objects.filter(table=table, source__source_type="FileUpload").exists()
-
- @parameterized.expand(
- [
- ("traversal", "../orders.csv"),
- ("other_folder", "team_1/orders.csv"),
- ("hidden", ".orders.csv"),
- ]
- )
- def test_a_filename_upload_file_could_not_have_returned_is_refused(self, _name: str, filename: str) -> None:
- response = self._create(filename=filename)
-
- assert response.status_code == status.HTTP_400_BAD_REQUEST
- assert not DataWarehouseTable.objects.filter(team=self.team).exists()
-
- def test_replacing_the_file_keeps_the_table_and_starts_a_billable_full_refresh(self) -> None:
- table, schema = self._table_with_source()
- self.get_columns.return_value = dict(REPLACEMENT_COLUMNS)
-
- response = self._replace(table)
-
- assert response.status_code == status.HTTP_202_ACCEPTED, response.json()
- assert response.json()["id"] == str(table.id)
- schema.refresh_from_db()
- schema.source.refresh_from_db()
- assert schema.source.job_inputs["upload_id"] == NEW_UPLOAD_ID
- assert schema.source.job_inputs["filename"] == "orders_v2.csv"
- assert schema.schema_metadata is not None
- assert schema.schema_metadata["csv"]["column_types"] == {"id": "integer", "placed_at": "datetime"}
- [superseded] = (schema.source.connection_metadata or {})["file_upload"]["superseded"]
- assert (superseded["upload_id"], superseded["filename"]) == (UPLOAD_ID, "orders.csv")
- assert self.trigger_count == 2
- assert self.trigger.call_args.kwargs["billable"] is True
- assert schema.sync_type == "full_refresh"
- # The old file stays until a sync of the new one succeeds.
- assert self.s3.removed == []
- table.refresh_from_db()
- assert table.name == "orders"
-
- def test_replacing_the_file_of_an_append_schema_switches_it_to_full_refresh(self) -> None:
- table, schema = self._table_with_source()
- ExternalDataSchema.objects.filter(pk=schema.pk).update(sync_type=ExternalDataSchema.SyncType.APPEND)
- self.get_columns.return_value = dict(REPLACEMENT_COLUMNS)
-
- response = self._replace(table)
-
- assert response.status_code == status.HTTP_202_ACCEPTED, response.json()
- schema.refresh_from_db()
- assert schema.sync_type == ExternalDataSchema.SyncType.FULL_REFRESH
-
- @parameterized.expand(["missing_file", "unreadable_file", "sync_running", "not_a_file_upload_table"])
- def test_a_replacement_that_cannot_go_ahead_changes_nothing(self, case: str) -> None:
- table, schema = self._table_with_source()
- if case == "missing_file":
- self.s3._exists_result = False
- elif case == "unreadable_file":
- self.get_columns.side_effect = Exception("DB::Exception: Cannot extract table structure")
- elif case == "sync_running":
- ExternalDataJob.objects.create(
- team=self.team, pipeline=schema.source, schema=schema, status=ExternalDataJob.Status.RUNNING
- )
- else:
- table = DataWarehouseTable.objects.create(
- team=self.team,
- name="linked",
- format="CSVWithNames",
- url_pattern=f"https://warehouse.posthog.test/file_uploads/team_{self.team.pk}/{UPLOAD_ID}/orders.csv",
- columns={},
- )
-
- response = self._replace(table)
-
- assert response.status_code == status.HTTP_400_BAD_REQUEST, response.json()
- schema.source.refresh_from_db()
- assert schema.source.job_inputs["upload_id"] == UPLOAD_ID
- assert "file_upload" not in (schema.source.connection_metadata or {})
- assert self.trigger_count == 1
-
- def test_the_source_update_endpoint_cannot_repoint_the_upload(self) -> None:
- _, schema = self._table_with_source()
-
- response = self.client.patch(
- f"/api/environments/{self.team.pk}/external_data_sources/{schema.source_id}/",
- {
- "job_inputs": {
- "upload_id": NEW_UPLOAD_ID,
- "filename": "../../team_1/secret.csv",
- "file_format": "parquet",
- "schema_name": "other",
- }
- },
- format="json",
- )
-
- assert response.status_code == status.HTTP_200_OK, response.json()
- schema.source.refresh_from_db()
- assert schema.source.job_inputs == {
- "upload_id": UPLOAD_ID,
- "filename": "orders.csv",
- "file_format": "csv",
- "schema_name": "orders",
- }
-
- def test_the_generic_source_endpoint_cannot_create_one(self) -> None:
- response = self.client.post(
- f"/api/environments/{self.team.pk}/external_data_sources/",
- {
- "source_type": "FileUpload",
- "payload": {
- "upload_id": UPLOAD_ID,
- "filename": "orders.csv",
- "file_format": "csv",
- "schema_name": "orders",
- },
- },
- format="json",
- )
-
- assert response.status_code == status.HTTP_400_BAD_REQUEST
- assert "can't be created here" in response.json()["message"]
- assert not ExternalDataSource.objects.filter(team=self.team).exists()
-
- def test_its_schedule_cannot_be_turned_on(self) -> None:
- _, schema = self._table_with_source()
-
- response = self.client.patch(
- f"/api/environments/{self.team.pk}/external_data_schemas/{schema.id}/",
- {"should_sync": True, "sync_frequency": "24hour"},
- format="json",
- )
-
- assert response.status_code == status.HTTP_400_BAD_REQUEST
- assert "can't be turned on" in str(response.json())
- schema.refresh_from_db()
- assert schema.should_sync is False
-
- def test_its_table_is_deleted_through_the_source(self) -> None:
- table, _ = self._table_with_source()
-
- response = self.client.delete(f"{self.base_url}/{table.id}")
-
- assert response.status_code == status.HTTP_400_BAD_REQUEST
- table.refresh_from_db()
- assert table.deleted is False
- assert self.s3.removed == []
-
- @patch(f"{SOURCE_VIEWS}.delete_discover_schemas_schedule")
- @patch(f"{SOURCE_VIEWS}.delete_external_data_schedule")
- @patch(f"{SOURCE_VIEWS}.bulk_delete_external_data_schedules", return_value=[])
- def test_deleting_the_source_deletes_the_table_and_its_files(self, *_mocks: object) -> None:
- table, schema = self._table_with_source()
- self.get_columns.return_value = dict(REPLACEMENT_COLUMNS)
- assert self._replace(table).status_code == status.HTTP_202_ACCEPTED
-
- with self.captureOnCommitCallbacks(execute=True):
- response = self.client.delete(f"/api/environments/{self.team.pk}/external_data_sources/{schema.source_id}")
-
- assert response.status_code == status.HTTP_204_NO_CONTENT
- table.refresh_from_db()
- assert table.deleted is True
- prefix = f"{settings.DATAWAREHOUSE_BUCKET}/file_uploads/team_{self.team.pk}/"
- assert sorted(self.s3.removed) == sorted(
- [f"{prefix}{NEW_UPLOAD_ID}/orders_v2.csv", f"{prefix}{UPLOAD_ID}/orders.csv"]
- )
-
- def _migrated_table(self, key: str) -> tuple[DataWarehouseTable, ExternalDataSchema]:
- table = DataWarehouseTable.objects.create(
- team=self.team,
- name="orders",
- format="CSVWithNames",
- url_pattern=f"https://warehouse.posthog.test/{key.format(team=self.team.pk, upload=UPLOAD_ID)}",
- columns=dict(FAKE_COLUMNS),
- options={"csv_allow_double_quotes": False},
- )
- [outcome] = migrate_team_self_managed_tables(self.team.pk)
- assert outcome.action == "linked", outcome
- assert outcome.status.schema_id is not None
- schema = ExternalDataSchema.objects.select_related("source").get(pk=outcome.status.schema_id)
- return table, schema
-
- def _legacy_file_action(self, filename: str = "orders_v2.csv"):
- return self.client.post(
- f"{self.base_url}/file/",
- {"file": SimpleUploadedFile(filename, b"id,total\n1,2\n"), "name": "orders", "format": "CSVWithNames"},
- format="multipart",
- )
-
- @parameterized.expand(
- [
- ("upload_folder_replace_file", "file_uploads/team_{team}/{upload}/orders.csv", "replace_file"),
- ("legacy_folder_replace_file", "managed/team_{team}/orders.csv", "replace_file"),
- ("legacy_folder_file_action", "managed/team_{team}/orders.csv", "file"),
- ]
- )
- def test_a_new_file_for_a_migrated_table_starts_the_sync_that_adopts_it(
- self, _name: str, key: str, path: str
- ) -> None:
- self.flag.side_effect = _warehouse_api_flag
- table, schema = self._migrated_table(key)
- old_url = table.url_pattern
- self.get_columns.return_value = dict(REPLACEMENT_COLUMNS)
-
- with patch(f"{VIEW_MODULE}.boto3.client") as legacy_s3:
- if path == "replace_file":
- response = self._replace(table)
- else:
- response = self._legacy_file_action()
-
- expected_status = status.HTTP_202_ACCEPTED if path == "replace_file" else status.HTTP_201_CREATED
- assert response.status_code == expected_status, response.json()
- assert response.json()["id"] == str(table.id)
- # The legacy action no longer writes over the file the table reads.
- legacy_s3.assert_not_called()
- schema.refresh_from_db()
- schema.source.refresh_from_db()
- job_inputs = schema.source.job_inputs
- if path == "replace_file":
- assert job_inputs["upload_id"] == NEW_UPLOAD_ID
- else:
- [written] = self.s3.written
- assert written == (f"test-bucket/file_uploads/team_{self.team.pk}/{job_inputs['upload_id']}/orders_v2.csv")
- assert (job_inputs["filename"], job_inputs["file_format"]) == ("orders_v2.csv", "csv")
- [superseded] = (schema.source.connection_metadata or {})["file_upload"]["superseded"]
- if key.startswith("managed/"):
- assert superseded["legacy_filename"] == "orders.csv"
- else:
- assert (superseded["upload_id"], superseded["filename"]) == (UPLOAD_ID, "orders.csv")
- assert schema.schema_metadata is not None
- assert schema.schema_metadata["csv"]["column_types"] == {"id": "integer", "placed_at": "datetime"}
- assert schema.schema_metadata["self_managed_table"]["table_id"] == str(table.id)
- self.schedule.assert_called_once_with(schema, create=True, should_sync=False, trigger_immediately=False)
- assert self.trigger_count == 1
- assert self.trigger.call_args.kwargs["billable"] is True
- # Until that sync publishes, the table reads its old file, and the sync still adopts the same row.
- table.refresh_from_db()
- assert (table.name, table.url_pattern, table.external_data_source_id) == ("orders", old_url, None)
- assert find_replaced_self_managed_table(schema, schema.source) == table
- assert self.s3.removed == []
-
- @parameterized.expand(["sync_running", "table_changed_since_migration"])
- def test_a_migrated_table_refuses_a_new_file_it_cant_take(self, case: str) -> None:
- table, schema = self._migrated_table("managed/team_{team}/orders.csv")
- if case == "sync_running":
- ExternalDataJob.objects.create(
- team=self.team, pipeline=schema.source, schema=schema, status=ExternalDataJob.Status.RUNNING
- )
- else:
- DataWarehouseTable.objects.filter(pk=table.pk).update(
- url_pattern=f"https://warehouse.posthog.test/managed/team_{self.team.pk}/other.csv"
- )
-
- response = self._replace(table)
-
- assert response.status_code == status.HTTP_400_BAD_REQUEST, response.json()
- schema.source.refresh_from_db()
- assert schema.source.job_inputs["upload_id"] == ""
- assert "file_upload" not in (schema.source.connection_metadata or {})
- self.trigger.assert_not_called()
-
- def test_a_migrated_source_can_be_updated_before_its_next_upload(self) -> None:
- _, schema = self._migrated_table("managed/team_{team}/orders.csv")
-
- response = self.client.patch(
- f"/api/environments/{self.team.pk}/external_data_sources/{schema.source_id}/",
- {"description": "Orders", "job_inputs": {"upload_id": NEW_UPLOAD_ID, "filename": "orders_v2.csv"}},
- format="json",
- )
-
- assert response.status_code == status.HTTP_200_OK, response.json()
- schema.source.refresh_from_db()
- assert schema.source.description == "Orders"
- assert (schema.source.job_inputs["upload_id"], schema.source.job_inputs["filename"]) == ("", "")
-
- def test_a_refused_legacy_file_action_deletes_the_file_it_stored(self) -> None:
- self.flag.side_effect = _warehouse_api_flag
- _, schema = self._migrated_table("managed/team_{team}/orders.csv")
- ExternalDataJob.objects.create(
- team=self.team, pipeline=schema.source, schema=schema, status=ExternalDataJob.Status.RUNNING
- )
- self.get_columns.return_value = dict(REPLACEMENT_COLUMNS)
-
- with patch(f"{VIEW_MODULE}.boto3.client"):
- response = self._legacy_file_action()
-
- assert response.status_code == status.HTTP_400_BAD_REQUEST, response.json()
- [written] = self.s3.written
- assert self.s3.removed == [written]
- schema.source.refresh_from_db()
- assert schema.source.job_inputs["upload_id"] == ""
- self.trigger.assert_not_called()
-
- @parameterized.expand(["reload", "resync"])
- def test_a_migrated_table_cant_sync_before_its_next_upload(self, action: str) -> None:
- _, schema = self._migrated_table("managed/team_{team}/orders.csv")
-
- response = self.client.post(f"/api/environments/{self.team.pk}/external_data_schemas/{schema.id}/{action}/")
-
- assert response.status_code == status.HTTP_400_BAD_REQUEST
- assert "Upload the file" in response.json()["message"]
- assert not ExternalDataJob.objects.filter(team=self.team).exists()
- self.trigger.assert_not_called()
-
- @parameterized.expand(["reload", "resync"])
- def test_a_table_that_imported_its_file_points_to_replace_file(self, action: str) -> None:
- _, schema = self._table_with_source()
- ExternalDataJob.objects.create(
- team=self.team, pipeline=schema.source, schema=schema, status=ExternalDataJob.Status.COMPLETED
- )
- self.trigger.reset_mock()
-
- response = self.client.post(f"/api/environments/{self.team.pk}/external_data_schemas/{schema.id}/{action}/")
-
- assert response.status_code == status.HTTP_400_BAD_REQUEST
- assert "Replace file" in response.json()["message"]
- self.trigger.assert_not_called()
-
- @parameterized.expand(
- [
- ("linked", None),
- ("pinned", "managed/team_{team}/orders.csv"),
- ("pinned_in_upload_folder", "file_uploads/team_{team}/{upload}/orders.csv"),
- ]
- )
- def test_the_schema_and_source_name_the_table_and_the_file(self, _name: str, migrated_key: str | None) -> None:
- if migrated_key is None:
- table, schema = self._table_with_source()
- else:
- table, schema = self._migrated_table(migrated_key)
- assert schema.table_id is None
-
- schema_body = self.client.get(f"/api/environments/{self.team.pk}/external_data_schemas/{schema.id}/").json()
- source_body = self.client.get(
- f"/api/environments/{self.team.pk}/external_data_sources/{schema.source_id}/"
- ).json()
-
- assert schema_body["table_id"] == str(table.id)
- assert source_body["schemas"][0]["table_id"] == str(table.id)
- assert source_body["file_name"] == "orders.csv"
- assert schema_body["source"]["file_name"] == "orders.csv"
- assert source_body["job_inputs"] == {}
-
- @parameterized.expand(["other_team", "deleted"])
- def test_a_pinned_table_that_is_not_live_in_the_team_is_not_returned(self, case: str) -> None:
- table, schema = self._migrated_table("managed/team_{team}/orders.csv")
- if case == "deleted":
- DataWarehouseTable.objects.filter(pk=table.pk).update(deleted=True)
- else:
- other_team = Team.objects.create(organization=self.organization, name="Other")
- DataWarehouseTable.objects.filter(pk=table.pk).update(team=other_team)
-
- body = self.client.get(f"/api/environments/{self.team.pk}/external_data_schemas/{schema.id}/").json()
-
- assert body["table_id"] is None
-
- @patch(f"{SOURCE_VIEWS}.delete_discover_schemas_schedule")
- @patch(f"{SOURCE_VIEWS}.delete_external_data_schedule")
- @patch(f"{SOURCE_VIEWS}.bulk_delete_external_data_schedules", return_value=[])
- def test_deleting_a_migrated_source_never_deletes_the_file_its_table_reads(self, *_mocks: object) -> None:
- table, schema = self._migrated_table("managed/team_{team}/orders.csv")
- self.get_columns.return_value = dict(REPLACEMENT_COLUMNS)
- # The sync of the new file never finishes, so the table still reads the old one.
- assert self._replace(table).status_code == status.HTTP_202_ACCEPTED
-
- with self.captureOnCommitCallbacks(execute=True):
- response = self.client.delete(f"/api/environments/{self.team.pk}/external_data_sources/{schema.source_id}")
-
- assert response.status_code == status.HTTP_204_NO_CONTENT
- table.refresh_from_db()
- assert table.deleted is False
- assert self.s3.removed == [f"test-bucket/file_uploads/team_{self.team.pk}/{NEW_UPLOAD_ID}/orders_v2.csv"]
diff --git a/products/logs/backend/test/test_alerts_api.py b/products/logs/backend/test/test_alerts_api.py
index ce47a85e0605..574271bd9eec 100644
--- a/products/logs/backend/test/test_alerts_api.py
+++ b/products/logs/backend/test/test_alerts_api.py
@@ -24,6 +24,7 @@ from posthog.models.activity_logging.activity_log import ActivityLog
from posthog.models.scoping import team_scope
from posthog.models.team.team import Team
from posthog.models.user import User
+from posthog.security.url_validation import PinnedUrlVerdict
from products.alerts_platform.backend.facade import testing as platform_testing
from products.alerts_platform.backend.facade.api import get_configuration, list_configurations, upsert_configuration
@@ -76,7 +77,9 @@ def _make_log_row(*, team_id: int, service: str, uuid: str, ts: datetime, body:
# Where a request-side delivery meets Temporal and the transports, which the tests stand in for.
_WEB_START = "products.alerts_platform.backend.temporal.web_start"
_ALERTS_API = "products.logs.backend.presentation.views.alerts_api"
-_NATIVE_PAGERDUTY_DELIVER = "products.alerts_platform.backend.delivery.pagerduty.PagerDutyTransport.deliver"
+# A native send leaves through the pinned adapter, so a stub there counts native sends with no network.
+_PINNED_URL_CHECK = "posthog.security.pinned_requests.validate_url_and_pin_ips"
+_PINNED_SEND = "posthog.security.pinned_requests.PinnedIPAdapter.send"
class TestLogsAlertAPI(APIBaseTest):
@@ -2906,7 +2909,8 @@ class TestLogsAlertAPI(APIBaseTest):
patch("posthog.permissions.posthog_feature_flag_enabled", return_value=True),
# Sends the resolve inline, so the test sees which path delivered it.
patch(f"{_WEB_START}.async_connect", AsyncMock(side_effect=RuntimeError("temporal unreachable"))),
- patch(_NATIVE_PAGERDUTY_DELIVER, return_value=None) as native,
+ patch(_PINNED_URL_CHECK, return_value=PinnedUrlVerdict(allowed=True, reason=None, pinned_ips=set())),
+ patch(_PINNED_SEND, return_value=MagicMock(status_code=202)) as native,
patch("products.logs.backend.platform_alert_legacy_events.emit_alert_internal_events") as emit,
self.captureOnCommitCallbacks(execute=True),
):
diff --git a/products/logs/backend/test/test_platform_cutover.py b/products/logs/backend/test/test_platform_cutover.py
index bcbe720512cf..71f3bf38e4da 100644
--- a/products/logs/backend/test/test_platform_cutover.py
+++ b/products/logs/backend/test/test_platform_cutover.py
@@ -35,7 +35,7 @@ from products.logs.backend.platform_cutover import CutoverSkip, RollbackSkip, cu
NOW = datetime(2026, 10, 13, 12, 0, tzinfo=UTC)
State = LogsAlertConfiguration.State
-LEGACY_BODY_FLAG = "products.alerts_platform.backend.delivery.team_flags.posthoganalytics.feature_enabled"
+LEGACY_BODY_FLAG = "posthoganalytics.feature_enabled"
STOCK_HOG = "return event"
DESTINATION_DATA: dict[DestinationType, dict[str, Any]] = {
DestinationType.SLACK: {"slack_workspace_id": 1, "slack_channel_id": "C-1"},
diff --git a/products/managed_warehouse/backend/tests/test_managed_ingress.py b/products/managed_warehouse/backend/tests/test_managed_ingress.py
new file mode 100644
index 000000000000..61f8e7f6aa45
--- /dev/null
+++ b/products/managed_warehouse/backend/tests/test_managed_ingress.py
@@ -0,0 +1,188 @@
+from unittest.mock import patch
+
+from django.test import SimpleTestCase, override_settings
+
+from parameterized import parameterized
+
+from products.managed_warehouse.backend.facade.api import get_managed_warehouse_target
+from products.managed_warehouse.backend.facade.contracts import ManagedWarehouseTarget
+
+_INGRESS_MODULE = "products.managed_warehouse.backend.managed_ingress"
+
+_NLB_A = "10.1.0.1"
+_NLB_B = "10.1.0.2"
+_TENANT_HOST = "acme.dw.us.postwh.com"
+
+
+def _target(host: str = _TENANT_HOST, ports: frozenset[int] = frozenset({5432})) -> ManagedWarehouseTarget:
+ return ManagedWarehouseTarget(host=host, ports=ports)
+
+
+class TestManagedWarehouseTargetRead(SimpleTestCase):
+ """The control-plane read: what it accepts as a ready target, and what it refuses."""
+
+ @override_settings(MANAGED_WAREHOUSE_INGRESS_IPS=f"{_NLB_A},{_NLB_B}")
+ def test_a_ready_warehouse_yields_its_host(self) -> None:
+ with (
+ patch(f"{_INGRESS_MODULE}.hogtower") as hogtower,
+ patch(f"{_INGRESS_MODULE}.cache"),
+ ):
+ hogtower.is_configured.return_value = True
+ hogtower.request.return_value = _response(
+ {"org_id": "org-1", "state": "ready", "connection": {"host": _TENANT_HOST}}
+ )
+ target = get_managed_warehouse_target("org-1", source_type="postgres")
+
+ assert target is not None
+ assert target.host == _TENANT_HOST
+ assert target.ports == frozenset({5432})
+
+ @parameterized.expand(
+ [
+ ("provisioning", {"state": "provisioning", "connection": {"host": _TENANT_HOST}}),
+ ("deleting", {"state": "deleting", "connection": {"host": _TENANT_HOST}}),
+ ("failed", {"state": "failed", "connection": {"host": _TENANT_HOST}}),
+ ("ready_without_connection", {"state": "ready"}),
+ ("ready_with_blank_host", {"state": "ready", "connection": {"host": " "}}),
+ ]
+ )
+ def test_a_warehouse_that_is_not_ready_yields_no_target(self, _name: str, body: dict) -> None:
+ with (
+ patch(f"{_INGRESS_MODULE}.hogtower") as hogtower,
+ patch(f"{_INGRESS_MODULE}.cache"),
+ ):
+ hogtower.is_configured.return_value = True
+ hogtower.request.return_value = _response({"org_id": "org-1", **body})
+ assert get_managed_warehouse_target("org-1", source_type="postgres") is None
+
+ def test_a_response_naming_another_organization_is_refused(self) -> None:
+ # A control-plane bug or a misrouted proxy must never let one tenant's host answer for
+ # another, which would be a cross-tenant read path.
+ with (
+ patch(f"{_INGRESS_MODULE}.hogtower") as hogtower,
+ patch(f"{_INGRESS_MODULE}.cache"),
+ patch(f"{_INGRESS_MODULE}.logger"),
+ ):
+ hogtower.is_configured.return_value = True
+ hogtower.request.return_value = _response(
+ {"org_id": "org-2", "state": "ready", "connection": {"host": "other.dw.us.postwh.com"}}
+ )
+ assert get_managed_warehouse_target("org-1", source_type="postgres") is None
+
+ def test_a_source_type_the_warehouse_does_not_serve_is_not_read(self) -> None:
+ # No control-plane call at all: the mapping decides before anything leaves the process.
+ with (
+ patch(f"{_INGRESS_MODULE}.hogtower") as hogtower,
+ patch(f"{_INGRESS_MODULE}.cache"),
+ ):
+ assert get_managed_warehouse_target("org-1", source_type="mysql") is None
+ hogtower.request.assert_not_called()
+
+ def test_an_unconfigured_control_plane_yields_no_target(self) -> None:
+ with (
+ patch(f"{_INGRESS_MODULE}.hogtower") as hogtower,
+ patch(f"{_INGRESS_MODULE}.cache"),
+ ):
+ hogtower.is_configured.return_value = False
+ assert get_managed_warehouse_target("org-1", source_type="postgres") is None
+
+ def test_a_control_plane_timeout_is_denied_and_cached_briefly(self) -> None:
+ # An outage must not turn every connect into a full-timeout call to an already-sick
+ # service, so a denial is cached for a short window.
+ import requests
+
+ with (
+ patch(f"{_INGRESS_MODULE}.hogtower") as hogtower,
+ patch(f"{_INGRESS_MODULE}.cache") as cache,
+ patch(f"{_INGRESS_MODULE}.logger"),
+ ):
+ hogtower.is_configured.return_value = True
+ hogtower.request.side_effect = requests.Timeout()
+ assert get_managed_warehouse_target("org-1", source_type="postgres") is None
+
+ assert cache.set.called
+
+ def test_a_cached_target_is_returned_without_a_control_plane_call(self) -> None:
+ cached = _target()
+ with (
+ patch(f"{_INGRESS_MODULE}.hogtower") as hogtower,
+ patch(f"{_INGRESS_MODULE}.cache") as cache,
+ ):
+ cache.get.return_value = cached
+ assert get_managed_warehouse_target("org-1", source_type="postgres") is cached
+ hogtower.request.assert_not_called()
+
+
+def _response(body: dict):
+ # The control-plane read only checks status_code and body, so a small local object stands in
+ # for the translator's response type without importing the presentation layer.
+ return type("Response", (), {"status_code": 200, "body": body})()
+
+
+class TestDenialCacheSurvivesSerialization(SimpleTestCase):
+ """A denial must be recognised after a round trip through the cache backend.
+
+ The cache pickles its values, so an `object()` sentinel comes back as a different object and
+ an identity comparison never matches. A denial that never hits is worse than no cache: every
+ denied dial, and every dial during a control-plane outage, pays the full control-plane
+ timeout against an already-sick service.
+ """
+
+ def test_a_denial_is_recognised_after_a_pickle_round_trip(self) -> None:
+ import pickle
+
+ from products.managed_warehouse.backend.facade.testing import NO_MANAGED_WAREHOUSE_TARGET
+
+ serialized = pickle.dumps(NO_MANAGED_WAREHOUSE_TARGET) # nosemgrep
+ restored = pickle.loads(serialized) # nosemgrep
+ assert restored == NO_MANAGED_WAREHOUSE_TARGET
+
+ def test_a_cached_denial_is_returned_without_a_control_plane_call(self) -> None:
+ from products.managed_warehouse.backend.facade.testing import NO_MANAGED_WAREHOUSE_TARGET
+
+ with (
+ patch(f"{_INGRESS_MODULE}.hogtower") as hogtower,
+ patch(f"{_INGRESS_MODULE}.cache") as cache,
+ ):
+ cache.get.return_value = NO_MANAGED_WAREHOUSE_TARGET
+ assert get_managed_warehouse_target("org-1", source_type="postgres") is None
+ hogtower.request.assert_not_called()
+
+ def test_a_cache_read_fault_denies_instead_of_raising(self) -> None:
+ # The guard runs inside a connect. An exception from the cache would replace a clear
+ # refusal with an opaque connection error.
+ with (
+ patch(f"{_INGRESS_MODULE}.cache") as cache,
+ patch(f"{_INGRESS_MODULE}.hogtower") as hogtower,
+ patch(f"{_INGRESS_MODULE}.logger"),
+ ):
+ cache.get.side_effect = RuntimeError("redis down")
+ hogtower.is_configured.return_value = False
+ assert get_managed_warehouse_target("org-1", source_type="postgres") is None
+
+ def test_a_cache_write_fault_still_answers_the_dial(self) -> None:
+ with (
+ patch(f"{_INGRESS_MODULE}.cache") as cache,
+ patch(f"{_INGRESS_MODULE}.hogtower") as hogtower,
+ patch(f"{_INGRESS_MODULE}.logger"),
+ ):
+ cache.get.return_value = None
+ cache.set.side_effect = RuntimeError("redis down")
+ hogtower.is_configured.return_value = True
+ hogtower.request.return_value = _response(
+ {"org_id": "org-1", "state": "ready", "connection": {"host": _TENANT_HOST}}
+ )
+ target = get_managed_warehouse_target("org-1", source_type="postgres")
+
+ assert target is not None
+ assert target.host == _TENANT_HOST
+
+ def test_a_control_plane_error_the_adapter_cannot_translate_denies(self) -> None:
+ with (
+ patch(f"{_INGRESS_MODULE}.cache"),
+ patch(f"{_INGRESS_MODULE}.hogtower") as hogtower,
+ patch(f"{_INGRESS_MODULE}.logger"),
+ ):
+ hogtower.is_configured.return_value = True
+ hogtower.request.side_effect = KeyError("unmapped route")
+ assert get_managed_warehouse_target("org-1", source_type="postgres") is None
diff --git a/products/managed_warehouse/backend/tests/test_sql_editor_trino.py b/products/managed_warehouse/backend/tests/test_sql_editor_trino.py
new file mode 100644
index 000000000000..d6f4e6edd139
--- /dev/null
+++ b/products/managed_warehouse/backend/tests/test_sql_editor_trino.py
@@ -0,0 +1,139 @@
+from collections.abc import Generator
+from contextlib import contextmanager, nullcontext
+
+from posthog.test.base import APIBaseTest
+from unittest.mock import MagicMock, patch
+
+from parameterized import parameterized
+
+from posthog.hogql.errors import ExposedHogQLError
+from posthog.hogql.query import HogQLQueryExecutor
+
+from products.managed_warehouse.backend.facade.contracts import ManagedWarehouseTrinoConnectionUnavailable
+from products.warehouse_sources.backend.facade.models import (
+ MANAGED_WAREHOUSE_SOURCE_PREFIX,
+ DataWarehouseTable,
+ ExternalDataSource,
+)
+from products.warehouse_sources.backend.facade.types import ExternalDataSourceType
+
+
+class TestManagedTrinoSqlEditor(APIBaseTest):
+ def setUp(self) -> None:
+ super().setUp()
+ resolver_patcher = patch(
+ "posthog.hogql.direct_sql.duckgres_adapter.resolve_psycopg_hostaddr_with_timeout",
+ return_value=["203.0.113.10"],
+ )
+ resolver_patcher.start()
+ self.addCleanup(resolver_patcher.stop)
+
+ @parameterized.expand([(True, False), (False, True)])
+ @patch("posthoganalytics.feature_enabled", return_value=True)
+ def test_managed_source_switches_raw_and_hogql_execution_to_trino(
+ self, raw: bool, canary: bool, _flag: MagicMock
+ ) -> None:
+ _flag.side_effect = lambda flag, *_args, **_kwargs: flag == "managed-warehouse-trino-query"
+ source = self._dynamic_managed_source()
+ DataWarehouseTable.objects.create(
+ team=self.team,
+ external_data_source=source,
+ name="posthog.events",
+ format=DataWarehouseTable.TableFormat.Parquet,
+ url_pattern="direct://trino",
+ columns={"uuid": {"hogql": "StringDatabaseField", "clickhouse": "String", "valid": True}},
+ options={
+ "direct_trino_catalog": "org_example",
+ "direct_trino_schema": "posthog",
+ "direct_trino_table": "events",
+ },
+ )
+ cursor = MagicMock()
+ cursor.fetchmany.return_value = [("row-uuid",)]
+ cursor.description = [("uuid", "varchar")]
+ connection = MagicMock()
+ connection.cursor.return_value = cursor
+ query = 'SELECT uuid FROM "posthog"."events"' if raw else "SELECT uuid FROM posthog.events"
+ with (
+ patch(
+ "products.managed_warehouse.backend.logic.sql_editor_trino.connect_managed_warehouse_trino",
+ return_value=nullcontext(connection),
+ ) as connect,
+ patch("posthog.hogql.direct_sql.duckgres_adapter.psycopg.connect") as duckgres,
+ ):
+ response = HogQLQueryExecutor(
+ query=query,
+ team=self.team,
+ user=self.user,
+ connection_id=str(source.id),
+ send_raw_query=raw,
+ trino_canary=canary,
+ ).execute()
+ with self.assertRaisesMessage(ExposedHogQLError, "warehouse engine changed"):
+ HogQLQueryExecutor(
+ query=query,
+ team=self.team,
+ user=self.user,
+ connection_id=str(source.id),
+ send_raw_query=raw,
+ expected_direct_engine="postgres",
+ ).execute()
+ self.assertEqual(response.results, [("row-uuid",)])
+ connect.assert_called_once_with(
+ str(self.team.organization_id),
+ principal=f"posthog:sql-editor:team:{self.team.pk}:user:{self.user.pk}",
+ canary=canary,
+ )
+ duckgres.assert_not_called()
+ self.assertIn('"posthog"."events"', cursor.execute.call_args.args[0])
+ if not raw:
+ self.assertIn('"org_example"', cursor.execute.call_args.args[0])
+ options = self.client.get(f"/api/projects/{self.team.pk}/external_data_sources/connections/").json()
+ self.assertEqual(options[0]["id"], str(source.pk))
+ self.assertEqual(options[0]["engine"], "trino")
+ self.assertTrue(options[0]["is_builtin_managed_warehouse"])
+
+ @parameterized.expand([("open",), ("enter",)])
+ @patch("posthoganalytics.feature_enabled", return_value=True)
+ def test_managed_trino_unavailable_does_not_fall_back_to_duckgres(self, stage: str, _flag: MagicMock) -> None:
+ source = self._dynamic_managed_source()
+
+ @contextmanager
+ def unavailable() -> Generator[None]:
+ if stage == "enter":
+ raise ManagedWarehouseTrinoConnectionUnavailable("not ready")
+ yield None
+
+ with (
+ patch(
+ "products.managed_warehouse.backend.logic.sql_editor_trino.connect_managed_warehouse_trino",
+ side_effect=ManagedWarehouseTrinoConnectionUnavailable("not ready") if stage == "open" else None,
+ return_value=unavailable(),
+ ),
+ patch("posthog.hogql.direct_sql.duckgres_adapter.psycopg.connect") as duckgres,
+ self.assertRaisesMessage(ExposedHogQLError, "Trino is not ready"),
+ ):
+ HogQLQueryExecutor(
+ query="SELECT 1", team=self.team, user=self.user, connection_id=str(source.id), send_raw_query=True
+ ).execute()
+ duckgres.assert_not_called()
+
+ def _dynamic_managed_source(self) -> ExternalDataSource:
+ source_id = "managed-service-source"
+ return ExternalDataSource.objects.create(
+ team=self.team,
+ source_id=source_id,
+ connection_id=f"{source_id}-connection",
+ status=ExternalDataSource.Status.COMPLETED,
+ source_type=ExternalDataSourceType.POSTGRES,
+ access_method=ExternalDataSource.AccessMethod.DIRECT,
+ direct_query_enabled=True,
+ prefix=MANAGED_WAREHOUSE_SOURCE_PREFIX,
+ job_inputs={},
+ connection_metadata={
+ "engine": "duckdb",
+ "system_managed": True,
+ "credential_kind": "duckgres_service",
+ "lifecycle_generation": 1,
+ },
+ )
diff --git a/products/managed_warehouse/turbo.json b/products/managed_warehouse/turbo.json
index 558938f600d4..519ba6d61e4d 100644
--- a/products/managed_warehouse/turbo.json
+++ b/products/managed_warehouse/turbo.json
@@ -17,8 +17,6 @@
"backend/presentation/**",
"backend/temporal/**",
"backend/tasks/**",
- "backend/logic/sql_editor_trino.py",
- "backend/managed_ingress.py",
"backend/models.py",
"backend/migrations/**",
"backend/signals.py"
diff --git a/products/metrics/turbo.json b/products/metrics/turbo.json
index 8bf8dcd66e50..a83fb5d3579e 100644
--- a/products/metrics/turbo.json
+++ b/products/metrics/turbo.json
@@ -9,7 +9,6 @@
"backend/hogql_queries/**",
"backend/tasks/**",
"backend/temporal/**",
- "backend/metric_query_runner.py",
"backend/dashboard_import/receivers.py",
"backend/models.py",
"backend/migrations/**"
diff --git a/products/signals/backend/facade/api.py b/products/signals/backend/facade/api.py
index 321652a0a7ea..59e6e3c7974f 100644
--- a/products/signals/backend/facade/api.py
+++ b/products/signals/backend/facade/api.py
@@ -31,15 +31,23 @@ from posthog.sync import database_sync_to_async
from posthog.temporal.common.client import async_connect
from products.access_control.backend.facade.user_access_control import UserAccessControl
-from products.signals.backend.activity_summary import read as activity_summary_read
+from products.signals.backend.activity_summary import (
+ read as activity_summary_read,
+ render as activity_summary_render,
+ search as activity_summary_search,
+)
+from products.signals.backend.activity_summary.prompt import ActivitySummaryMode as ActivitySummaryMode
from products.signals.backend.activity_summary.read import (
+ ActivitySummaryStatus as ActivitySummaryStatus,
ActivitySummaryTheme as ActivitySummaryTheme,
ActivitySummaryVersionView as ActivitySummaryVersionView,
ActivitySummaryView as ActivitySummaryView,
)
from products.signals.backend.activity_summary.search import (
ActivitySummarySearchResult as ActivitySummarySearchResult,
- SearchMode,
+ MatchKind as MatchKind,
+ SearchMode as SearchMode,
+ SemanticSearchUnavailable as SemanticSearchUnavailable,
search_summaries,
)
from products.signals.backend.anomaly_scores import search as anomaly_search
@@ -144,6 +152,11 @@ logger = structlog.get_logger(__name__)
MAX_SIGNAL_DESCRIPTION_TOKENS = 8000
MAX_SIGNAL_REMEDIATION_TOKENS = 16000
+ActivitySummaryProductKey = activity_summary_render.Product
+ACTIVITY_SUMMARY_VERSIONS_DEFAULT_LIMIT = activity_summary_read.DEFAULT_VERSIONS_LIMIT
+ACTIVITY_SUMMARY_SEARCH_DEFAULT_LIMIT = activity_summary_search.DEFAULT_LIMIT
+ACTIVITY_SUMMARY_SEARCH_MAX_LIMIT = activity_summary_search.MAX_LIMIT
+
@frozen
class InboxSummaryPullRequest:
diff --git a/products/signals/backend/presentation/activity_summary.py b/products/signals/backend/presentation/activity_summary.py
index 95b17da8b88e..b5a0dd12dc42 100644
--- a/products/signals/backend/presentation/activity_summary.py
+++ b/products/signals/backend/presentation/activity_summary.py
@@ -24,25 +24,26 @@ from posthog.event_usage import report_user_action
from posthog.models import Team, User
from posthog.permissions import APIScopePermission, is_service_auth
-from products.signals.backend.activity_summary.prompt import ActivitySummaryMode
-from products.signals.backend.activity_summary.read import DEFAULT_VERSIONS_LIMIT, ActivitySummaryStatus
-from products.signals.backend.activity_summary.render import Product
-from products.signals.backend.activity_summary.search import (
- DEFAULT_LIMIT as SEARCH_DEFAULT_LIMIT,
- MAX_LIMIT as SEARCH_MAX_LIMIT,
+from products.signals.backend.facade import api as signals_facade
+from products.signals.backend.facade.api import (
+ ACTIVITY_SUMMARY_SEARCH_DEFAULT_LIMIT,
+ ACTIVITY_SUMMARY_SEARCH_MAX_LIMIT,
+ ACTIVITY_SUMMARY_VERSIONS_DEFAULT_LIMIT,
+ ActivitySummaryMode,
+ ActivitySummaryProductKey,
+ ActivitySummaryStatus,
MatchKind,
SearchMode,
SemanticSearchUnavailable,
)
-from products.signals.backend.facade import api as signals_facade
from products.signals.backend.presentation.scout_permissions import ScoutCanonicalTeamAccessPermission
READ_EVENT = "activity summary read"
-# Built from `Product`, so the API enum cannot drift from the keys that `computed.py` writes.
+# Built from `ActivitySummaryProductKey`, so the API enum cannot drift from the keys that `computed.py` writes.
ActivitySummaryProduct = models.TextChoices( # type: ignore[call-overload] # the stubs lack the functional API
- "ActivitySummaryProduct", [(product.name, product.value) for product in Product]
+ "ActivitySummaryProduct", [(product.name, product.value) for product in ActivitySummaryProductKey]
)
@@ -208,7 +209,7 @@ class ActivitySummaryVersionSerializer(serializers.Serializer):
class ActivitySummaryVersionsQuerySerializer(serializers.Serializer):
limit = serializers.IntegerField(
min_value=1,
- default=DEFAULT_VERSIONS_LIMIT,
+ default=ACTIVITY_SUMMARY_VERSIONS_DEFAULT_LIMIT,
help_text="The most versions to return, newest first. Capped at the number of versions kept.",
)
version = serializers.IntegerField(
@@ -323,8 +324,8 @@ class ActivitySummarySearchQuerySerializer(serializers.Serializer):
)
limit = serializers.IntegerField(
min_value=1,
- max_value=SEARCH_MAX_LIMIT,
- default=SEARCH_DEFAULT_LIMIT,
+ max_value=ACTIVITY_SUMMARY_SEARCH_MAX_LIMIT,
+ default=ACTIVITY_SUMMARY_SEARCH_DEFAULT_LIMIT,
help_text="The most results to return, best match first.",
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/test_managed_warehouse_exemption.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/test_managed_warehouse_exemption.py
index 75e4d738a992..eea1f543dc6b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/test_managed_warehouse_exemption.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/test_managed_warehouse_exemption.py
@@ -20,7 +20,6 @@ from django.test import SimpleTestCase, override_settings
from parameterized import parameterized
-from products.managed_warehouse.backend.facade.api import get_managed_warehouse_target
from products.managed_warehouse.backend.facade.contracts import ManagedWarehouseTarget
from products.warehouse_sources.backend.temporal.data_imports.sources.common.mixins import (
HostNotAllowedError,
@@ -31,7 +30,8 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.mix
)
_MIXINS_MODULE = "products.warehouse_sources.backend.temporal.data_imports.sources.common.mixins"
-_INGRESS_MODULE = "products.managed_warehouse.backend.managed_ingress"
+# The guard imports this facade function at the call, so a patch on the facade reaches it.
+_TARGET_READ = "products.managed_warehouse.backend.facade.api.get_managed_warehouse_target"
# Two addresses, because the incident observed the shared load balancer answering on both.
_NLB_A = "10.1.0.1"
@@ -161,7 +161,7 @@ class TestManagedWarehouseExemptionInTheGuard(SimpleTestCase):
patch(f"{_MIXINS_MODULE}._managed_warehouse_organization_id", return_value="org-1"),
patch(f"{_MIXINS_MODULE}.logger"),
patch(
- f"{_INGRESS_MODULE}.get_managed_warehouse_target",
+ _TARGET_READ,
return_value=_target(),
),
):
@@ -218,7 +218,7 @@ class TestManagedWarehouseExemptionInTheGuard(SimpleTestCase):
patch(f"{_MIXINS_MODULE}._managed_warehouse_organization_id", return_value="org-1"),
patch(f"{_MIXINS_MODULE}.logger"),
patch(
- f"{_INGRESS_MODULE}.get_managed_warehouse_target",
+ _TARGET_READ,
return_value=_target(),
),
):
@@ -236,7 +236,7 @@ class TestManagedWarehouseExemptionInTheGuard(SimpleTestCase):
patch(f"{_MIXINS_MODULE}._managed_warehouse_organization_id", return_value="org-1"),
patch(f"{_MIXINS_MODULE}.logger"),
patch(
- f"{_INGRESS_MODULE}.get_managed_warehouse_target",
+ _TARGET_READ,
return_value=_target(),
),
):
@@ -253,7 +253,7 @@ class TestManagedWarehouseExemptionInTheGuard(SimpleTestCase):
patch(f"{_MIXINS_MODULE}._managed_warehouse_organization_id", return_value="org-1"),
patch(f"{_MIXINS_MODULE}.logger"),
patch(
- f"{_INGRESS_MODULE}.get_managed_warehouse_target",
+ _TARGET_READ,
return_value=_target(),
),
):
@@ -271,7 +271,7 @@ class TestManagedWarehouseExemptionInTheGuard(SimpleTestCase):
patch(f"{_MIXINS_MODULE}.socket.getaddrinfo", return_value=[(None, None, None, None, (_NLB_A, 0))]),
patch(f"{_MIXINS_MODULE}._managed_warehouse_organization_id", return_value="org-1"),
patch(f"{_MIXINS_MODULE}.logger"),
- patch(f"{_INGRESS_MODULE}.get_managed_warehouse_target", return_value=None),
+ patch(_TARGET_READ, return_value=None),
):
resolution = resolve_safe_host(_TENANT_HOST, 999, port=5432, source_type="postgres")
@@ -294,7 +294,7 @@ class TestManagedWarehouseExemptionInTheGuard(SimpleTestCase):
patch(f"{_MIXINS_MODULE}.socket.getaddrinfo", return_value=[(None, None, None, None, (_NLB_A, 0))]),
patch(f"{_MIXINS_MODULE}._managed_warehouse_organization_id", return_value="org-1"),
patch(f"{_MIXINS_MODULE}.logger"),
- patch(f"{_INGRESS_MODULE}.get_managed_warehouse_target", return_value=None),
+ patch(_TARGET_READ, return_value=None),
):
resolution = resolve_safe_host(_TENANT_HOST, 999, port=5432, source_type="postgres")
@@ -316,7 +316,7 @@ class TestManagedWarehouseExemptionInTheGuard(SimpleTestCase):
patch(f"{_MIXINS_MODULE}.socket.getaddrinfo", return_value=[(None, None, None, None, (_NLB_A, 0))]),
patch(f"{_MIXINS_MODULE}._managed_warehouse_organization_id", return_value="org-1"),
patch(f"{_MIXINS_MODULE}.logger"),
- patch(f"{_INGRESS_MODULE}.get_managed_warehouse_target", return_value=_target()),
+ patch(_TARGET_READ, return_value=_target()),
override_settings(MANAGED_WAREHOUSE_INGRESS_IPS=""),
):
resolution = resolve_safe_host(_TENANT_HOST, 999, port=5432, source_type="postgres")
@@ -324,115 +324,14 @@ class TestManagedWarehouseExemptionInTheGuard(SimpleTestCase):
assert resolution.connect_host is None
-class TestManagedWarehouseTargetRead(SimpleTestCase):
- """The control-plane read: what it accepts as a ready target, and what it refuses."""
-
- @override_settings(MANAGED_WAREHOUSE_INGRESS_IPS=f"{_NLB_A},{_NLB_B}")
- def test_a_ready_warehouse_yields_its_host(self) -> None:
- with (
- patch(f"{_INGRESS_MODULE}.hogtower") as hogtower,
- patch(f"{_INGRESS_MODULE}.cache"),
- ):
- hogtower.is_configured.return_value = True
- hogtower.request.return_value = _response(
- {"org_id": "org-1", "state": "ready", "connection": {"host": _TENANT_HOST}}
- )
- target = get_managed_warehouse_target("org-1", source_type="postgres")
-
- assert target is not None
- assert target.host == _TENANT_HOST
- assert target.ports == frozenset({5432})
-
- @parameterized.expand(
- [
- ("provisioning", {"state": "provisioning", "connection": {"host": _TENANT_HOST}}),
- ("deleting", {"state": "deleting", "connection": {"host": _TENANT_HOST}}),
- ("failed", {"state": "failed", "connection": {"host": _TENANT_HOST}}),
- ("ready_without_connection", {"state": "ready"}),
- ("ready_with_blank_host", {"state": "ready", "connection": {"host": " "}}),
- ]
- )
- def test_a_warehouse_that_is_not_ready_yields_no_target(self, _name: str, body: dict) -> None:
- with (
- patch(f"{_INGRESS_MODULE}.hogtower") as hogtower,
- patch(f"{_INGRESS_MODULE}.cache"),
- ):
- hogtower.is_configured.return_value = True
- hogtower.request.return_value = _response({"org_id": "org-1", **body})
- assert get_managed_warehouse_target("org-1", source_type="postgres") is None
-
- def test_a_response_naming_another_organization_is_refused(self) -> None:
- # A control-plane bug or a misrouted proxy must never let one tenant's host answer for
- # another, which would be a cross-tenant read path.
- with (
- patch(f"{_INGRESS_MODULE}.hogtower") as hogtower,
- patch(f"{_INGRESS_MODULE}.cache"),
- patch(f"{_INGRESS_MODULE}.logger"),
- ):
- hogtower.is_configured.return_value = True
- hogtower.request.return_value = _response(
- {"org_id": "org-2", "state": "ready", "connection": {"host": "other.dw.us.postwh.com"}}
- )
- assert get_managed_warehouse_target("org-1", source_type="postgres") is None
-
- def test_a_source_type_the_warehouse_does_not_serve_is_not_read(self) -> None:
- # No control-plane call at all: the mapping decides before anything leaves the process.
- with (
- patch(f"{_INGRESS_MODULE}.hogtower") as hogtower,
- patch(f"{_INGRESS_MODULE}.cache"),
- ):
- assert get_managed_warehouse_target("org-1", source_type="mysql") is None
- hogtower.request.assert_not_called()
-
- def test_an_unconfigured_control_plane_yields_no_target(self) -> None:
- with (
- patch(f"{_INGRESS_MODULE}.hogtower") as hogtower,
- patch(f"{_INGRESS_MODULE}.cache"),
- ):
- hogtower.is_configured.return_value = False
- assert get_managed_warehouse_target("org-1", source_type="postgres") is None
-
- def test_a_control_plane_timeout_is_denied_and_cached_briefly(self) -> None:
- # An outage must not turn every connect into a full-timeout call to an already-sick
- # service, so a denial is cached for a short window.
- import requests
-
- with (
- patch(f"{_INGRESS_MODULE}.hogtower") as hogtower,
- patch(f"{_INGRESS_MODULE}.cache") as cache,
- patch(f"{_INGRESS_MODULE}.logger"),
- ):
- hogtower.is_configured.return_value = True
- hogtower.request.side_effect = requests.Timeout()
- assert get_managed_warehouse_target("org-1", source_type="postgres") is None
-
- assert cache.set.called
-
- def test_a_cached_target_is_returned_without_a_control_plane_call(self) -> None:
- cached = _target()
- with (
- patch(f"{_INGRESS_MODULE}.hogtower") as hogtower,
- patch(f"{_INGRESS_MODULE}.cache") as cache,
- ):
- cache.get.return_value = cached
- assert get_managed_warehouse_target("org-1", source_type="postgres") is cached
- hogtower.request.assert_not_called()
-
-
-def _response(body: dict):
- # The control-plane read only checks status_code and body, so a small local object stands in
- # for the translator's response type without importing the presentation layer.
- return type("Response", (), {"status_code": 200, "body": body})()
-
-
class TestNoImportOfTheManagedWarehouseProductAtModuleLoad(SimpleTestCase):
"""The guard must not pull the control-plane read into every worker that loads it.
`mixins` is imported when the Temporal worker starts. The product's Django app registry
- (models, apps) loads with `django.setup()` whatever this file does, so the assertion names
- the modules the guard itself could add: the presentation layer and the control-plane read.
- A module-level import of those would cost every worker and would fail at startup rather
- than at the one call site that needs them.
+ (models, apps) loads with `django.setup()` whatever this file does, so any managed warehouse
+ module that appears after that comes from the guard. The guard reaches the product only at
+ the call: a module-level import of its presentation layer or its control-plane read would
+ cost every worker and would fail at startup rather than at the one call site that needs them.
"""
def test_importing_the_guard_does_not_load_the_control_plane_path(self) -> None:
@@ -446,11 +345,7 @@ class TestNoImportOfTheManagedWarehouseProductAtModuleLoad(SimpleTestCase):
"before = set(sys.modules)\n"
"from products.warehouse_sources.backend.temporal.data_imports.sources.common import mixins\n"
"added = set(sys.modules) - before\n"
- "forbidden = sorted(m for m in added if m in {\n"
- " 'products.managed_warehouse.backend.presentation.views',\n"
- " 'products.managed_warehouse.backend.presentation.hogtower',\n"
- " 'products.managed_warehouse.backend.managed_ingress',\n"
- "})\n"
+ "forbidden = sorted(m for m in added if m.startswith('products.managed_warehouse.'))\n"
"assert not forbidden, forbidden\n"
"print('clean')\n"
)
@@ -481,7 +376,7 @@ class TestPinnedHostKwargsForAManagedWarehouse(SimpleTestCase):
with (
patch(f"{_MIXINS_MODULE}._managed_warehouse_organization_id", return_value="org-1"),
patch(f"{_MIXINS_MODULE}.logger"),
- patch(f"{_INGRESS_MODULE}.get_managed_warehouse_target", return_value=_target()),
+ patch(_TARGET_READ, return_value=_target()),
patch(f"{_MIXINS_MODULE}.host_lookup_is_skipped", return_value=False),
patch(
f"{_MIXINS_MODULE}._resolve_hostaddr_with_timeout",
@@ -502,7 +397,7 @@ class TestPinnedHostKwargsForAManagedWarehouse(SimpleTestCase):
with (
patch(f"{_MIXINS_MODULE}._managed_warehouse_organization_id", return_value="org-1"),
patch(f"{_MIXINS_MODULE}.logger"),
- patch(f"{_INGRESS_MODULE}.get_managed_warehouse_target", return_value=_target()),
+ patch(_TARGET_READ, return_value=_target()),
patch(f"{_MIXINS_MODULE}.host_lookup_is_skipped", return_value=False),
patch(f"{_MIXINS_MODULE}._resolve_hostaddr_with_timeout", return_value=[]),
):
@@ -510,75 +405,6 @@ class TestPinnedHostKwargsForAManagedWarehouse(SimpleTestCase):
pinned_host_kwargs(_TENANT_HOST, port=5432, connect_timeout=5, team_id=999, source_type="postgres")
-class TestDenialCacheSurvivesSerialization(SimpleTestCase):
- """A denial must be recognised after a round trip through the cache backend.
-
- The cache pickles its values, so an `object()` sentinel comes back as a different object and
- an identity comparison never matches. A denial that never hits is worse than no cache: every
- denied dial, and every dial during a control-plane outage, pays the full control-plane
- timeout against an already-sick service.
- """
-
- def test_a_denial_is_recognised_after_a_pickle_round_trip(self) -> None:
- import pickle
-
- from products.managed_warehouse.backend.facade.testing import NO_MANAGED_WAREHOUSE_TARGET
-
- serialized = pickle.dumps(NO_MANAGED_WAREHOUSE_TARGET) # nosemgrep
- restored = pickle.loads(serialized) # nosemgrep
- assert restored == NO_MANAGED_WAREHOUSE_TARGET
-
- def test_a_cached_denial_is_returned_without_a_control_plane_call(self) -> None:
- from products.managed_warehouse.backend.facade.testing import NO_MANAGED_WAREHOUSE_TARGET
-
- with (
- patch(f"{_INGRESS_MODULE}.hogtower") as hogtower,
- patch(f"{_INGRESS_MODULE}.cache") as cache,
- ):
- cache.get.return_value = NO_MANAGED_WAREHOUSE_TARGET
- assert get_managed_warehouse_target("org-1", source_type="postgres") is None
- hogtower.request.assert_not_called()
-
- def test_a_cache_read_fault_denies_instead_of_raising(self) -> None:
- # The guard runs inside a connect. An exception from the cache would replace a clear
- # refusal with an opaque connection error.
- with (
- patch(f"{_INGRESS_MODULE}.cache") as cache,
- patch(f"{_INGRESS_MODULE}.hogtower") as hogtower,
- patch(f"{_INGRESS_MODULE}.logger"),
- ):
- cache.get.side_effect = RuntimeError("redis down")
- hogtower.is_configured.return_value = False
- assert get_managed_warehouse_target("org-1", source_type="postgres") is None
-
- def test_a_cache_write_fault_still_answers_the_dial(self) -> None:
- with (
- patch(f"{_INGRESS_MODULE}.cache") as cache,
- patch(f"{_INGRESS_MODULE}.hogtower") as hogtower,
- patch(f"{_INGRESS_MODULE}.logger"),
- ):
- cache.get.return_value = None
- cache.set.side_effect = RuntimeError("redis down")
- hogtower.is_configured.return_value = True
- hogtower.request.return_value = _response(
- {"org_id": "org-1", "state": "ready", "connection": {"host": _TENANT_HOST}}
- )
- target = get_managed_warehouse_target("org-1", source_type="postgres")
-
- assert target is not None
- assert target.host == _TENANT_HOST
-
- def test_a_control_plane_error_the_adapter_cannot_translate_denies(self) -> None:
- with (
- patch(f"{_INGRESS_MODULE}.cache"),
- patch(f"{_INGRESS_MODULE}.hogtower") as hogtower,
- patch(f"{_INGRESS_MODULE}.logger"),
- ):
- hogtower.is_configured.return_value = True
- hogtower.request.side_effect = KeyError("unmapped route")
- assert get_managed_warehouse_target("org-1", source_type="postgres") is None
-
-
class TestHostComparisonIsNormalized(SimpleTestCase):
"""A control-plane host with a capital or a trailing dot must still match.
@@ -656,7 +482,7 @@ class TestPostgresConnectPathsAgreeWithValidation(SimpleTestCase):
patch(f"{_MIXINS_MODULE}.socket.getaddrinfo", return_value=[(None, None, None, None, (_NLB_A, 0))]),
patch(f"{_MIXINS_MODULE}._managed_warehouse_organization_id", return_value="org-1"),
patch(f"{_MIXINS_MODULE}.logger"),
- patch(f"{_INGRESS_MODULE}.get_managed_warehouse_target", return_value=_target()),
+ patch(_TARGET_READ, return_value=_target()),
patch(
"products.warehouse_sources.backend.temporal.data_imports.sources.postgres.source.get_postgres_schemas",
side_effect=_no_connection,
diff --git a/posthog/test/activity_logging/test_external_data_schema_activity_logging.py b/products/warehouse_sources/backend/tests/test_external_data_schema_activity_logging.py
similarity index 100%
rename from posthog/test/activity_logging/test_external_data_schema_activity_logging.py
rename to products/warehouse_sources/backend/tests/test_external_data_schema_activity_logging.py
diff --git a/products/warehouse_sources/backend/tests/test_file_upload_sources.py b/products/warehouse_sources/backend/tests/test_file_upload_sources.py
new file mode 100644
index 000000000000..3ad2da187ef9
--- /dev/null
+++ b/products/warehouse_sources/backend/tests/test_file_upload_sources.py
@@ -0,0 +1,544 @@
+import io
+
+from posthog.test.base import APIBaseTest
+from unittest.mock import MagicMock, patch
+
+from django.conf import settings
+from django.core.files.uploadedfile import SimpleUploadedFile
+from django.test import override_settings
+
+from parameterized import parameterized
+from rest_framework import status
+
+from posthog.models.team import Team
+
+from products.warehouse_sources.backend.facade.testing import (
+ find_replaced_self_managed_table,
+ migrate_team_self_managed_tables,
+)
+from products.warehouse_sources.backend.models.external_data_job import ExternalDataJob
+from products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchema
+from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource
+from products.warehouse_sources.backend.models.table import DataWarehouseTable
+
+VIEW_MODULE = "products.data_warehouse.backend.presentation.views.table"
+MODEL_MODULE = "products.warehouse_sources.backend.models.table"
+UPLOAD_ID = "11111111-2222-4333-8444-555555555555"
+NEW_UPLOAD_ID = "22222222-2222-4333-8444-555555555555"
+
+# Stand-in for what `DataWarehouseTable.get_columns()` reads off the object in place — patched so the
+# create tests never reach chdb/ClickHouse, which can't introspect a fake S3 object.
+FAKE_COLUMNS = {
+ "id": {"hogql": "IntegerDatabaseField", "clickhouse": "Nullable(Int64)"},
+ "total": {"hogql": "IntegerDatabaseField", "clickhouse": "Nullable(Int64)"},
+}
+
+
+class _CapturingBuffer(io.BytesIO):
+ """A BytesIO that publishes its contents to a sink on close, so the endpoint's
+ context-managed write is captured once the file handle is closed."""
+
+ def __init__(self, sink: dict[str, bytes], path: str) -> None:
+ super().__init__()
+ self._sink = sink
+ self._path = path
+
+ def close(self) -> None:
+ self._sink[self._path] = self.getvalue()
+ super().close()
+
+
+class _FakeS3:
+ """Captures what the endpoint writes, standing in for the object store boundary."""
+
+ def __init__(self, *, exists_result: bool = True) -> None:
+ self.written: dict[str, bytes] = {}
+ self.removed: list[str] = []
+ self._exists_result = exists_result
+
+ def open(self, path: str, mode: str) -> io.BytesIO:
+ return _CapturingBuffer(self.written, path)
+
+ def exists(self, path: str) -> bool:
+ return self._exists_result
+
+ def rm(self, path: str) -> None:
+ self.removed.append(path)
+
+
+SERVICE_MODULE = "products.warehouse_sources.backend.file_upload_sources"
+STORAGE_MODULE = "products.warehouse_sources.backend.temporal.data_imports.sources.file_upload.storage"
+SOURCE_VIEWS = "products.warehouse_sources.backend.presentation.views.external_data_source.base"
+REPLACEMENT_COLUMNS = {
+ "id": {"hogql": "IntegerDatabaseField", "clickhouse": "Nullable(Int64)"},
+ "placed_at": {"hogql": "DateTimeDatabaseField", "clickhouse": "Nullable(DateTime64(6))"},
+}
+
+
+def _warehouse_api_flag(flag_key: str, *args: object, **kwargs: object) -> bool:
+ return flag_key == "warehouse-api"
+
+
+@override_settings(DATAWAREHOUSE_BUCKET="test-bucket", DATAWAREHOUSE_BUCKET_DOMAIN="warehouse.posthog.test")
+class TestFileUploadSources(APIBaseTest):
+ def setUp(self) -> None:
+ super().setUp()
+ self.base_url = f"/api/environments/{self.team.pk}/warehouse_tables"
+ self.s3 = _FakeS3()
+ self.trigger_count = 0
+ patches = [
+ patch("posthoganalytics.feature_enabled", return_value=False),
+ patch(f"{VIEW_MODULE}.get_s3_client", return_value=self.s3),
+ patch(f"{STORAGE_MODULE}.get_s3_client", return_value=self.s3),
+ patch(f"{VIEW_MODULE}.validate_data_warehouse_table_columns.delay"),
+ patch(f"{MODEL_MODULE}.DataWarehouseTable.detect_csv_double_quotes_setting", return_value=False),
+ patch(f"{SERVICE_MODULE}.sync_connect"),
+ patch(f"{SERVICE_MODULE}.sync_external_data_job_workflow"),
+ patch(f"{SERVICE_MODULE}.trigger_ad_hoc_sync", side_effect=self._trigger),
+ ]
+ mocks = [p.start() for p in patches]
+ for p in patches:
+ self.addCleanup(p.stop)
+ self.flag, self.schedule, self.trigger = mocks[0], mocks[6], mocks[7]
+ self.columns = patch(f"{MODEL_MODULE}.DataWarehouseTable.get_columns", return_value=dict(FAKE_COLUMNS))
+ self.get_columns = self.columns.start()
+ self.addCleanup(self.columns.stop)
+
+ def _trigger(self, *_args: object, **_kwargs: object) -> MagicMock:
+ self.trigger_count += 1
+ return MagicMock(workflow_id=f"workflow-{self.trigger_count}", schedule_paused_now=False)
+
+ def _create(self, upload_id: str = UPLOAD_ID, **payload: str):
+ return self.client.post(
+ f"{self.base_url}/create_from_upload/?include_columns=false",
+ data={"upload_id": upload_id, "filename": "orders.csv", "file_format": "csv", "table_name": "orders"}
+ | payload,
+ format="json",
+ )
+
+ def _table_with_source(self) -> tuple[DataWarehouseTable, ExternalDataSchema]:
+ response = self._create()
+ assert response.status_code == status.HTTP_201_CREATED, response.json()
+ table = DataWarehouseTable.objects.get(id=response.json()["id"])
+ schema = ExternalDataSchema.objects.select_related("source").get(table=table)
+ return table, schema
+
+ def _replace(self, table: DataWarehouseTable, upload_id: str = NEW_UPLOAD_ID, **payload: str):
+ return self.client.post(
+ f"{self.base_url}/{table.id}/replace_file/?include_columns=false",
+ data={"upload_id": upload_id, "filename": "orders_v2.csv", "file_format": "csv"} | payload,
+ format="json",
+ )
+
+ def test_an_upload_becomes_a_file_upload_source_and_one_billable_sync(self) -> None:
+ response = self._create()
+
+ assert response.status_code == status.HTTP_201_CREATED, response.json()
+ table = DataWarehouseTable.objects.get(id=response.json()["id"])
+ # Until the first sync imports it, the table reads the file in place.
+ assert (table.name, table.external_data_source_id, table.credential_id) == ("orders", None, None)
+ assert (
+ table.url_pattern
+ == f"https://warehouse.posthog.test/file_uploads/team_{self.team.pk}/{UPLOAD_ID}/orders.csv"
+ )
+ assert response.json()["external_schema"] is not None
+
+ schema = ExternalDataSchema.objects.select_related("source").get(table=table)
+ assert schema.source.source_type == "FileUpload"
+ assert schema.source.job_inputs == {
+ "upload_id": UPLOAD_ID,
+ "filename": "orders.csv",
+ "file_format": "csv",
+ "schema_name": "orders",
+ }
+ assert (schema.name, schema.should_sync, schema.sync_type) == ("orders", False, "full_refresh")
+ assert schema.schema_metadata is not None
+ assert schema.schema_metadata["csv"] == {
+ "has_header": True,
+ "allow_double_quotes": False,
+ "column_types": {"id": "integer", "total": "integer"},
+ }
+ self.schedule.assert_called_once_with(schema, create=True, should_sync=False, trigger_immediately=False)
+ assert self.trigger.call_args.kwargs == {
+ "billable": True,
+ "reset_pipeline": False,
+ "workflow_id_prefix": "file-upload",
+ }
+
+ def test_a_sync_that_fails_to_start_still_leaves_a_working_table(self) -> None:
+ self.trigger.side_effect = RuntimeError("temporal is down")
+
+ response = self._create()
+
+ assert response.status_code == status.HTTP_201_CREATED, response.json()
+ table = DataWarehouseTable.objects.get(id=response.json()["id"])
+ assert ExternalDataSchema.objects.filter(table=table, source__source_type="FileUpload").exists()
+
+ @parameterized.expand(
+ [
+ ("traversal", "../orders.csv"),
+ ("other_folder", "team_1/orders.csv"),
+ ("hidden", ".orders.csv"),
+ ]
+ )
+ def test_a_filename_upload_file_could_not_have_returned_is_refused(self, _name: str, filename: str) -> None:
+ response = self._create(filename=filename)
+
+ assert response.status_code == status.HTTP_400_BAD_REQUEST
+ assert not DataWarehouseTable.objects.filter(team=self.team).exists()
+
+ def test_replacing_the_file_keeps_the_table_and_starts_a_billable_full_refresh(self) -> None:
+ table, schema = self._table_with_source()
+ self.get_columns.return_value = dict(REPLACEMENT_COLUMNS)
+
+ response = self._replace(table)
+
+ assert response.status_code == status.HTTP_202_ACCEPTED, response.json()
+ assert response.json()["id"] == str(table.id)
+ schema.refresh_from_db()
+ schema.source.refresh_from_db()
+ assert schema.source.job_inputs["upload_id"] == NEW_UPLOAD_ID
+ assert schema.source.job_inputs["filename"] == "orders_v2.csv"
+ assert schema.schema_metadata is not None
+ assert schema.schema_metadata["csv"]["column_types"] == {"id": "integer", "placed_at": "datetime"}
+ [superseded] = (schema.source.connection_metadata or {})["file_upload"]["superseded"]
+ assert (superseded["upload_id"], superseded["filename"]) == (UPLOAD_ID, "orders.csv")
+ assert self.trigger_count == 2
+ assert self.trigger.call_args.kwargs["billable"] is True
+ assert schema.sync_type == "full_refresh"
+ # The old file stays until a sync of the new one succeeds.
+ assert self.s3.removed == []
+ table.refresh_from_db()
+ assert table.name == "orders"
+
+ def test_replacing_the_file_of_an_append_schema_switches_it_to_full_refresh(self) -> None:
+ table, schema = self._table_with_source()
+ ExternalDataSchema.objects.filter(pk=schema.pk).update(sync_type=ExternalDataSchema.SyncType.APPEND)
+ self.get_columns.return_value = dict(REPLACEMENT_COLUMNS)
+
+ response = self._replace(table)
+
+ assert response.status_code == status.HTTP_202_ACCEPTED, response.json()
+ schema.refresh_from_db()
+ assert schema.sync_type == ExternalDataSchema.SyncType.FULL_REFRESH
+
+ @parameterized.expand(["missing_file", "unreadable_file", "sync_running", "not_a_file_upload_table"])
+ def test_a_replacement_that_cannot_go_ahead_changes_nothing(self, case: str) -> None:
+ table, schema = self._table_with_source()
+ if case == "missing_file":
+ self.s3._exists_result = False
+ elif case == "unreadable_file":
+ self.get_columns.side_effect = Exception("DB::Exception: Cannot extract table structure")
+ elif case == "sync_running":
+ ExternalDataJob.objects.create(
+ team=self.team, pipeline=schema.source, schema=schema, status=ExternalDataJob.Status.RUNNING
+ )
+ else:
+ table = DataWarehouseTable.objects.create(
+ team=self.team,
+ name="linked",
+ format="CSVWithNames",
+ url_pattern=f"https://warehouse.posthog.test/file_uploads/team_{self.team.pk}/{UPLOAD_ID}/orders.csv",
+ columns={},
+ )
+
+ response = self._replace(table)
+
+ assert response.status_code == status.HTTP_400_BAD_REQUEST, response.json()
+ schema.source.refresh_from_db()
+ assert schema.source.job_inputs["upload_id"] == UPLOAD_ID
+ assert "file_upload" not in (schema.source.connection_metadata or {})
+ assert self.trigger_count == 1
+
+ def test_the_source_update_endpoint_cannot_repoint_the_upload(self) -> None:
+ _, schema = self._table_with_source()
+
+ response = self.client.patch(
+ f"/api/environments/{self.team.pk}/external_data_sources/{schema.source_id}/",
+ {
+ "job_inputs": {
+ "upload_id": NEW_UPLOAD_ID,
+ "filename": "../../team_1/secret.csv",
+ "file_format": "parquet",
+ "schema_name": "other",
+ }
+ },
+ format="json",
+ )
+
+ assert response.status_code == status.HTTP_200_OK, response.json()
+ schema.source.refresh_from_db()
+ assert schema.source.job_inputs == {
+ "upload_id": UPLOAD_ID,
+ "filename": "orders.csv",
+ "file_format": "csv",
+ "schema_name": "orders",
+ }
+
+ def test_the_generic_source_endpoint_cannot_create_one(self) -> None:
+ response = self.client.post(
+ f"/api/environments/{self.team.pk}/external_data_sources/",
+ {
+ "source_type": "FileUpload",
+ "payload": {
+ "upload_id": UPLOAD_ID,
+ "filename": "orders.csv",
+ "file_format": "csv",
+ "schema_name": "orders",
+ },
+ },
+ format="json",
+ )
+
+ assert response.status_code == status.HTTP_400_BAD_REQUEST
+ assert "can't be created here" in response.json()["message"]
+ assert not ExternalDataSource.objects.filter(team=self.team).exists()
+
+ def test_its_schedule_cannot_be_turned_on(self) -> None:
+ _, schema = self._table_with_source()
+
+ response = self.client.patch(
+ f"/api/environments/{self.team.pk}/external_data_schemas/{schema.id}/",
+ {"should_sync": True, "sync_frequency": "24hour"},
+ format="json",
+ )
+
+ assert response.status_code == status.HTTP_400_BAD_REQUEST
+ assert "can't be turned on" in str(response.json())
+ schema.refresh_from_db()
+ assert schema.should_sync is False
+
+ def test_its_table_is_deleted_through_the_source(self) -> None:
+ table, _ = self._table_with_source()
+
+ response = self.client.delete(f"{self.base_url}/{table.id}")
+
+ assert response.status_code == status.HTTP_400_BAD_REQUEST
+ table.refresh_from_db()
+ assert table.deleted is False
+ assert self.s3.removed == []
+
+ @patch(f"{SOURCE_VIEWS}.delete_discover_schemas_schedule")
+ @patch(f"{SOURCE_VIEWS}.delete_external_data_schedule")
+ @patch(f"{SOURCE_VIEWS}.bulk_delete_external_data_schedules", return_value=[])
+ def test_deleting_the_source_deletes_the_table_and_its_files(self, *_mocks: object) -> None:
+ table, schema = self._table_with_source()
+ self.get_columns.return_value = dict(REPLACEMENT_COLUMNS)
+ assert self._replace(table).status_code == status.HTTP_202_ACCEPTED
+
+ with self.captureOnCommitCallbacks(execute=True):
+ response = self.client.delete(f"/api/environments/{self.team.pk}/external_data_sources/{schema.source_id}")
+
+ assert response.status_code == status.HTTP_204_NO_CONTENT
+ table.refresh_from_db()
+ assert table.deleted is True
+ prefix = f"{settings.DATAWAREHOUSE_BUCKET}/file_uploads/team_{self.team.pk}/"
+ assert sorted(self.s3.removed) == sorted(
+ [f"{prefix}{NEW_UPLOAD_ID}/orders_v2.csv", f"{prefix}{UPLOAD_ID}/orders.csv"]
+ )
+
+ def _migrated_table(self, key: str) -> tuple[DataWarehouseTable, ExternalDataSchema]:
+ table = DataWarehouseTable.objects.create(
+ team=self.team,
+ name="orders",
+ format="CSVWithNames",
+ url_pattern=f"https://warehouse.posthog.test/{key.format(team=self.team.pk, upload=UPLOAD_ID)}",
+ columns=dict(FAKE_COLUMNS),
+ options={"csv_allow_double_quotes": False},
+ )
+ [outcome] = migrate_team_self_managed_tables(self.team.pk)
+ assert outcome.action == "linked", outcome
+ assert outcome.status.schema_id is not None
+ schema = ExternalDataSchema.objects.select_related("source").get(pk=outcome.status.schema_id)
+ return table, schema
+
+ def _legacy_file_action(self, filename: str = "orders_v2.csv"):
+ return self.client.post(
+ f"{self.base_url}/file/",
+ {"file": SimpleUploadedFile(filename, b"id,total\n1,2\n"), "name": "orders", "format": "CSVWithNames"},
+ format="multipart",
+ )
+
+ @parameterized.expand(
+ [
+ ("upload_folder_replace_file", "file_uploads/team_{team}/{upload}/orders.csv", "replace_file"),
+ ("legacy_folder_replace_file", "managed/team_{team}/orders.csv", "replace_file"),
+ ("legacy_folder_file_action", "managed/team_{team}/orders.csv", "file"),
+ ]
+ )
+ def test_a_new_file_for_a_migrated_table_starts_the_sync_that_adopts_it(
+ self, _name: str, key: str, path: str
+ ) -> None:
+ self.flag.side_effect = _warehouse_api_flag
+ table, schema = self._migrated_table(key)
+ old_url = table.url_pattern
+ self.get_columns.return_value = dict(REPLACEMENT_COLUMNS)
+
+ with patch(f"{VIEW_MODULE}.boto3.client") as legacy_s3:
+ if path == "replace_file":
+ response = self._replace(table)
+ else:
+ response = self._legacy_file_action()
+
+ expected_status = status.HTTP_202_ACCEPTED if path == "replace_file" else status.HTTP_201_CREATED
+ assert response.status_code == expected_status, response.json()
+ assert response.json()["id"] == str(table.id)
+ # The legacy action no longer writes over the file the table reads.
+ legacy_s3.assert_not_called()
+ schema.refresh_from_db()
+ schema.source.refresh_from_db()
+ job_inputs = schema.source.job_inputs
+ if path == "replace_file":
+ assert job_inputs["upload_id"] == NEW_UPLOAD_ID
+ else:
+ [written] = self.s3.written
+ assert written == (f"test-bucket/file_uploads/team_{self.team.pk}/{job_inputs['upload_id']}/orders_v2.csv")
+ assert (job_inputs["filename"], job_inputs["file_format"]) == ("orders_v2.csv", "csv")
+ [superseded] = (schema.source.connection_metadata or {})["file_upload"]["superseded"]
+ if key.startswith("managed/"):
+ assert superseded["legacy_filename"] == "orders.csv"
+ else:
+ assert (superseded["upload_id"], superseded["filename"]) == (UPLOAD_ID, "orders.csv")
+ assert schema.schema_metadata is not None
+ assert schema.schema_metadata["csv"]["column_types"] == {"id": "integer", "placed_at": "datetime"}
+ assert schema.schema_metadata["self_managed_table"]["table_id"] == str(table.id)
+ self.schedule.assert_called_once_with(schema, create=True, should_sync=False, trigger_immediately=False)
+ assert self.trigger_count == 1
+ assert self.trigger.call_args.kwargs["billable"] is True
+ # Until that sync publishes, the table reads its old file, and the sync still adopts the same row.
+ table.refresh_from_db()
+ assert (table.name, table.url_pattern, table.external_data_source_id) == ("orders", old_url, None)
+ assert find_replaced_self_managed_table(schema, schema.source) == table
+ assert self.s3.removed == []
+
+ @parameterized.expand(["sync_running", "table_changed_since_migration"])
+ def test_a_migrated_table_refuses_a_new_file_it_cant_take(self, case: str) -> None:
+ table, schema = self._migrated_table("managed/team_{team}/orders.csv")
+ if case == "sync_running":
+ ExternalDataJob.objects.create(
+ team=self.team, pipeline=schema.source, schema=schema, status=ExternalDataJob.Status.RUNNING
+ )
+ else:
+ DataWarehouseTable.objects.filter(pk=table.pk).update(
+ url_pattern=f"https://warehouse.posthog.test/managed/team_{self.team.pk}/other.csv"
+ )
+
+ response = self._replace(table)
+
+ assert response.status_code == status.HTTP_400_BAD_REQUEST, response.json()
+ schema.source.refresh_from_db()
+ assert schema.source.job_inputs["upload_id"] == ""
+ assert "file_upload" not in (schema.source.connection_metadata or {})
+ self.trigger.assert_not_called()
+
+ def test_a_migrated_source_can_be_updated_before_its_next_upload(self) -> None:
+ _, schema = self._migrated_table("managed/team_{team}/orders.csv")
+
+ response = self.client.patch(
+ f"/api/environments/{self.team.pk}/external_data_sources/{schema.source_id}/",
+ {"description": "Orders", "job_inputs": {"upload_id": NEW_UPLOAD_ID, "filename": "orders_v2.csv"}},
+ format="json",
+ )
+
+ assert response.status_code == status.HTTP_200_OK, response.json()
+ schema.source.refresh_from_db()
+ assert schema.source.description == "Orders"
+ assert (schema.source.job_inputs["upload_id"], schema.source.job_inputs["filename"]) == ("", "")
+
+ def test_a_refused_legacy_file_action_deletes_the_file_it_stored(self) -> None:
+ self.flag.side_effect = _warehouse_api_flag
+ _, schema = self._migrated_table("managed/team_{team}/orders.csv")
+ ExternalDataJob.objects.create(
+ team=self.team, pipeline=schema.source, schema=schema, status=ExternalDataJob.Status.RUNNING
+ )
+ self.get_columns.return_value = dict(REPLACEMENT_COLUMNS)
+
+ with patch(f"{VIEW_MODULE}.boto3.client"):
+ response = self._legacy_file_action()
+
+ assert response.status_code == status.HTTP_400_BAD_REQUEST, response.json()
+ [written] = self.s3.written
+ assert self.s3.removed == [written]
+ schema.source.refresh_from_db()
+ assert schema.source.job_inputs["upload_id"] == ""
+ self.trigger.assert_not_called()
+
+ @parameterized.expand(["reload", "resync"])
+ def test_a_migrated_table_cant_sync_before_its_next_upload(self, action: str) -> None:
+ _, schema = self._migrated_table("managed/team_{team}/orders.csv")
+
+ response = self.client.post(f"/api/environments/{self.team.pk}/external_data_schemas/{schema.id}/{action}/")
+
+ assert response.status_code == status.HTTP_400_BAD_REQUEST
+ assert "Upload the file" in response.json()["message"]
+ assert not ExternalDataJob.objects.filter(team=self.team).exists()
+ self.trigger.assert_not_called()
+
+ @parameterized.expand(["reload", "resync"])
+ def test_a_table_that_imported_its_file_points_to_replace_file(self, action: str) -> None:
+ _, schema = self._table_with_source()
+ ExternalDataJob.objects.create(
+ team=self.team, pipeline=schema.source, schema=schema, status=ExternalDataJob.Status.COMPLETED
+ )
+ self.trigger.reset_mock()
+
+ response = self.client.post(f"/api/environments/{self.team.pk}/external_data_schemas/{schema.id}/{action}/")
+
+ assert response.status_code == status.HTTP_400_BAD_REQUEST
+ assert "Replace file" in response.json()["message"]
+ self.trigger.assert_not_called()
+
+ @parameterized.expand(
+ [
+ ("linked", None),
+ ("pinned", "managed/team_{team}/orders.csv"),
+ ("pinned_in_upload_folder", "file_uploads/team_{team}/{upload}/orders.csv"),
+ ]
+ )
+ def test_the_schema_and_source_name_the_table_and_the_file(self, _name: str, migrated_key: str | None) -> None:
+ if migrated_key is None:
+ table, schema = self._table_with_source()
+ else:
+ table, schema = self._migrated_table(migrated_key)
+ assert schema.table_id is None
+
+ schema_body = self.client.get(f"/api/environments/{self.team.pk}/external_data_schemas/{schema.id}/").json()
+ source_body = self.client.get(
+ f"/api/environments/{self.team.pk}/external_data_sources/{schema.source_id}/"
+ ).json()
+
+ assert schema_body["table_id"] == str(table.id)
+ assert source_body["schemas"][0]["table_id"] == str(table.id)
+ assert source_body["file_name"] == "orders.csv"
+ assert schema_body["source"]["file_name"] == "orders.csv"
+ assert source_body["job_inputs"] == {}
+
+ @parameterized.expand(["other_team", "deleted"])
+ def test_a_pinned_table_that_is_not_live_in_the_team_is_not_returned(self, case: str) -> None:
+ table, schema = self._migrated_table("managed/team_{team}/orders.csv")
+ if case == "deleted":
+ DataWarehouseTable.objects.filter(pk=table.pk).update(deleted=True)
+ else:
+ other_team = Team.objects.create(organization=self.organization, name="Other")
+ DataWarehouseTable.objects.filter(pk=table.pk).update(team=other_team)
+
+ body = self.client.get(f"/api/environments/{self.team.pk}/external_data_schemas/{schema.id}/").json()
+
+ assert body["table_id"] is None
+
+ @patch(f"{SOURCE_VIEWS}.delete_discover_schemas_schedule")
+ @patch(f"{SOURCE_VIEWS}.delete_external_data_schedule")
+ @patch(f"{SOURCE_VIEWS}.bulk_delete_external_data_schedules", return_value=[])
+ def test_deleting_a_migrated_source_never_deletes_the_file_its_table_reads(self, *_mocks: object) -> None:
+ table, schema = self._migrated_table("managed/team_{team}/orders.csv")
+ self.get_columns.return_value = dict(REPLACEMENT_COLUMNS)
+ # The sync of the new file never finishes, so the table still reads the old one.
+ assert self._replace(table).status_code == status.HTTP_202_ACCEPTED
+
+ with self.captureOnCommitCallbacks(execute=True):
+ response = self.client.delete(f"/api/environments/{self.team.pk}/external_data_sources/{schema.source_id}")
+
+ assert response.status_code == status.HTTP_204_NO_CONTENT
+ table.refresh_from_db()
+ assert table.deleted is False
+ assert self.s3.removed == [f"test-bucket/file_uploads/team_{self.team.pk}/{NEW_UPLOAD_ID}/orders_v2.csv"]
diff --git a/products/warehouse_sources/backend/tests/test_source_credentials.py b/products/warehouse_sources/backend/tests/test_source_credentials.py
index 69c543f41e96..d3c9a69f7f1a 100644
--- a/products/warehouse_sources/backend/tests/test_source_credentials.py
+++ b/products/warehouse_sources/backend/tests/test_source_credentials.py
@@ -41,7 +41,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.job
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistry
from products.warehouse_sources.backend.types import ExternalDataSourceType
-from products.workflows.backend.facade.contracts import WorkflowSummary
+from products.workflows.backend.facade.testing import create_workflow_for_test
PG_FIELDS: dict[str, Any] = {
"host": "db.example.com",
@@ -374,25 +374,27 @@ class TestWarehouseSourceCredentialsStore(BaseTest):
pk = self._create(source, PG_FIELDS)
if other_user == "source":
self._referencing(pk)
- workflows = (
- [WorkflowSummary(id="wf1", name="Nightly sync", status="active")] if other_user == "workflow" else []
- )
+ if other_user == "workflow":
+ create_workflow_for_test(
+ team_id=self.team.pk,
+ name="Nightly sync",
+ status="active",
+ actions=[
+ {"id": "query", "type": "function", "config": {"inputs": {"db": {"value": {"integrationId": pk}}}}}
+ ],
+ )
- with patch(
- "products.workflows.backend.integration_usages.get_active_workflows_using_integration",
- return_value=workflows,
- ):
- if outcome == "conflict":
- with self.assertRaises(SharedCredentialsError) as raised:
- WarehouseSourceCredentialsStore.update(source, new_fields, allow_shared=allow_shared)
- assert raised.exception.integration_id == pk
- assert [usage.type for usage in raised.exception.usages] == [
- "workflow" if other_user == "workflow" else "warehouse_source"
- ]
- assert source.credentials_integration_id == str(pk)
- assert WarehouseSourceCredentialsStore.get(source) == PG_FIELDS
- return
- reference = WarehouseSourceCredentialsStore.update(source, new_fields, allow_shared=allow_shared)
+ if outcome == "conflict":
+ with self.assertRaises(SharedCredentialsError) as raised:
+ WarehouseSourceCredentialsStore.update(source, new_fields, allow_shared=allow_shared)
+ assert raised.exception.integration_id == pk
+ assert [usage.type for usage in raised.exception.usages] == [
+ "workflow" if other_user == "workflow" else "warehouse_source"
+ ]
+ assert source.credentials_integration_id == str(pk)
+ assert WarehouseSourceCredentialsStore.get(source) == PG_FIELDS
+ return
+ reference = WarehouseSourceCredentialsStore.update(source, new_fields, allow_shared=allow_shared)
new_pk = reference.legacy_id
assert new_pk is not None
diff --git a/products/warehouse_sources/backend/tests/test_sync_alerts.py b/products/warehouse_sources/backend/tests/test_sync_alerts.py
index 40eb0ec97dcb..aed3ab76d478 100644
--- a/products/warehouse_sources/backend/tests/test_sync_alerts.py
+++ b/products/warehouse_sources/backend/tests/test_sync_alerts.py
@@ -5,12 +5,14 @@ from uuid import UUID
from posthog.test.base import APIBaseTest
from unittest.mock import patch
+from django.core import mail
from django.utils import timezone
from parameterized import parameterized
from rest_framework import status
from posthog.constants import AvailableFeature
+from posthog.models.instance_setting import set_instance_setting
from posthog.models.organization import Organization, OrganizationMembership
from posthog.models.scoping import team_scope
from posthog.models.team.team import Team
@@ -720,13 +722,13 @@ class TestWarehouseAlerts(APIBaseTest):
record_outcomes(self.team.id, list(evaluation.outcomes), self.cutoff)
(delivery,) = evaluation.deliveries
- with patch("products.alerts_platform.backend.delivery.email.EmailMessage") as email_message:
+ set_instance_setting("EMAIL_HOST", "localhost")
+ with self.settings(EMAIL_ENABLED=True):
sent = platform_testing.deliver(delivery)
assert sent == 1
assert delivery.configuration_id == str(configuration_id)
- assert "Sync failures" in email_message.call_args.kwargs["subject"]
- assert [call.kwargs["email"] for call in email_message.return_value.add_recipient.call_args_list] == sorted(
+ assert all("Sync failures" in message.subject for message in mail.outbox)
+ assert sorted(address for message in mail.outbox for address in message.to) == sorted(
[self.user.email, "[email protected]"]
)
- email_message.return_value.send.assert_called_once_with()
diff --git a/products/warehouse_sources/turbo.json b/products/warehouse_sources/turbo.json
index fe1a9ba0cf2a..42edd16156d5 100644
--- a/products/warehouse_sources/turbo.json
+++ b/products/warehouse_sources/turbo.json
@@ -11,8 +11,6 @@
"backend/models/**",
"backend/migrations/**",
"backend/tasks/**",
- "backend/activity_logging.py",
- "backend/file_upload_sources.py",
"backend/temporal/data_imports/naming_convention.py",
"backend/temporal/data_imports/pipelines/core/account_property_row_sink.py",
"backend/temporal/data_imports/pipelines/core/consts.py",
@@ -35,7 +33,6 @@
"backend/temporal/data_imports/sources/bigquery/**",
"backend/temporal/data_imports/sources/clickhouse/**",
"backend/temporal/data_imports/sources/common/**",
- "backend/temporal/data_imports/sources/file_upload/storage.py",
"backend/temporal/data_imports/sources/google_search_console/**",
"backend/temporal/data_imports/sources/motherduck/**",
"backend/temporal/data_imports/sources/mssql/**",
diff --git a/pyproject.toml b/pyproject.toml
index 489107af1f00..e897f3eae303 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -820,14 +820,6 @@ ignore_imports = [
"products.actions.backend.presentation.views.action -> products.actions.backend.models.selector_match_change",
"products.actions.backend.presentation.views.action -> products.cdp.backend.models.hog_functions.hog_function",
"products.actions.backend.presentation.views.action -> products.cohorts.backend.models.cohort",
- # TODO: signals presentation wave. The activity summary viewset reads its modes, limits and
- # render types from backend/activity_summary/ directly. That directory has no __init__.py, so
- # grimp cannot see these edges and only the AST surface check in `hogli product:lint` holds
- # them. Expose what the view needs through the facade and delete each entry.
- "products.signals.backend.presentation.activity_summary -> products.signals.backend.activity_summary.prompt",
- "products.signals.backend.presentation.activity_summary -> products.signals.backend.activity_summary.read",
- "products.signals.backend.presentation.activity_summary -> products.signals.backend.activity_summary.render",
- "products.signals.backend.presentation.activity_summary -> products.signals.backend.activity_summary.search",
]
# routes.py is the one module core reads to assemble a product's API router, so it is the