Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

refactor(data-warehouse): use the integrations SDK in warehouse sources and batch exports (#124997)

Tom Owersb9b32a367409
diff --git a/.agents/skills/implementing-warehouse-sources/SKILL.md b/.agents/skills/implementing-warehouse-sources/SKILL.md
index 017cef242467..826ce2ae0a2b 100644
--- a/.agents/skills/implementing-warehouse-sources/SKILL.md
+++ b/.agents/skills/implementing-warehouse-sources/SKILL.md
@@ -908,7 +908,7 @@ client where the driver allows it (see `pinned_host_kwargs` in `common/mixins.py
From `products/warehouse_sources/backend/temporal/data_imports/sources/common/mixins.py`:
- `SSHTunnelMixin` — `with_ssh_tunnel()` context plus `make_ssh_tunnel_func()` for deferred tunnel opening. Both re-check the SSH host and the database host on every open.
-- `OAuthMixin` — `get_oauth_integration()` to pull `Integration` from the DB.
+- `OAuthMixin` — `get_oauth_integration()` to pull the `Integration` row from the DB. It raises `ValueError("Integration not found: <id>")` when the row is gone or is not an OAuth row. Code that reads a row directly uses `IntegrationManager` from `posthog.integrations`, for example `IntegrationManager.google_ads.oauth.get(team_id, integration_id)`, and catches `IntegrationNotFound`. Never use `Integration.objects`.
- `ValidateDatabaseHostMixin` — `is_database_host_valid()` to block internal VPC IPs (unless SSH tunnel is used).
## Icons
diff --git a/posthog/integrations/model_access_baseline.txt b/posthog/integrations/model_access_baseline.txt
index 043ac04f673f..bfb79ce0daad 100644
--- a/posthog/integrations/model_access_baseline.txt
+++ b/posthog/integrations/model_access_baseline.txt
@@ -30,18 +30,7 @@ products/alerts/backend/presentation/views/alert.py 2
products/alerts_platform/backend/delivery/slack.py 2
products/batch_exports/backend/debug/debugger.py 1
products/batch_exports/backend/debug/start_session.py 1
-products/batch_exports/backend/facade/api.py 2
-products/batch_exports/backend/logic/exports.py 3
products/batch_exports/backend/models/batch_export.py 1
-products/batch_exports/backend/presentation/views/batch_export/destinations.py 3
-products/batch_exports/backend/presentation/views/batch_export/exports.py 1
-products/batch_exports/backend/temporal/destinations/azure_blob_batch_export.py 2
-products/batch_exports/backend/temporal/destinations/bigquery_batch_export.py 2
-products/batch_exports/backend/temporal/destinations/databricks_batch_export.py 2
-products/batch_exports/backend/temporal/destinations/postgres_batch_export.py 2
-products/batch_exports/backend/temporal/destinations/redshift_batch_export.py 3
-products/batch_exports/backend/temporal/destinations/s3_batch_export.py 2
-products/batch_exports/backend/temporal/destinations/snowflake_batch_export.py 2
products/billing_alerts/backend/facade/api.py 2
products/business_knowledge/backend/github_repos.py 2
products/canvas/backend/welcome.py 2
@@ -190,32 +179,7 @@ products/tasks/scripts/run_agent_in_docker.py 2
products/visual_review/backend/logic/github_api.py 2
products/visual_review/backend/logic/team_channels.py 2
products/warehouse_sources/backend/models/external_data_destination.py 1
-products/warehouse_sources/backend/presentation/destination_connection_check.py 1
-products/warehouse_sources/backend/presentation/destination_connection_testing.py 1
-products/warehouse_sources/backend/presentation/views/external_data_destination.py 2
-products/warehouse_sources/backend/presentation/views/external_data_source/oauth_accounts.py 2
-products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/bigquery.py 2
-products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/databricks.py 2
-products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/postgres.py 2
-products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/redshift.py 1
-products/warehouse_sources/backend/temporal/data_imports/sources/apple_search_ads/oauth.py 1
-products/warehouse_sources/backend/temporal/data_imports/sources/bigquery/bigquery.py 2
-products/warehouse_sources/backend/temporal/data_imports/sources/common/mixins.py 2
products/warehouse_sources/backend/temporal/data_imports/sources/custom/source.py 1
-products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/google_ads.py 2
-products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/source.py 1
-products/warehouse_sources/backend/temporal/data_imports/sources/google_analytics/google_analytics.py 2
-products/warehouse_sources/backend/temporal/data_imports/sources/google_analytics/source.py 1
-products/warehouse_sources/backend/temporal/data_imports/sources/google_search_console/google_search_console.py 2
-products/warehouse_sources/backend/temporal/data_imports/sources/google_search_console/source.py 1
-products/warehouse_sources/backend/temporal/data_imports/sources/helpscout/source.py 1
-products/warehouse_sources/backend/temporal/data_imports/sources/linkedin_ads/linkedin_ads.py 2
-products/warehouse_sources/backend/temporal/data_imports/sources/linkedin_ads/source.py 2
-products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/meta_ads.py 2
-products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/source.py 1
-products/warehouse_sources/backend/temporal/data_imports/sources/resend/oauth.py 2
-products/warehouse_sources/backend/temporal/data_imports/sources/slack/source.py 1
-products/warehouse_sources/backend/temporal/data_imports/sources/twitter_ads/twitter_ads.py 1
products/wizard/backend/logic/workers/repository_publisher.py 3
products/workflows/backend/management/commands/delete_stale_ses_identity.py 2
products/workflows/backend/management/commands/disable_ses_feedback_forwarding.py 2
diff --git a/products/batch_exports/backend/facade/api.py b/products/batch_exports/backend/facade/api.py
index df64a26e8fba..b59ac56816c1 100644
--- a/products/batch_exports/backend/facade/api.py
+++ b/products/batch_exports/backend/facade/api.py
@@ -33,8 +33,8 @@ from django.db.models.functions import Coalesce
import structlog
+from posthog.integrations import IntegrationManager
from posthog.models import Team, User
-from posthog.models.integration import Integration
from products.batch_exports.backend import (
destination_tests,
@@ -51,6 +51,7 @@ from products.batch_exports.backend.models.batch_export import (
)
from . import contracts
+from .enums import BATCH_EXPORT_INTEGRATION_KINDS
if TYPE_CHECKING:
from temporalio.client import Client
@@ -635,7 +636,7 @@ def run_destination_test_step(
test_configuration = dict(config)
if integration_id is not None:
- integration = Integration.objects.get(id=integration_id, team_id=team_id)
+ integration = IntegrationManager.kinds(*BATCH_EXPORT_INTEGRATION_KINDS).get(team_id, integration_id)
test_configuration = {
**test_configuration,
**integration.config,
diff --git a/products/batch_exports/backend/facade/enums.py b/products/batch_exports/backend/facade/enums.py
index 9847bb2eb06b..b489773c96c8 100644
--- a/products/batch_exports/backend/facade/enums.py
+++ b/products/batch_exports/backend/facade/enums.py
@@ -109,6 +109,18 @@ OBJECT_STORAGE_DESTINATIONS: frozenset[str] = frozenset(
}
)
+# The integration kinds that a batch export destination can use for credentials.
+BATCH_EXPORT_INTEGRATION_KINDS: tuple[str, ...] = (
+ "aws-redshift",
+ "aws-s3",
+ "azure-blob",
+ "databricks",
+ "google-cloud-service-account",
+ "postgresql",
+ "s3-compatible",
+ "snowflake",
+)
+
S3_SUPPORTED_COMPRESSIONS: dict[str, list[str]] = {
"Parquet": ["zstd", "lz4", "snappy", "gzip", "brotli"],
"JSONLines": ["gzip", "brotli"],
diff --git a/products/batch_exports/backend/logic/exports.py b/products/batch_exports/backend/logic/exports.py
index 9a1e365e4e85..7af55e4a36ee 100644
--- a/products/batch_exports/backend/logic/exports.py
+++ b/products/batch_exports/backend/logic/exports.py
@@ -9,12 +9,12 @@ import contextlib
import dataclasses
from uuid import UUID
+from posthog.integrations import IntegrationKind, IntegrationManager, IntegrationNotFound, IntegrationRecord
from posthog.models.integration import (
AzureBlobIntegration,
AzureBlobIntegrationError,
DatabricksIntegration,
DatabricksIntegrationError,
- Integration,
SnowflakeIntegration,
SnowflakeIntegrationError,
)
@@ -30,6 +30,7 @@ from products.batch_exports.backend import service
from products.batch_exports.backend.facade import contracts
from products.batch_exports.backend.facade.enums import (
AZURE_BLOB_SUPPORTED_COMPRESSIONS,
+ BATCH_EXPORT_INTEGRATION_KINDS,
OBJECT_STORAGE_DESTINATIONS,
S3_FAMILY_TYPES,
S3_SUPPORTED_COMPRESSIONS,
@@ -40,9 +41,9 @@ from products.batch_exports.backend.models.batch_export import BatchExport
# S3-family destinations that may authenticate via an Integration, mapped to
# the linked integration's kind. Adding a future S3-family destination (e.g. a
# first-class GCS-via-S3 type) is a one-line addition here.
-_S3_DESTINATION_TO_INTEGRATION_KIND: dict[str, Integration.IntegrationKind] = {
- BatchExportDestinationType.AWS_S3: Integration.IntegrationKind.AWS_S3,
- BatchExportDestinationType.S3_COMPATIBLE: Integration.IntegrationKind.S3_COMPATIBLE,
+_S3_DESTINATION_TO_INTEGRATION_KIND: dict[str, IntegrationKind] = {
+ BatchExportDestinationType.AWS_S3: IntegrationKind.AWS_S3,
+ BatchExportDestinationType.S3_COMPATIBLE: IntegrationKind.S3_COMPATIBLE,
}
# The only URLs an HTTP destination may send to, as SSRF protection.
@@ -61,12 +62,12 @@ def _get_batch_export(team_id: int, export_id: UUID, *related: str) -> BatchExpo
raise contracts.BatchExportNotFoundError() from e
-def _get_integration(team_id: int, integration_id: int | None) -> Integration | None:
+def _get_integration(team_id: int, integration_id: int | None) -> IntegrationRecord | None:
if integration_id is None:
return None
try:
- return Integration.objects.get(id=integration_id, team_id=team_id)
- except Integration.DoesNotExist as e:
+ return IntegrationManager.kinds(*BATCH_EXPORT_INTEGRATION_KINDS).get(team_id, integration_id)
+ except IntegrationNotFound as e:
raise contracts.InvalidBatchExportDestinationError(
"The integration for this destination no longer exists. Choose another integration and try again."
) from e
@@ -219,11 +220,7 @@ def _check_redshift_copy_inputs(
continue
# These ids live inside `config` rather than the team-scoped `integration`
# field, so team ownership must be checked here.
- if not Integration.objects.filter(
- id=copy_integration_id,
- team_id=team_id,
- kind=Integration.IntegrationKind.AWS_S3,
- ).exists():
+ if copy_integration_id not in IntegrationManager.aws_s3.credentials.get_many(team_id, [copy_integration_id]):
raise contracts.InvalidBatchExportDestinationError(
f"'{field_name}' does not reference an AWS S3 integration of this project."
)
@@ -293,6 +290,7 @@ def validate_destination(
merged_config = _recursive_dict_merge(existing_config, config)
# The integration the request sent. None when it sent none, or sent an explicit null.
+ requested_integration: IntegrationRecord | None
if instance is not None and integration_id is not None and instance.destination.integration_id == integration_id:
# A PATCH usually sends the export's current integration again, which is already loaded.
requested_integration = instance.destination.integration
@@ -361,7 +359,7 @@ def validate_destination(
# validate the Integration is valid (this is mandatory for Databricks batch exports)
if requested_integration is None:
raise contracts.InvalidBatchExportDestinationError("Integration is required for Databricks batch exports")
- if requested_integration.kind != Integration.IntegrationKind.DATABRICKS:
+ if requested_integration.kind != IntegrationKind.DATABRICKS:
raise contracts.InvalidBatchExportDestinationError("Integration is not a Databricks integration.")
# try instantiate the integration to check if it's valid
try:
@@ -375,13 +373,13 @@ def validate_destination(
# (`instance is None`); existing inline-credential exports stay valid when edited.
if requested_integration is None and instance is None:
raise contracts.InvalidBatchExportDestinationError("Integration is required for Postgres batch exports")
- if requested_integration is not None and requested_integration.kind != Integration.IntegrationKind.POSTGRESQL:
+ if requested_integration is not None and requested_integration.kind != IntegrationKind.POSTGRESQL:
raise contracts.InvalidBatchExportDestinationError("Integration is not a PostgreSQL integration.")
if destination_type == BatchExportDestinationType.BIGQUERY:
if requested_integration is None:
raise contracts.InvalidBatchExportDestinationError("Integration is required for BigQuery batch exports")
- if requested_integration.kind != Integration.IntegrationKind.GOOGLE_CLOUD_SERVICE_ACCOUNT:
+ if requested_integration.kind != IntegrationKind.GOOGLE_CLOUD_SERVICE_ACCOUNT:
raise contracts.InvalidBatchExportDestinationError(
"Integration is not a Google Cloud service account integration."
)
@@ -390,7 +388,7 @@ def validate_destination(
# validate the Integration is valid (this is mandatory for Azure Blob batch exports)
if requested_integration is None:
raise contracts.InvalidBatchExportDestinationError("Integration is required for Azure Blob batch exports")
- if requested_integration.kind != Integration.IntegrationKind.AZURE_BLOB:
+ if requested_integration.kind != IntegrationKind.AZURE_BLOB:
raise contracts.InvalidBatchExportDestinationError("Integration is not an Azure Blob integration.")
# try instantiate the integration to check if it's valid
try:
@@ -417,7 +415,7 @@ def validate_destination(
if requested_integration is not None:
# (Team ownership is already enforced by the team-scoped `integration` field.)
- if requested_integration.kind != Integration.IntegrationKind.AWS_REDSHIFT:
+ if requested_integration.kind != IntegrationKind.AWS_REDSHIFT:
raise contracts.InvalidBatchExportDestinationError("Integration is not a Redshift integration.")
# Only plain Redshift integrations store a host; AWS-flavor ones obtain
diff --git a/products/batch_exports/backend/presentation/views/batch_export/destinations.py b/products/batch_exports/backend/presentation/views/batch_export/destinations.py
index 35739e24a37c..2b8e5d3d98e7 100644
--- a/products/batch_exports/backend/presentation/views/batch_export/destinations.py
+++ b/products/batch_exports/backend/presentation/views/batch_export/destinations.py
@@ -8,12 +8,15 @@ from django.db import models
from drf_spectacular.utils import PolymorphicProxySerializer, extend_schema_field
from rest_framework import serializers
-from posthog.api.scoped_related_fields import TeamScopedPrimaryKeyRelatedField
-from posthog.models.integration import Integration
+from posthog.integrations.fields import IntegrationReferenceField
from products.batch_exports.backend.facade import api as batch_exports_api
from products.batch_exports.backend.facade.contracts import InvalidDestinationConfigError
-from products.batch_exports.backend.facade.enums import AZURE_BLOB_SUPPORTED_COMPRESSIONS, S3_SUPPORTED_COMPRESSIONS
+from products.batch_exports.backend.facade.enums import (
+ AZURE_BLOB_SUPPORTED_COMPRESSIONS,
+ BATCH_EXPORT_INTEGRATION_KINDS,
+ S3_SUPPORTED_COMPRESSIONS,
+)
from products.batch_exports.backend.models.batch_export import BatchExportDestination
@@ -526,15 +529,15 @@ class BatchExportDestinationSerializer(serializers.ModelSerializer):
"stripped from responses."
),
)
- integration = TeamScopedPrimaryKeyRelatedField(
- queryset=Integration.objects.all(),
+ integration = IntegrationReferenceField(
+ kinds=BATCH_EXPORT_INTEGRATION_KINDS,
required=False,
allow_null=True,
help_text="The integration for this destination.",
)
- integration_id = TeamScopedPrimaryKeyRelatedField(
+ integration_id = IntegrationReferenceField(
+ kinds=BATCH_EXPORT_INTEGRATION_KINDS,
write_only=True,
- queryset=Integration.objects.all(),
source="integration",
required=False,
allow_null=True,
diff --git a/products/batch_exports/backend/presentation/views/batch_export/exports.py b/products/batch_exports/backend/presentation/views/batch_export/exports.py
index 07faa66e0d17..b4c276596855 100644
--- a/products/batch_exports/backend/presentation/views/batch_export/exports.py
+++ b/products/batch_exports/backend/presentation/views/batch_export/exports.py
@@ -24,8 +24,8 @@ from posthog.hogql.visitor import TraversingVisitor, clone_expr
from posthog.api.log_entries import LogEntryMixin
from posthog.api.routing import TeamAndOrgViewSetMixin
from posthog.api.utils import action
+from posthog.integrations import IntegrationRecord
from posthog.models import Team, User
-from posthog.models.integration import Integration
from products.access_control.backend.facade.api import get_restricted_properties_with_group_type_index_for_team
from products.batch_exports.backend.facade import api as batch_exports_api
@@ -654,7 +654,7 @@ class BatchExportSerializer(serializers.ModelSerializer):
# Running validation for a `detail=True` action.
export_id = uuid.UUID(str(view.kwargs["pk"]))
- integration: Integration | None = destination_attrs.get("integration")
+ integration: IntegrationRecord | None = destination_attrs.get("integration")
try:
batch_exports_api.validate_batch_export_destination(
self.context["team_id"],
@@ -979,7 +979,7 @@ class BatchExportViewSet(TeamAndOrgViewSetMixin, LogEntryMixin, viewsets.ModelVi
def _run_destination_test_step(self, serializer: serializers.BaseSerializer, step: int) -> response.Response:
destination = serializer.validated_data["destination"]
- integration: Integration | None = destination.get("integration")
+ integration: IntegrationRecord | None = destination.get("integration")
try:
result = batch_exports_api.run_destination_test_step(
self.team_id,
diff --git a/products/batch_exports/backend/temporal/destinations/azure_blob_batch_export.py b/products/batch_exports/backend/temporal/destinations/azure_blob_batch_export.py
index f36635d3a6d4..6d4e385f93b6 100644
--- a/products/batch_exports/backend/temporal/destinations/azure_blob_batch_export.py
+++ b/products/batch_exports/backend/temporal/destinations/azure_blob_batch_export.py
@@ -15,7 +15,8 @@ from structlog.contextvars import bind_contextvars
from temporalio import activity, exceptions, workflow
from temporalio.common import RetryPolicy
-from posthog.models.integration import AzureBlobIntegration, Integration
+from posthog.integrations import IntegrationManager, IntegrationNotFound
+from posthog.models.integration import AzureBlobIntegration
from posthog.models.integration.azure_blob import EndpointNotAllowedError, validate_azure_blob_connection_string
from posthog.security.pinned_aiohttp import pinned_session
from posthog.temporal.common.base import PostHogWorkflow
@@ -124,8 +125,8 @@ class AzureBlobInsertInputs(BatchExportInsertInputs):
async def _get_azure_blob_integration(integration_id: int, team_id: int) -> AzureBlobIntegration:
try:
- integration = await Integration.objects.aget(id=integration_id, team_id=team_id)
- except Integration.DoesNotExist:
+ integration = await IntegrationManager.azure_blob.credentials.aget(team_id, integration_id)
+ except IntegrationNotFound:
raise AzureBlobIntegrationNotFoundError(integration_id, team_id)
return AzureBlobIntegration(integration)
diff --git a/products/batch_exports/backend/temporal/destinations/bigquery_batch_export.py b/products/batch_exports/backend/temporal/destinations/bigquery_batch_export.py
index adfad2392378..07fbb41588e2 100644
--- a/products/batch_exports/backend/temporal/destinations/bigquery_batch_export.py
+++ b/products/batch_exports/backend/temporal/destinations/bigquery_batch_export.py
@@ -41,7 +41,8 @@ from temporalio.common import RetryPolicy
from posthog.credentials_service.callers import CredentialsCaller
from posthog.credentials_service.settings_fallback import get_or_settings
-from posthog.models.integration import GoogleCloudServiceAccountIntegration, Integration
+from posthog.integrations import IntegrationManager, IntegrationNotFound
+from posthog.models.integration import GoogleCloudServiceAccountIntegration
from posthog.models.integration.google_cloud import InvalidGoogleTokenUriError, require_google_token_uri
from posthog.models.team import Team
from posthog.temporal.common.base import PostHogWorkflow
@@ -1486,8 +1487,10 @@ async def _get_google_cloud_service_account_integration(
return None
try:
- integration = await Integration.objects.aget(id=inputs.integration_id, team_id=inputs.team_id)
- except Integration.DoesNotExist:
+ integration = await IntegrationManager.google_cloud_service_account.credentials.aget(
+ inputs.team_id, inputs.integration_id
+ )
+ except IntegrationNotFound:
raise BigQueryIntegrationNotFoundError(
f"Google Cloud service account integration with id '{inputs.integration_id}' not found"
)
diff --git a/products/batch_exports/backend/temporal/destinations/databricks_batch_export.py b/products/batch_exports/backend/temporal/destinations/databricks_batch_export.py
index bdffa4b36dd6..557bf5323e99 100644
--- a/products/batch_exports/backend/temporal/destinations/databricks_batch_export.py
+++ b/products/batch_exports/backend/temporal/destinations/databricks_batch_export.py
@@ -24,7 +24,8 @@ from structlog.contextvars import bind_contextvars
from temporalio import activity, exceptions, workflow
from temporalio.common import RetryPolicy
-from posthog.models.integration import DatabricksIntegration, Integration
+from posthog.integrations import IntegrationManager, IntegrationNotFound
+from posthog.models.integration import DatabricksIntegration
from posthog.models.integration.databricks import NOT_A_DATABRICKS_HOST_MESSAGE, is_databricks_workspace_host
from posthog.temporal.common.base import PostHogWorkflow
from posthog.temporal.common.heartbeat import Heartbeater
@@ -1125,8 +1126,8 @@ async def _get_databricks_integration(inputs: DatabricksInsertInputs) -> Databri
DatabricksIntegrationError: If the Databricks integration is not valid.
"""
try:
- integration = await Integration.objects.aget(id=inputs.integration_id, team_id=inputs.team_id)
- except Integration.DoesNotExist:
+ integration = await IntegrationManager.databricks.credentials.aget(inputs.team_id, inputs.integration_id)
+ except IntegrationNotFound:
raise DatabricksIntegrationNotFoundError(f"Databricks integration with id '{inputs.integration_id}' not found")
return DatabricksIntegration(integration)
diff --git a/products/batch_exports/backend/temporal/destinations/postgres_batch_export.py b/products/batch_exports/backend/temporal/destinations/postgres_batch_export.py
index bae7bf367e38..67d9e52be2f3 100644
--- a/products/batch_exports/backend/temporal/destinations/postgres_batch_export.py
+++ b/products/batch_exports/backend/temporal/destinations/postgres_batch_export.py
@@ -22,14 +22,8 @@ from structlog.contextvars import bind_contextvars
from temporalio import activity, exceptions, workflow
from temporalio.common import RetryPolicy
-from posthog.models.integration import (
- MISSING_CERT_PATH,
- TLS,
- Authority,
- Credentials,
- Integration,
- PostgreSQLIntegration,
-)
+from posthog.integrations import IntegrationManager, IntegrationNotFound
+from posthog.models.integration import MISSING_CERT_PATH, TLS, Authority, Credentials, PostgreSQLIntegration
from posthog.psycopg_helpers import is_resolvable_hostname, resolve_psycopg_hostaddr_with_timeout
from posthog.security.url_validation import (
INVALID_HOST_MESSAGE,
@@ -960,8 +954,8 @@ async def _get_postgresql_integration(
return None
try:
- integration = await Integration.objects.aget(id=inputs.integration_id, team_id=inputs.team_id)
- except Integration.DoesNotExist:
+ integration = await IntegrationManager.postgresql.credentials.aget(inputs.team_id, inputs.integration_id)
+ except IntegrationNotFound:
raise PostgreSQLIntegrationNotFoundError(f"PostgreSQL integration with id '{inputs.integration_id}' not found")
return PostgreSQLIntegration(integration)
diff --git a/products/batch_exports/backend/temporal/destinations/redshift_batch_export.py b/products/batch_exports/backend/temporal/destinations/redshift_batch_export.py
index e4366943d84f..a46287eda88d 100644
--- a/products/batch_exports/backend/temporal/destinations/redshift_batch_export.py
+++ b/products/batch_exports/backend/temporal/destinations/redshift_batch_export.py
@@ -21,6 +21,7 @@ from temporalio import activity, exceptions, workflow
from temporalio.common import RetryPolicy
from posthog.dataclasses import frozen
+from posthog.integrations import IntegrationManager, IntegrationNotFound
from posthog.models import Team
from posthog.models.integration import (
TLS,
@@ -30,7 +31,6 @@ from posthog.models.integration import (
AWSS3Integration,
AWSS3RoleBasedIntegration,
Credentials,
- Integration,
IntegrationError,
RedshiftIntegration,
)
@@ -901,15 +901,10 @@ async def _get_redshift_integration(
) -> AWSRedshiftRoleBasedIntegration | AWSRedshiftIntegration | RedshiftIntegration:
"""Fetch a Redshift integration from the database."""
try:
- integration = await Integration.objects.aget(id=integration_id, team_id=team_id)
- except Integration.DoesNotExist:
+ integration = await IntegrationManager.aws_redshift.credentials.aget(team_id, integration_id)
+ except IntegrationNotFound:
raise IntegrationError(f"Redshift integration with ID '{integration_id}' not found for team '{team_id}'")
- if integration.kind != Integration.IntegrationKind.AWS_REDSHIFT:
- raise IntegrationError(
- f"Integration with ID '{integration_id}' for team '{team_id}' is not a Redshift integration "
- f"(kind='{integration.kind}')"
- )
if "aws_role_arn" in integration.config:
return AWSRedshiftRoleBasedIntegration(integration)
elif "aws_access_key_id" in integration.sensitive_config:
@@ -921,15 +916,10 @@ async def _get_redshift_integration(
async def _get_aws_s3_integration(integration_id: int, team_id: int) -> AWSS3RoleBasedIntegration | AWSS3Integration:
"""Fetch an AWS S3 integration from the database."""
try:
- integration = await Integration.objects.aget(id=integration_id, team_id=team_id)
- except Integration.DoesNotExist:
+ integration = await IntegrationManager.aws_s3.credentials.aget(team_id, integration_id)
+ except IntegrationNotFound:
raise IntegrationError(f"AWS S3 integration with ID '{integration_id}' not found for team '{team_id}'")
- if integration.kind != Integration.IntegrationKind.AWS_S3:
- raise IntegrationError(
- f"Integration with ID '{integration_id}' for team '{team_id}' is not an S3 integration "
- f"(kind='{integration.kind}')"
- )
if "aws_role_arn" in integration.config:
return AWSS3RoleBasedIntegration(integration)
else:
diff --git a/products/batch_exports/backend/temporal/destinations/s3_batch_export.py b/products/batch_exports/backend/temporal/destinations/s3_batch_export.py
index 8091c11b6a55..13e5cab92b14 100644
--- a/products/batch_exports/backend/temporal/destinations/s3_batch_export.py
+++ b/products/batch_exports/backend/temporal/destinations/s3_batch_export.py
@@ -29,10 +29,10 @@ from temporalio.common import RetryPolicy
from posthog.credentials_service.callers import CredentialsCaller
from posthog.credentials_service.settings_fallback import get_or_settings
from posthog.dataclasses import frozen
+from posthog.integrations import IntegrationKind, IntegrationManager, IntegrationNotFound
from posthog.models.integration import (
AWSS3Integration,
AWSS3RoleBasedIntegration,
- Integration,
IntegrationError,
S3CompatibleIntegration,
)
@@ -178,16 +178,16 @@ async def _get_s3_integration(
the credentials are malformed.
"""
try:
- integration = await Integration.objects.aget(id=integration_id, team_id=team_id)
- except Integration.DoesNotExist:
+ integration = await IntegrationManager.any.aget(team_id, integration_id)
+ except IntegrationNotFound:
raise S3IntegrationNotFoundError(integration_id, team_id)
- if integration.kind == Integration.IntegrationKind.AWS_S3:
+ if integration.kind == IntegrationKind.AWS_S3:
if "aws_role_arn" in integration.config:
return AWSS3RoleBasedIntegration(integration)
return AWSS3Integration(integration)
- if integration.kind == Integration.IntegrationKind.S3_COMPATIBLE:
+ if integration.kind == IntegrationKind.S3_COMPATIBLE:
return S3CompatibleIntegration(integration)
raise IntegrationError(
diff --git a/products/batch_exports/backend/temporal/destinations/snowflake_batch_export.py b/products/batch_exports/backend/temporal/destinations/snowflake_batch_export.py
index 9342df013747..6ee45791735e 100644
--- a/products/batch_exports/backend/temporal/destinations/snowflake_batch_export.py
+++ b/products/batch_exports/backend/temporal/destinations/snowflake_batch_export.py
@@ -25,7 +25,8 @@ from structlog.contextvars import bind_contextvars
from temporalio import activity, exceptions, workflow
from temporalio.common import RetryPolicy
-from posthog.models.integration import Integration, SnowflakeIntegration
+from posthog.integrations import IntegrationManager, IntegrationNotFound
+from posthog.models.integration import SnowflakeIntegration
from posthog.temporal.common.base import PostHogWorkflow
from posthog.temporal.common.heartbeat import Heartbeater
from posthog.temporal.common.logger import get_logger, get_write_only_logger
@@ -197,10 +198,8 @@ async def _get_snowflake_integration(integration_id: int, team_id: int) -> Snowf
`SnowflakeIntegration` raises `SnowflakeIntegrationError` if the config is malformed.
"""
try:
- integration = await Integration.objects.aget(
- id=integration_id, team_id=team_id, kind=Integration.IntegrationKind.SNOWFLAKE
- )
- except Integration.DoesNotExist:
+ integration = await IntegrationManager.snowflake.credentials.aget(team_id, integration_id)
+ except IntegrationNotFound:
raise SnowflakeIntegrationNotFoundError(integration_id, team_id)
return SnowflakeIntegration(integration)
diff --git a/products/batch_exports/backend/tests/api/test_create_databricks.py b/products/batch_exports/backend/tests/api/test_create_databricks.py
index ca764f56d138..4108a4e4f61b 100644
--- a/products/batch_exports/backend/tests/api/test_create_databricks.py
+++ b/products/batch_exports/backend/tests/api/test_create_databricks.py
@@ -221,7 +221,7 @@ def test_creating_databricks_batch_export_fails_if_integration_is_not_the_correc
integration = Integration.objects.create(
team=team,
- kind=Integration.IntegrationKind.SLACK,
+ kind=Integration.IntegrationKind.S3_COMPATIBLE,
integration_id="my-server-hostname",
config={"server_hostname": "my-server-hostname"},
sensitive_config={"client_id": "my-client-id"},
diff --git a/products/batch_exports/backend/tests/api/test_serializers.py b/products/batch_exports/backend/tests/api/test_serializers.py
index b78684b014c1..fd1dd0fdf662 100644
--- a/products/batch_exports/backend/tests/api/test_serializers.py
+++ b/products/batch_exports/backend/tests/api/test_serializers.py
@@ -12,7 +12,6 @@ from rest_framework import serializers
from posthog.hogql import ast
from posthog.hogql.property_access_types import RestrictedProperty
-from posthog.api.scoped_related_fields import TeamScopedPrimaryKeyRelatedField
from posthog.models import Organization, PropertyDefinition, Team
from posthog.models.integration import Integration
from posthog.models.scoping import team_scope
@@ -380,7 +379,4 @@ class TestBatchExportDestinationSerializerTeamScoping(BaseTest):
own_integration = self._make_integration(self.team)
serializer = BatchExportDestinationSerializer(context={"team_id": self.team.pk})
- # Field-level queryset filter should include same-team integrations.
- field = cast(TeamScopedPrimaryKeyRelatedField, serializer.fields[field_name])
- queryset = field.get_queryset()
- assert queryset is not None and own_integration in queryset
+ assert serializer.fields[field_name].run_validation(own_integration.pk) == own_integration
diff --git a/products/batch_exports/backend/tests/test_facade.py b/products/batch_exports/backend/tests/test_facade.py
index 6ed666e0dc3a..31115f8d5a8e 100644
--- a/products/batch_exports/backend/tests/test_facade.py
+++ b/products/batch_exports/backend/tests/test_facade.py
@@ -10,6 +10,7 @@ from temporalio.client import Client
from posthog.api.test.test_organization import create_organization
from posthog.api.test.test_team import create_team
+from posthog.integrations import IntegrationNotFound
from posthog.models import Integration, User
from posthog.temporal.common.client import sync_connect
@@ -427,7 +428,7 @@ def test_destination_test_step_refuses_an_integration_from_another_team(team, or
with (
mock.patch("products.batch_exports.backend.destination_tests.get_destination_test"),
- pytest.raises(Integration.DoesNotExist),
+ pytest.raises(IntegrationNotFound),
):
api.run_destination_test_step(
team.pk,
diff --git a/products/warehouse_sources/backend/presentation/destination_connection_check.py b/products/warehouse_sources/backend/presentation/destination_connection_check.py
index 43d15e4f0ddc..9262beaab5cc 100644
--- a/products/warehouse_sources/backend/presentation/destination_connection_check.py
+++ b/products/warehouse_sources/backend/presentation/destination_connection_check.py
@@ -8,7 +8,8 @@ from enum import StrEnum
import psycopg
from psycopg import errors as pg_errors
-from posthog.models.integration import Integration, PostgreSQLIntegration
+from posthog.integrations import IntegrationRecord
+from posthog.models.integration import PostgreSQLIntegration
from posthog.models.integration.postgres import MISSING_CERT_PATH
from products.warehouse_sources.backend.facade.source_management import HostNotAllowedError, pinned_host_kwargs
@@ -89,7 +90,7 @@ def _query_deadline(connection: psycopg.Connection, timeout_seconds: float) -> I
timer.join()
-def check_postgres_destination(integration: Integration, config: dict[str, str] | None) -> None:
+def check_postgres_destination(integration: IntegrationRecord, config: dict[str, str] | None) -> None:
"""Connect to the destination with a short timeout and check that a writer can create tables.
Raises `DestinationConnectionCheckError`. The writer runs `CREATE SCHEMA IF NOT EXISTS`, so a
diff --git a/products/warehouse_sources/backend/presentation/destination_connection_testing.py b/products/warehouse_sources/backend/presentation/destination_connection_testing.py
index 7c5b950f10a0..dd8508534c15 100644
--- a/products/warehouse_sources/backend/presentation/destination_connection_testing.py
+++ b/products/warehouse_sources/backend/presentation/destination_connection_testing.py
@@ -6,7 +6,7 @@ from uuid import uuid4
from django.db import models
from posthog.dataclasses import frozen
-from posthog.models.integration import Integration
+from posthog.integrations import IntegrationKind, IntegrationRecord
from products.batch_exports.backend.facade import api as batch_exports_api
from products.batch_exports.backend.facade.contracts import (
@@ -135,15 +135,15 @@ def has_connection_test(destination_type: str) -> bool:
)
-def connection_test_type(destination_type: str, integration: Integration) -> str | None:
+def connection_test_type(destination_type: str, integration: IntegrationRecord) -> str | None:
"""The batch exports test that covers this destination, or None when there is none."""
if destination_type == ExternalDataDestination.Type.S3:
- return "AwsS3" if integration.kind == Integration.IntegrationKind.AWS_S3 else "S3Compatible"
+ return "AwsS3" if integration.kind == IntegrationKind.AWS_S3 else "S3Compatible"
return _TEST_TYPES.get(destination_type)
def run_connection_test(
- destination_type: str, integration: Integration, config: Mapping[str, Any] | None
+ destination_type: str, integration: IntegrationRecord, config: Mapping[str, Any] | None
) -> ConnectionTestRun | None:
"""Run each step of the destination's connection test, and stop at the first failed step.
@@ -193,7 +193,7 @@ _POSTGRES_STEP = DestinationTestStep(
)
-def run_postgres_connection_test(integration: Integration, config: Mapping[str, Any] | None) -> ConnectionTestRun:
+def run_postgres_connection_test(integration: IntegrationRecord, config: Mapping[str, Any] | None) -> ConnectionTestRun:
"""Run the Postgres check as a single step, so it reports like the other connection tests."""
try:
check_postgres_destination(integration, dict(config) if isinstance(config, Mapping) else None)
diff --git a/products/warehouse_sources/backend/presentation/views/external_data_destination.py b/products/warehouse_sources/backend/presentation/views/external_data_destination.py
index 811443ec1247..42982eaaba91 100644
--- a/products/warehouse_sources/backend/presentation/views/external_data_destination.py
+++ b/products/warehouse_sources/backend/presentation/views/external_data_destination.py
@@ -13,9 +13,9 @@ from rest_framework.response import Response
from posthog.api.mixins import ValidatedRequest, validated_request
from posthog.api.routing import TeamAndOrgViewSetMixin
-from posthog.api.scoped_related_fields import TeamScopedPrimaryKeyRelatedField
from posthog.api.utils import action
-from posthog.models.integration import Integration
+from posthog.integrations import IntegrationKind, IntegrationRecord
+from posthog.integrations.fields import IntegrationReferenceField
from posthog.permissions import is_service_auth
from products.access_control.backend.facade.user_access_control import access_level_satisfied_for_resource
@@ -49,21 +49,22 @@ from products.warehouse_sources.backend.presentation.views.external_data_schema
# `Type` enum carries the rest so the column and its data do not change as each one lands, but
# a user cannot select one until its writer ships.
DESTINATION_INTEGRATION_KINDS: dict[str, tuple[str, ...]] = {
- str(ExternalDataDestination.Type.POSTGRES): (str(Integration.IntegrationKind.POSTGRESQL),),
- str(ExternalDataDestination.Type.REDSHIFT): (str(Integration.IntegrationKind.AWS_REDSHIFT),),
- str(ExternalDataDestination.Type.SNOWFLAKE): (str(Integration.IntegrationKind.SNOWFLAKE),),
- str(ExternalDataDestination.Type.DATABRICKS): (str(Integration.IntegrationKind.DATABRICKS),),
+ str(ExternalDataDestination.Type.POSTGRES): (str(IntegrationKind.POSTGRESQL),),
+ str(ExternalDataDestination.Type.REDSHIFT): (str(IntegrationKind.AWS_REDSHIFT),),
+ str(ExternalDataDestination.Type.SNOWFLAKE): (str(IntegrationKind.SNOWFLAKE),),
+ str(ExternalDataDestination.Type.DATABRICKS): (str(IntegrationKind.DATABRICKS),),
str(ExternalDataDestination.Type.S3): (
- str(Integration.IntegrationKind.AWS_S3),
- str(Integration.IntegrationKind.S3_COMPATIBLE),
+ str(IntegrationKind.AWS_S3),
+ str(IntegrationKind.S3_COMPATIBLE),
),
- str(ExternalDataDestination.Type.BIGQUERY): (str(Integration.IntegrationKind.GOOGLE_CLOUD_SERVICE_ACCOUNT),),
- str(ExternalDataDestination.Type.AZURE_BLOB): (str(Integration.IntegrationKind.AZURE_BLOB),),
+ str(ExternalDataDestination.Type.BIGQUERY): (str(IntegrationKind.GOOGLE_CLOUD_SERVICE_ACCOUNT),),
+ str(ExternalDataDestination.Type.AZURE_BLOB): (str(IntegrationKind.AZURE_BLOB),),
}
# Types a user may create. The PostHog warehouse row is created by the sync itself the first
# time a schema resolves to it, and every team has exactly one, so it is not user-managed.
USER_CREATABLE_TYPES = frozenset(DESTINATION_INTEGRATION_KINDS)
+ALL_DESTINATION_INTEGRATION_KINDS = sorted({kind for kinds in DESTINATION_INTEGRATION_KINDS.values() for kind in kinds})
# Built from `Type`, so the values cannot drift from the model. django-stubs only types the
# literal form of the functional API.
@@ -218,8 +219,8 @@ class ExternalDataDestinationSerializer(serializers.ModelSerializer):
"format. Credentials are not stored here. They live on the linked integration."
),
)
- integration = TeamScopedPrimaryKeyRelatedField(
- queryset=Integration.objects.all(),
+ integration = IntegrationReferenceField(
+ kinds=ALL_DESTINATION_INTEGRATION_KINDS,
required=False,
allow_null=True,
help_text="Integration holding this destination's credentials. Required for every type except the PostHog warehouse.",
@@ -357,14 +358,14 @@ class ExternalDataDestinationSerializer(serializers.ModelSerializer):
# Only creation tests the connection. A destination keeps its server, database and schema
# after that (see `_reject_retargeting`), so there is nothing new to test on update.
- def _check_postgres_connection(self, integration: Integration, config: dict[str, Any] | None) -> None:
+ def _check_postgres_connection(self, integration: IntegrationRecord, config: dict[str, Any] | None) -> None:
try:
check_postgres_destination(integration, config)
except DestinationConnectionCheckError as error:
field = "config" if error.failure in CONFIG_FAILURES else "integration"
raise ValidationError({field: str(error)})
- def _run_connection_test(self, destination_type: str, integration: Integration, config: Any) -> None:
+ def _run_connection_test(self, destination_type: str, integration: IntegrationRecord, config: Any) -> None:
run = run_connection_test(destination_type, integration, config)
if run is not None and run.failure is not None:
raise ValidationError({run.failure.field: str(run.failure)})
diff --git a/products/warehouse_sources/backend/presentation/views/external_data_source/oauth_accounts.py b/products/warehouse_sources/backend/presentation/views/external_data_source/oauth_accounts.py
index a412cd155f19..77a371ace194 100644
--- a/products/warehouse_sources/backend/presentation/views/external_data_source/oauth_accounts.py
+++ b/products/warehouse_sources/backend/presentation/views/external_data_source/oauth_accounts.py
@@ -13,7 +13,7 @@ from rest_framework.request import Request
from rest_framework.response import Response
from posthog.api.utils import action
-from posthog.models.integration import Integration
+from posthog.integrations import IntegrationManager
from posthog.permissions import TeamMemberAdminManagementPermission
from products.warehouse_sources.backend.facade.source_management import (
@@ -146,9 +146,9 @@ class ExternalDataSourceOAuthAccountsMixin(base.ExternalDataSourceViewSetBase):
expected_kinds = helpers.get_oauth_integration_kinds(source.get_source_config.fields)
if not expected_kinds:
raise ValidationError(f"Source type {source_type} does not support listing OAuth accounts")
- if not Integration.objects.filter(
- id=integration_id_int, team_id=self.team_id, kind__in=expected_kinds
- ).exists():
+ if integration_id_int not in IntegrationManager.kinds(*expected_kinds).get_many(
+ self.team_id, [integration_id_int]
+ ):
# One message for "gone" and "wrong kind" alike: from the UI both mean the picker is holding
# a connection this source can't use, and neither tells the caller anything about ids it
# isn't already allowed to see.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/test_postgres_configuration_errors.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/test_postgres_configuration_errors.py
index fe3e00d8b8d4..4044106ccfa4 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/test_postgres_configuration_errors.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/test_postgres_configuration_errors.py
@@ -5,7 +5,7 @@ from unittest import mock
import psycopg
-from posthog.models.integration import Integration
+from posthog.integrations import IntegrationNotFound, IntegrationReader
from posthog.security.url_validation import UNREACHABLE_HOST_MESSAGE
from products.batch_exports.backend.facade.destinations.postgres import PostgreSQLClient, PostgreSQLConnectionError
@@ -232,7 +232,7 @@ class TestWriterClient:
writer = PostgresDestinationWriter(_ctx(integration_id))
with (
- mock.patch.object(Integration.objects, "aget", side_effect=Integration.DoesNotExist),
+ mock.patch.object(IntegrationReader, "aget", side_effect=IntegrationNotFound),
mock.patch.object(psycopg.AsyncConnection, "connect") as connect,
pytest.raises(DestinationConfigurationError) as caught,
):
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/bigquery.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/bigquery.py
index 03538eece8da..b2a62b709f7f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/bigquery.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/bigquery.py
@@ -210,9 +210,11 @@ class BigQueryDestinationWriter:
if self._ctx.integration_id is None:
raise ValueError(f"Destination {self._ctx.destination_name} has no integration to connect with")
- from posthog.models.integration import Integration # noqa: PLC0415 — avoids a model import cycle
+ from posthog.integrations import IntegrationManager # noqa: PLC0415 — avoids a model import cycle
- integration = Integration.objects.get(id=self._ctx.integration_id, team_id=self._ctx.team_id)
+ integration = IntegrationManager.google_cloud_service_account.credentials.get(
+ self._ctx.team_id, self._ctx.integration_id
+ )
google_cloud_integration = GoogleCloudServiceAccountIntegration(integration)
if not google_cloud_integration.has_key():
# Keyless means PostHog impersonates the team's service account instead of holding
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/databricks.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/databricks.py
index c778a7514d0a..1c15cc810200 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/databricks.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/databricks.py
@@ -28,7 +28,7 @@ from typing import ClassVar
import pyarrow as pa
import pyarrow.parquet as pq
-from posthog.models.integration import Integration
+from posthog.integrations import IntegrationManager, IntegrationNotFound
from posthog.models.integration.databricks import DatabricksIntegration
from products.batch_exports.backend.facade.destinations.databricks import (
@@ -219,10 +219,8 @@ def fields_for(schema: pa.Schema, *, with_batch_index: bool) -> list[DatabricksF
async def _load_integration(integration_id: int, team_id: int) -> DatabricksIntegration:
try:
- integration = await Integration.objects.aget(
- id=integration_id, team_id=team_id, kind=Integration.IntegrationKind.DATABRICKS
- )
- except Integration.DoesNotExist:
+ integration = await IntegrationManager.databricks.credentials.aget(team_id, integration_id)
+ except IntegrationNotFound:
raise DatabricksIntegrationNotFoundError(
f"Databricks integration with ID '{integration_id}' not found for team '{team_id}'"
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/postgres.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/postgres.py
index bd59e6610fb9..70b6213812b8 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/postgres.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/postgres.py
@@ -40,7 +40,8 @@ import psycopg
import pyarrow as pa
from psycopg import sql
-from posthog.models.integration import Integration, PostgreSQLIntegration
+from posthog.integrations import IntegrationManager, IntegrationNotFound, IntegrationRecord
+from posthog.models.integration import PostgreSQLIntegration
from posthog.psycopg_helpers import has_ipv6_route, is_resolvable_hostname, is_temporary_resolution_failure
from products.batch_exports.backend.facade.destinations.postgres import (
@@ -259,18 +260,16 @@ class PostgresDestinationWriter:
# --- connection -------------------------------------------------------------------
- def _client_from_integration(self, integration: Integration) -> PostgreSQLClient:
+ def _client_from_integration(self, integration: IntegrationRecord) -> PostgreSQLClient:
# `from_inputs` accepts anything exposing credentials/authority/tls, which the
# integration wrapper does, so the batch export path and this one resolve credentials
# identically.
return PostgreSQLClient.from_inputs(PostgreSQLIntegration(integration), database=self._database)
- async def _load_integration(self, integration_id: int) -> Integration:
+ async def _load_integration(self, integration_id: int) -> IntegrationRecord:
try:
- return await Integration.objects.aget(
- id=integration_id, team_id=self._ctx.team_id, kind=self.integration_kind
- )
- except Integration.DoesNotExist as err:
+ return await IntegrationManager.kinds(self.integration_kind).aget(self._ctx.team_id, integration_id)
+ except IntegrationNotFound as err:
# The same error batch exports raises, which its destinations never retry: a
# deleted or re-kinded integration cannot be recovered by trying again.
raise PostgreSQLIntegrationNotFoundError(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/redshift.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/redshift.py
index e3a14c78c5ce..908dea22ad11 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/redshift.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/redshift.py
@@ -30,7 +30,8 @@ import pyarrow as pa
from psycopg import sql
from psycopg.errors import InsufficientPrivilege
-from posthog.models.integration import Integration, RedshiftIntegration
+from posthog.integrations import IntegrationRecord
+from posthog.models.integration import RedshiftIntegration
from products.batch_exports.backend.facade.destinations.postgres import Fields, PostgreSQLClient
from products.batch_exports.backend.facade.destinations.redshift import RedshiftClient
@@ -135,7 +136,7 @@ class RedshiftDestinationWriter(PostgresDestinationWriter):
with self._configuration_errors():
await super().abort_run(ctx)
- def _client_from_integration(self, integration: Integration) -> PostgreSQLClient:
+ def _client_from_integration(self, integration: IntegrationRecord) -> PostgreSQLClient:
return RedshiftClient.from_inputs(RedshiftIntegration(integration), database=self._database)
def _column_type(self, arrow_type: pa.DataType) -> str:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/apple_search_ads/oauth.py b/products/warehouse_sources/backend/temporal/data_imports/sources/apple_search_ads/oauth.py
index 5acc1f83685c..f5a1d6077c48 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/apple_search_ads/oauth.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/apple_search_ads/oauth.py
@@ -5,7 +5,8 @@ The customer grants PostHog access from the connect form, and the grant is store
so the token is resolved per authentication rather than once per run.
"""
-from posthog.models.integration import ERROR_TOKEN_REFRESH_FAILED, Integration, OauthIntegration
+from posthog.integrations import IntegrationRecord
+from posthog.models.integration import ERROR_TOKEN_REFRESH_FAILED, OauthIntegration
from products.warehouse_sources.backend.temporal.data_imports.sources.apple_search_ads.apple_search_ads import (
AppleSearchAdsAuthError,
@@ -20,7 +21,7 @@ class AppleAdsOauthError(AppleSearchAdsAuthError):
"""Raised where the client expects an auth failure, so `validate_credentials` reports it."""
-def apple_ads_access_token(integration: Integration) -> str:
+def apple_ads_access_token(integration: IntegrationRecord) -> str:
"""Current bearer token for an Apple Ads grant, refreshing it when it has expired."""
oauth_integration = OauthIntegration(integration)
if oauth_integration.access_token_expired():
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bigquery/bigquery.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bigquery/bigquery.py
index 51cca63e0740..4da734a6058f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bigquery/bigquery.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bigquery/bigquery.py
@@ -58,7 +58,8 @@ from structlog.types import FilteringBoundLogger
from posthog.dataclasses import frozen
from posthog.exceptions_capture import capture_exception
-from posthog.models.integration import GoogleCloudServiceAccountIntegration, Integration
+from posthog.integrations import IntegrationManager, IntegrationNotFound
+from posthog.models.integration import GoogleCloudServiceAccountIntegration
from posthog.models.integration.google_cloud import (
GOOGLE_SERVICE_ACCOUNT_INVALID_TOKEN_URI_ERROR,
InvalidGoogleTokenUriError,
@@ -716,13 +717,10 @@ def _impersonated_credentials(service_account_email: str, team_id: int) -> googl
def _resolve_auth_from_integration(integration_id: int, team_id: int) -> BigQueryAuthInfo:
- integration = Integration.objects.filter(
- id=integration_id,
- team_id=team_id,
- kind=Integration.IntegrationKind.GOOGLE_CLOUD_SERVICE_ACCOUNT,
- ).first()
- if integration is None:
- raise BigQueryAuthResolutionError(BIGQUERY_INTEGRATION_NOT_FOUND_ERROR)
+ try:
+ integration = IntegrationManager.google_cloud_service_account.credentials.get(team_id, integration_id)
+ except IntegrationNotFound:
+ raise BigQueryAuthResolutionError(BIGQUERY_INTEGRATION_NOT_FOUND_ERROR) from None
google_cloud_integration = GoogleCloudServiceAccountIntegration(integration)
if google_cloud_integration.has_key():
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/mixins.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/mixins.py
index 73d03212aba3..a70b32a534e4 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/mixins.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/mixins.py
@@ -15,7 +15,7 @@ from sshtunnel import BaseSSHTunnelForwarderError, SSHTunnelForwarder
from posthog.cloud_utils import is_cloud
from posthog.dataclasses import frozen
-from posthog.models.integration import Integration
+from posthog.integrations import AuthType, IntegrationManager, IntegrationNotFound, IntegrationRecord
from posthog.psycopg_helpers import (
is_resolvable_hostname,
is_temporary_resolution_failure,
@@ -1004,7 +1004,7 @@ def _integration_fetch_backoff_sleep(attempt: int) -> None:
class OAuthMixin:
"""Mixin for OAuth-based sources"""
- def get_oauth_integration(self, integration_id: int, team_id: int) -> Integration:
+ def get_oauth_integration(self, integration_id: int, team_id: int) -> IntegrationRecord:
"""Get OAuth integration from integration ID.
Temporal activities run in a long-lived worker outside Django's request cycle, so a pooled
@@ -1015,7 +1015,7 @@ class OAuthMixin:
marks its connection unusable, so ``close_old_connections()`` evicts it between attempts and
the next attempt runs on a fresh one; the sleep lets a saturated pool drain. A genuinely
missing integration is surfaced as the stable, non-retryable ``ValueError`` and is not
- retried.
+ retried. A row that is not an OAuth row counts as missing.
"""
if not integration_id:
raise ValueError(f"Missing integration ID")
@@ -1023,8 +1023,8 @@ class OAuthMixin:
attempt = 0
while True:
try:
- return Integration.objects.get(id=integration_id, team_id=team_id)
- except Integration.DoesNotExist:
+ integration = IntegrationManager.any.get(team_id, integration_id)
+ except IntegrationNotFound:
raise ValueError(f"Integration not found: {integration_id}") from None
except OperationalError:
attempt += 1
@@ -1032,6 +1032,12 @@ class OAuthMixin:
raise
close_old_connections()
_integration_fetch_backoff_sleep(attempt)
+ continue
+ # The reference does not name its kind here, so the read covers every kind. A row of another
+ # auth type (for example a token row of an OAuth kind) has no OAuth tokens to read or refresh.
+ if integration.auth_type != AuthType.OAUTH:
+ raise ValueError(f"Integration not found: {integration_id}")
+ return integration
def get_oauth_accounts(
self, integration_id: int, team_id: int, search: str | None = None
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/test_mixins.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/test_mixins.py
index d47333455e9c..f778b47a7ab0 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/test_mixins.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/test_mixins.py
@@ -13,7 +13,7 @@ from parameterized import parameterized
from sshtunnel import BaseSSHTunnelForwarderError
from posthog.dataclasses import frozen
-from posthog.models.integration import Integration
+from posthog.integrations import IntegrationNotFound
from posthog.temporal.common.errors import NonReportableError
from products.warehouse_sources.backend.temporal.data_imports.external_data_job import (
@@ -675,11 +675,11 @@ class TestOAuthMixinIntegrationFetchResilience(SimpleTestCase):
]
)
def test_retries_transient_db_error_then_succeeds(self, _name: str, message: str):
- integration = object()
+ integration = mock.Mock(auth_type="oauth")
get = mock.Mock(side_effect=[OperationalError(message), OperationalError(message), integration])
with (
- patch(f"{_MIXINS_MODULE}.Integration.objects.get", get),
+ patch(f"{_MIXINS_MODULE}.IntegrationManager.any.get", get),
patch(f"{_MIXINS_MODULE}.close_old_connections") as close,
patch(f"{_MIXINS_MODULE}.time.sleep") as sleep,
):
@@ -695,10 +695,10 @@ class TestOAuthMixinIntegrationFetchResilience(SimpleTestCase):
def test_success_does_not_evict_connection(self):
# Closing the connection on the happy path would tear down the caller's open transaction
# (e.g. inside a transactional test), so eviction must only happen between retries.
- integration = object()
+ integration = mock.Mock(auth_type="oauth")
with (
- patch(f"{_MIXINS_MODULE}.Integration.objects.get", return_value=integration),
+ patch(f"{_MIXINS_MODULE}.IntegrationManager.any.get", return_value=integration),
patch(f"{_MIXINS_MODULE}.close_old_connections") as close,
patch(f"{_MIXINS_MODULE}.time.sleep") as sleep,
):
@@ -712,7 +712,7 @@ class TestOAuthMixinIntegrationFetchResilience(SimpleTestCase):
get = mock.Mock(side_effect=OperationalError("query_wait_timeout"))
with (
- patch(f"{_MIXINS_MODULE}.Integration.objects.get", get),
+ patch(f"{_MIXINS_MODULE}.IntegrationManager.any.get", get),
patch(f"{_MIXINS_MODULE}.close_old_connections"),
patch(f"{_MIXINS_MODULE}.time.sleep") as sleep,
):
@@ -725,10 +725,10 @@ class TestOAuthMixinIntegrationFetchResilience(SimpleTestCase):
assert sleep.call_args_list == [mock.call(2), mock.call(4), mock.call(6)]
def test_missing_integration_is_not_retried(self):
- get = mock.Mock(side_effect=Integration.DoesNotExist())
+ get = mock.Mock(side_effect=IntegrationNotFound())
with (
- patch(f"{_MIXINS_MODULE}.Integration.objects.get", get),
+ patch(f"{_MIXINS_MODULE}.IntegrationManager.any.get", get),
patch(f"{_MIXINS_MODULE}.close_old_connections"),
patch(f"{_MIXINS_MODULE}.time.sleep") as sleep,
):
@@ -740,13 +740,25 @@ class TestOAuthMixinIntegrationFetchResilience(SimpleTestCase):
assert get.call_count == 1
sleep.assert_not_called()
+ @parameterized.expand([("token",), ("credentials",)])
+ def test_row_of_another_auth_type_is_surfaced_as_not_found(self, auth_type: str):
+ get = mock.Mock(return_value=mock.Mock(auth_type=auth_type))
+
+ with (
+ patch(f"{_MIXINS_MODULE}.IntegrationManager.any.get", get),
+ patch(f"{_MIXINS_MODULE}.close_old_connections"),
+ patch(f"{_MIXINS_MODULE}.time.sleep"),
+ ):
+ with pytest.raises(ValueError, match="^Integration not found: 1$"):
+ OAuthMixin().get_oauth_integration(integration_id=1, team_id=2)
+
def test_deletion_during_retry_is_surfaced_as_not_found(self):
- # The row vanishes after a transient failure: the retry must convert DoesNotExist into the
- # stable non-retryable ValueError rather than letting raw DoesNotExist escape and be retried.
- get = mock.Mock(side_effect=[OperationalError("query_wait_timeout"), Integration.DoesNotExist()])
+ # The row vanishes after a transient failure: the retry must convert IntegrationNotFound into the
+ # stable non-retryable ValueError rather than letting the raw error escape and be retried.
+ get = mock.Mock(side_effect=[OperationalError("query_wait_timeout"), IntegrationNotFound()])
with (
- patch(f"{_MIXINS_MODULE}.Integration.objects.get", get),
+ patch(f"{_MIXINS_MODULE}.IntegrationManager.any.get", get),
patch(f"{_MIXINS_MODULE}.close_old_connections"),
patch(f"{_MIXINS_MODULE}.time.sleep"),
):
@@ -757,7 +769,7 @@ class TestOAuthMixinIntegrationFetchResilience(SimpleTestCase):
@parameterized.expand(
[
- ("deleted_integration", Integration.DoesNotExist(), 4212),
+ ("deleted_integration", IntegrationNotFound(), 4212),
("unset_integration_id", None, 0),
]
)
@@ -770,7 +782,7 @@ class TestOAuthMixinIntegrationFetchResilience(SimpleTestCase):
get = mock.Mock(side_effect=side_effect)
with (
- patch(f"{_MIXINS_MODULE}.Integration.objects.get", get),
+ patch(f"{_MIXINS_MODULE}.IntegrationManager.any.get", get),
patch(f"{_MIXINS_MODULE}.close_old_connections"),
patch(f"{_MIXINS_MODULE}.time.sleep"),
):
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/google_ads.py b/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/google_ads.py
index 0c26ace20e70..557ecfbcc3bc 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/google_ads.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/google_ads.py
@@ -33,7 +33,7 @@ from google.auth import exceptions as google_auth_exceptions
from google.oauth2 import service_account
from google.protobuf.json_format import MessageToJson
-from posthog.models.integration import Integration
+from posthog.integrations import IntegrationManager, IntegrationNotFound, IntegrationRecord
from products.warehouse_sources.backend.temporal.data_imports.naming_convention import NamingConvention
from products.warehouse_sources.backend.temporal.data_imports.pipelines.helpers import incremental_type_to_initial_value
@@ -205,7 +205,7 @@ def _load_client_with_transient_retry(
_MAX_INTEGRATION_FETCH_ATTEMPTS = 4
-def _get_integration(integration_id: int, team_id: int) -> Integration:
+def _get_integration(integration_id: int, team_id: int) -> IntegrationRecord:
"""Fetch the OAuth ``Integration`` row, retrying a transient DB failure with backoff.
Temporal activities run in a long-lived worker that never goes through Django's request
@@ -216,13 +216,16 @@ def _get_integration(integration_id: int, team_id: int) -> Integration:
query marks one unusable, drops it), so each attempt runs on a fresh connection; the short
backoff also gives a saturated pool time to drain rather than retrying straight back into the
same wait timeout. This read is idempotent, so it is safe to repeat. Mirrors the backoff shape
- of the client-init and search retries. ``Integration.DoesNotExist`` is left to propagate.
+ of the client-init and search retries. A missing row raises ``IntegrationNotFound`` with the text "Integration not found: <id>".
"""
attempt = 0
while True:
close_old_connections()
try:
- return Integration.objects.get(id=integration_id, team_id=team_id)
+ return IntegrationManager.google_ads.oauth.get(team_id, integration_id)
+ except IntegrationNotFound as error:
+ # A fixed text, because the error maps match on it. The SDK message is not part of its contract.
+ raise IntegrationNotFound(f"Integration not found: {integration_id}") from error
except OperationalError:
attempt += 1
if attempt >= _MAX_INTEGRATION_FETCH_ATTEMPTS:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/source.py
index 8cbc772e9747..8753575e9243 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/source.py
@@ -7,12 +7,12 @@ from django.core.cache import cache
import requests
from rest_framework.exceptions import ValidationError
+from posthog.integrations import IntegrationNotFound
from posthog.models.integration import (
ERROR_TOKEN_REFRESH_FAILED,
GoogleAdsIntegration,
GoogleAdsQuotaExhaustedError,
GoogleAdsScopeInsufficientError,
- Integration,
OauthIntegration,
google_ads_hierarchy_level,
)
@@ -188,6 +188,8 @@ class GoogleAdsSource(
# recover — the user must reconnect their Google Ads account. Model-specific so we don't
# swallow unrelated `DoesNotExist` errors from other models, which may be real bugs.
"Integration matching query does not exist": "Your Google Ads connection is no longer available — it may have been disconnected. Please reconnect your Google Ads account.",
+ # `_get_integration` raises `IntegrationNotFound("Integration not found: <id>")` for the same case.
+ "Integration not found": "Your Google Ads connection is no longer available — it may have been disconnected. Please reconnect your Google Ads account.",
# gapic wraps a transport-level UNAUTHENTICATED into google.api_core.exceptions.Unauthenticated,
# whose str() is "401 Request is missing required authentication credential. ..." — it never
# contains the bare "UNAUTHENTICATED" token, so the gRPC-status keys above don't catch it.
@@ -587,7 +589,7 @@ class GoogleAdsSource(
'enable "Using MCC account?" and enter your manager\'s customer ID, then try again.',
)
return True, None
- except Integration.DoesNotExist:
+ except IntegrationNotFound:
return (
False,
"The Google Ads connection for this source no longer exists. Please reconnect your Google Ads account.",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/tests/test_google_ads_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/tests/test_google_ads_source.py
index e27849a30c45..775f7be28b74 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/tests/test_google_ads_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/tests/test_google_ads_source.py
@@ -25,7 +25,7 @@ from google.ads.googleads.v23.resources.types.change_event import ChangeEvent
from google.api_core import exceptions as google_api_exceptions
from google.auth import exceptions as google_auth_exceptions
-from posthog.models.integration import Integration
+from posthog.integrations import IntegrationNotFound
from products.warehouse_sources.backend.temporal.data_imports.sources.common.integration_accounts import (
IntegrationAccountListingError,
@@ -238,9 +238,10 @@ class TestGoogleAdsNonRetryableErrors:
@pytest.mark.parametrize(
"error_msg",
[
- # `str(Integration.DoesNotExist)` as raised by `google_ads_client` during a sync when
- # the OAuth integration row has been deleted/disconnected.
+ # The ORM text for a deleted OAuth integration row.
"Integration matching query does not exist.",
+ # `_get_integration` raises `IntegrationNotFound` with this text for the same case.
+ "Integration not found: 42",
],
)
def test_missing_integration_is_non_retryable(self, error_msg):
@@ -479,7 +480,7 @@ class TestValidateCredentials:
config = GoogleAdsSourceConfig(customer_id="1234567890", google_ads_integration_id=1, start_date=" 2020-01-01 ")
with mock.patch(
"products.warehouse_sources.backend.temporal.data_imports.sources.google_ads.google_ads.google_ads_client",
- side_effect=Integration.DoesNotExist(),
+ side_effect=IntegrationNotFound(),
):
ok, message = GoogleAdsSource().validate_credentials(config, team_id=1)
@@ -489,12 +490,12 @@ class TestValidateCredentials:
def test_missing_integration_does_not_exist_returns_reconnect_message(self):
# `google_ads_client` calls `Integration.objects.get(...)`, which raises the typed
- # `Integration.DoesNotExist` when the OAuth connection row is gone. Surface an
+ # `IntegrationNotFound` when the OAuth connection row is gone. Surface an
# actionable reconnect message instead of the raw ORM error.
config = GoogleAdsSourceConfig(customer_id="1234567890", google_ads_integration_id=1)
with mock.patch(
"products.warehouse_sources.backend.temporal.data_imports.sources.google_ads.google_ads.google_ads_client",
- side_effect=Integration.DoesNotExist(),
+ side_effect=IntegrationNotFound(),
):
ok, message = GoogleAdsSource().validate_credentials(config, team_id=1)
@@ -1351,9 +1352,7 @@ class TestSearchFieldsTransientRetry:
assert sleep.call_count == 0
-_INTEGRATION_GET_PATH = (
- "products.warehouse_sources.backend.temporal.data_imports.sources.google_ads.google_ads.Integration.objects.get"
-)
+_INTEGRATION_GET_PATH = "products.warehouse_sources.backend.temporal.data_imports.sources.google_ads.google_ads.IntegrationManager.google_ads.oauth.get"
_CLOSE_CONNECTIONS_PATH = (
"products.warehouse_sources.backend.temporal.data_imports.sources.google_ads.google_ads.close_old_connections"
)
@@ -1418,10 +1417,10 @@ class TestGetIntegrationDbResilience:
assert sleep.call_args_list == [mock.call(2), mock.call(4), mock.call(6)]
def test_missing_integration_is_not_retried(self):
- get = mock.Mock(side_effect=Integration.DoesNotExist())
+ get = mock.Mock(side_effect=IntegrationNotFound("SDK text"))
with mock.patch(_INTEGRATION_GET_PATH, get), mock.patch(_CLOSE_CONNECTIONS_PATH):
- with pytest.raises(Integration.DoesNotExist):
+ with pytest.raises(IntegrationNotFound, match="^Integration not found: 1$"):
_get_integration(integration_id=1, team_id=2)
# A deleted connection row is non-retryable elsewhere — don't mask it as a transient drop.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/google_analytics/google_analytics.py b/products/warehouse_sources/backend/temporal/data_imports/sources/google_analytics/google_analytics.py
index 37c9621eafee..20b958de0681 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/google_analytics/google_analytics.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/google_analytics/google_analytics.py
@@ -13,7 +13,7 @@ from google.auth.transport.requests import AuthorizedSession
from google.oauth2.credentials import Credentials as OAuthCredentials
from posthog.dataclasses import frozen
-from posthog.models.integration import Integration
+from posthog.integrations import IntegrationManager, IntegrationNotFound, IntegrationRecord
from products.warehouse_sources.backend.temporal.data_imports.naming_convention import NamingConvention
from products.warehouse_sources.backend.temporal.data_imports.sources.common import credentials_service
@@ -85,7 +85,7 @@ def _backoff_sleep(attempt: int) -> None:
_MAX_INTEGRATION_FETCH_ATTEMPTS = 4
-def _get_integration(integration_id: int, team_id: int) -> Integration:
+def _get_integration(integration_id: int, team_id: int) -> IntegrationRecord:
"""Fetch the OAuth ``Integration`` row, retrying a transient DB failure with backoff.
Temporal activities run in a long-lived worker that never goes through Django's request
@@ -95,14 +95,17 @@ def _get_integration(integration_id: int, team_id: int) -> Integration:
``close_old_connections()`` evicts connections already known to be stale (and, after a failed
query marks one unusable, drops it), so each attempt runs on a fresh connection; the short
backoff also gives a saturated pool time to drain rather than retrying straight back into the
- same wait timeout. This read is idempotent, so it is safe to repeat. ``Integration.DoesNotExist``
- is left to propagate.
+ same wait timeout. This read is idempotent, so it is safe to repeat. A missing row raises ``IntegrationNotFound``
+ with the text "Integration not found: <id>".
"""
attempt = 0
while True:
close_old_connections()
try:
- return Integration.objects.get(id=integration_id, team_id=team_id)
+ return IntegrationManager.google_analytics.oauth.get(team_id, integration_id)
+ except IntegrationNotFound as error:
+ # A fixed text, because the error maps match on it. The SDK message is not part of its contract.
+ raise IntegrationNotFound(f"Integration not found: {integration_id}") from error
except OperationalError:
attempt += 1
if attempt >= _MAX_INTEGRATION_FETCH_ATTEMPTS:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/google_analytics/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/google_analytics/source.py
index 759181b5a200..28462143865d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/google_analytics/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/google_analytics/source.py
@@ -5,7 +5,7 @@ import structlog
from google.auth.exceptions import RefreshError
from posthog.exceptions_capture import capture_exception
-from posthog.models.integration import Integration
+from posthog.integrations import IntegrationNotFound
from posthog.schema_enums import ProductKey
from products.warehouse_sources.backend.facade.codebase_detection import detect
@@ -326,7 +326,7 @@ class GoogleAnalyticsSource(ResumableSource[GoogleAnalyticsSourceConfig, GoogleA
try:
session = google_analytics_session(config.google_analytics_integration_id, team_id)
- except Integration.DoesNotExist:
+ except IntegrationNotFound:
# The stored OAuth integration row has been deleted/disconnected before validation runs.
# Caught explicitly so an unrelated model's DoesNotExist still surfaces as a real bug below.
return (
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/google_analytics/tests/test_google_analytics.py b/products/warehouse_sources/backend/temporal/data_imports/sources/google_analytics/tests/test_google_analytics.py
index 2c33940a06b4..c7fbccee1069 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/google_analytics/tests/test_google_analytics.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/google_analytics/tests/test_google_analytics.py
@@ -11,7 +11,7 @@ import requests
from google.auth.exceptions import RefreshError
from google.oauth2.credentials import Credentials as OAuthCredentials
-from posthog.models.integration import Integration
+from posthog.integrations import IntegrationNotFound
from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
RecordedRequest,
@@ -111,20 +111,20 @@ def test_credentials_refreshes_stale_db_connection_before_query(monkeypatch):
integration.refresh_token = "refresh-token"
def fake_get(*args, **kwargs):
- calls.append("Integration.objects.get")
+ calls.append("IntegrationManager.get")
return integration
- monkeypatch.setattr(ga.Integration.objects, "get", fake_get)
+ monkeypatch.setattr(ga.IntegrationManager.google_analytics.oauth, "get", fake_get)
creds = _credentials(integration_id=1, team_id=1)
- assert calls == ["close_old_connections", "Integration.objects.get"]
+ assert calls == ["close_old_connections", "IntegrationManager.get"]
assert creds.refresh_token == "refresh-token"
_INTEGRATION_GET_PATH = (
"products.warehouse_sources.backend.temporal.data_imports.sources.google_analytics.google_analytics."
- "Integration.objects.get"
+ "IntegrationManager.google_analytics.oauth.get"
)
_CLOSE_CONNECTIONS_PATH = (
"products.warehouse_sources.backend.temporal.data_imports.sources.google_analytics.google_analytics."
@@ -151,10 +151,10 @@ class TestGetIntegrationDbResilience:
assert get.call_count == 4
def test_missing_integration_is_not_retried(self):
- get = mock.Mock(side_effect=Integration.DoesNotExist())
+ get = mock.Mock(side_effect=IntegrationNotFound("SDK text"))
with mock.patch(_INTEGRATION_GET_PATH, get), mock.patch(_CLOSE_CONNECTIONS_PATH), mock.patch(_SLEEP_PATH):
- with pytest.raises(Integration.DoesNotExist):
+ with pytest.raises(IntegrationNotFound, match="^Integration not found: 1$"):
_get_integration(integration_id=1, team_id=2)
# A deleted integration row is non-retryable — don't mask it as a transient drop.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/google_analytics/tests/test_google_analytics_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/google_analytics/tests/test_google_analytics_source.py
index 05a3c45c7b77..af3e594b758b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/google_analytics/tests/test_google_analytics_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/google_analytics/tests/test_google_analytics_source.py
@@ -4,7 +4,7 @@ from unittest import mock
import requests
from google.auth.exceptions import RefreshError
-from posthog.models.integration import Integration
+from posthog.integrations import IntegrationNotFound
from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import error_message_matches
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.googleanalytics import (
@@ -227,10 +227,10 @@ def test_validate_credentials_hides_unexpected_metadata_failure_detail():
def test_validate_credentials_handles_missing_integration():
# A deleted/disconnected OAuth row makes `google_analytics_session` raise the typed
- # `Integration.DoesNotExist`; surface a reconnect message instead of the raw ORM error.
+ # `IntegrationNotFound`; surface a reconnect message instead of the raw ORM error.
with mock.patch(
"products.warehouse_sources.backend.temporal.data_imports.sources.google_analytics.source.google_analytics_session",
- side_effect=Integration.DoesNotExist(),
+ side_effect=IntegrationNotFound(),
):
ok, message = GoogleAnalyticsSource().validate_credentials(_config(), team_id=1)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/google_search_console/google_search_console.py b/products/warehouse_sources/backend/temporal/data_imports/sources/google_search_console/google_search_console.py
index af6ca9c09f91..944772741e7a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/google_search_console/google_search_console.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/google_search_console/google_search_console.py
@@ -14,7 +14,7 @@ from google.auth.exceptions import RefreshError, TransportError
from google.auth.transport.requests import AuthorizedSession
from google.oauth2.credentials import Credentials as OAuthCredentials
-from posthog.models.integration import Integration
+from posthog.integrations import IntegrationManager, IntegrationNotFound, IntegrationRecord
from products.warehouse_sources.backend.temporal.data_imports.naming_convention import NamingConvention
from products.warehouse_sources.backend.temporal.data_imports.sources.common import credentials_service
@@ -246,7 +246,7 @@ def _backoff_sleep(attempt: int) -> None:
_MAX_INTEGRATION_FETCH_ATTEMPTS = 4
-def _get_integration(integration_id: int, team_id: int) -> Integration:
+def _get_integration(integration_id: int, team_id: int) -> IntegrationRecord:
"""Fetch the OAuth ``Integration`` row, retrying a transient DB failure with backoff.
Temporal activities run in a long-lived worker outside Django's request cycle, and this
@@ -258,13 +258,16 @@ def _get_integration(integration_id: int, team_id: int) -> Integration:
failed query marks one unusable, drops it), so each attempt runs on a fresh connection; the
short backoff also gives a saturated pool time to drain rather than retrying straight back
into the same wait timeout. This read is idempotent, so repeating it is safe.
- ``Integration.DoesNotExist`` is left to propagate.
+ A missing row raises ``IntegrationNotFound`` with the text "Integration not found: <id>".
"""
attempt = 0
while True:
close_old_connections()
try:
- return Integration.objects.get(id=integration_id, team_id=team_id)
+ return IntegrationManager.google_search_console.oauth.get(team_id, integration_id)
+ except IntegrationNotFound as error:
+ # A fixed text, because the error maps match on it. The SDK message is not part of its contract.
+ raise IntegrationNotFound(f"Integration not found: {integration_id}") from error
except OperationalError:
attempt += 1
if attempt >= _MAX_INTEGRATION_FETCH_ATTEMPTS:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/google_search_console/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/google_search_console/source.py
index fa52d10d4414..3d498a4f5df1 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/google_search_console/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/google_search_console/source.py
@@ -4,7 +4,7 @@ import requests
from google.auth.exceptions import RefreshError
from posthog.exceptions_capture import capture_exception
-from posthog.models.integration import Integration
+from posthog.integrations import IntegrationNotFound
from products.warehouse_sources.backend.facade.codebase_detection import detect
from products.warehouse_sources.backend.facade.source_config import (
@@ -122,6 +122,8 @@ class GoogleSearchConsoleSource(
# references an OAuth integration row that has since been deleted (account disconnected).
# No retry can recreate the row, so stop and ask the user to reconnect.
"Integration matching query does not exist": "The Google Search Console connection for this source no longer exists. Please reconnect your Google account.",
+ # `_get_integration` raises `IntegrationNotFound("Integration not found: <id>")` for the same case.
+ "Integration not found": "The Google Search Console connection for this source no longer exists. Please reconnect your Google account.",
# `RefreshError: invalid_grant` is raised while AuthorizedSession refreshes the OAuth
# access token — the stored refresh token has been revoked, expired, or invalidated
# (app access revoked, password change, long inactivity). It never recovers on retry,
@@ -157,7 +159,7 @@ class GoogleSearchConsoleSource(
# Search Console sites are few, so `search` is ignored here and the endpoint filters the list.
try:
session = google_search_console_session(integration_id, team_id)
- except Integration.DoesNotExist:
+ except IntegrationNotFound:
raise IntegrationAccountListingError(
"The Google Search Console connection for this source no longer exists. "
"Please reconnect your Google account."
@@ -298,7 +300,7 @@ class GoogleSearchConsoleSource(
try:
session = google_search_console_session(config.google_search_console_integration_id, team_id)
- except Integration.DoesNotExist:
+ except IntegrationNotFound:
return (
False,
"The Google Search Console connection for this source no longer exists. Please reconnect your Google account.",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/google_search_console/tests/test_google_search_console.py b/products/warehouse_sources/backend/temporal/data_imports/sources/google_search_console/tests/test_google_search_console.py
index 9bd527ea8456..33ffdb097cf5 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/google_search_console/tests/test_google_search_console.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/google_search_console/tests/test_google_search_console.py
@@ -8,6 +8,8 @@ from django.db import OperationalError
import requests
from google.auth.exceptions import RefreshError, TransportError
+from posthog.integrations import IntegrationNotFound
+
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.googlesearchconsole import (
GoogleSearchConsoleSourceConfig,
)
@@ -99,14 +101,14 @@ def test_credentials_refreshes_stale_db_connection_before_query(monkeypatch):
integration.refresh_token = "refresh-token"
def fake_get(*args, **kwargs):
- calls.append("Integration.objects.get")
+ calls.append("IntegrationManager.get")
return integration
- monkeypatch.setattr(gsc.Integration.objects, "get", fake_get)
+ monkeypatch.setattr(gsc.IntegrationManager.google_search_console.oauth, "get", fake_get)
creds = _credentials(integration_id=1, team_id=1)
- assert calls == ["close_old_connections", "Integration.objects.get"]
+ assert calls == ["close_old_connections", "IntegrationManager.get"]
assert creds.refresh_token == "refresh-token"
@@ -114,7 +116,7 @@ def test_get_integration_reraises_after_exhausting_attempts(monkeypatch):
get = mock.Mock(side_effect=OperationalError("query_wait_timeout"))
monkeypatch.setattr(gsc, "close_old_connections", lambda: None)
- monkeypatch.setattr(gsc.Integration.objects, "get", get)
+ monkeypatch.setattr(gsc.IntegrationManager.google_search_console.oauth, "get", get)
sleeps: list[float] = []
monkeypatch.setattr(gsc.time, "sleep", lambda seconds: sleeps.append(seconds))
@@ -126,6 +128,17 @@ def test_get_integration_reraises_after_exhausting_attempts(monkeypatch):
assert sleeps == [2, 4, 6]
+def test_get_integration_raises_a_stable_missing_text_without_retry(monkeypatch):
+ get = mock.Mock(side_effect=IntegrationNotFound("SDK text"))
+ monkeypatch.setattr(gsc, "close_old_connections", lambda: None)
+ monkeypatch.setattr(gsc.IntegrationManager.google_search_console.oauth, "get", get)
+
+ with pytest.raises(IntegrationNotFound, match="^Integration not found: 1$"):
+ _get_integration(integration_id=1, team_id=2)
+
+ assert get.call_count == 1
+
+
def _make_response(config: GoogleSearchConsoleSourceConfig, rows_per_call: list[list[dict]]):
"""Build a SourceResponse where _query_search_analytics returns the given pages in order."""
inputs = mock.MagicMock()
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/google_search_console/tests/test_google_search_console_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/google_search_console/tests/test_google_search_console_source.py
index a29e25c20761..ee7ea0e16797 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/google_search_console/tests/test_google_search_console_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/google_search_console/tests/test_google_search_console_source.py
@@ -3,6 +3,8 @@ from unittest import mock
import requests
+from posthog.integrations import IntegrationNotFound
+
from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import error_message_matches
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.googlesearchconsole import (
GoogleSearchConsoleSourceConfig,
@@ -164,11 +166,10 @@ def test_get_oauth_accounts_names_a_next_step_when_the_account_owns_no_property(
def test_validate_credentials_missing_integration_returns_reconnect_message():
- from posthog.models.integration import Integration
with mock.patch(
"products.warehouse_sources.backend.temporal.data_imports.sources.google_search_console.source.google_search_console_session",
- side_effect=Integration.DoesNotExist(),
+ side_effect=IntegrationNotFound(),
):
ok, message = GoogleSearchConsoleSource().validate_credentials(_config(), team_id=1)
@@ -317,7 +318,7 @@ def test_validate_credentials_normalizes_site_url_before_lookup(entered, site_ur
def test_validate_credentials_handles_missing_integration():
# A disconnected/deleted OAuth integration makes the credentials lookup raise
- # `Integration.DoesNotExist` ("... matching query does not exist"). Surface an
+ # `IntegrationNotFound` ("... matching query does not exist"). Surface an
# actionable reconnect message instead of the raw ORM error.
with mock.patch(
"products.warehouse_sources.backend.temporal.data_imports.sources.google_search_console.source.google_search_console_session",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/helpscout/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/helpscout/source.py
index 2409c839b898..fde16f5f76b6 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/helpscout/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/helpscout/source.py
@@ -1,6 +1,7 @@
from typing import Optional, cast
-from posthog.models.integration import Integration, OauthIntegration
+from posthog.integrations import IntegrationKind
+from posthog.models.integration import OauthIntegration
from products.warehouse_sources.backend.facade.codebase_detection import detect
from products.warehouse_sources.backend.facade.source_config import (
@@ -108,7 +109,7 @@ class HelpScoutSource(OAuthMixin, ResumableSource[HelpScoutSourceConfig, HelpSco
# The lookup only scopes to the team, so a config pointing at some other provider's
# integration in the same team would otherwise send that provider's access token to the
# Help Scout API. Reject it before the token is read or refreshed.
- if integration.kind != Integration.IntegrationKind.HELPSCOUT:
+ if integration.kind != IntegrationKind.HELPSCOUT:
raise ValueError(f"Integration not found: {config.helpscout_integration_id}")
oauth_integration = OauthIntegration(integration)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/linkedin_ads/linkedin_ads.py b/products/warehouse_sources/backend/temporal/data_imports/sources/linkedin_ads/linkedin_ads.py
index a66b91ff947d..252c836a1270 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/linkedin_ads/linkedin_ads.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/linkedin_ads/linkedin_ads.py
@@ -10,7 +10,8 @@ import pyarrow as pa
import structlog
from structlog.types import FilteringBoundLogger
-from posthog.models.integration import ERROR_TOKEN_REFRESH_FAILED, Integration, OauthIntegration
+from posthog.integrations import IntegrationManager, IntegrationNotFound, IntegrationRecord
+from posthog.models.integration import ERROR_TOKEN_REFRESH_FAILED, OauthIntegration
from products.warehouse_sources.backend.temporal.data_imports.naming_convention import NamingConvention
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.batcher import Batcher
@@ -146,7 +147,7 @@ def _backoff_sleep(attempt: int) -> None:
_MAX_INTEGRATION_FETCH_ATTEMPTS = 4
-def _get_integration(integration_id: int, team_id: int) -> Integration:
+def _get_integration(integration_id: int, team_id: int) -> IntegrationRecord:
"""Fetch the OAuth ``Integration`` row, retrying a transient DB failure with backoff.
Temporal activities run in a long-lived worker that never goes through Django's request
@@ -156,14 +157,17 @@ def _get_integration(integration_id: int, team_id: int) -> Integration:
``close_old_connections()`` evicts connections already known to be stale (and, after a failed
query marks one unusable, drops it), so each attempt runs on a fresh connection; the short
backoff also gives a saturated pool time to drain rather than retrying straight back into the
- same wait timeout. This read is idempotent, so it is safe to repeat. ``Integration.DoesNotExist``
- is left to propagate.
+ same wait timeout. This read is idempotent, so it is safe to repeat. A missing row raises ``IntegrationNotFound``
+ with the text "Integration not found: <id>".
"""
attempt = 0
while True:
close_old_connections()
try:
- return Integration.objects.get(id=integration_id, team_id=team_id)
+ return IntegrationManager.linkedin_ads.oauth.get(team_id, integration_id)
+ except IntegrationNotFound as error:
+ # A fixed text, because the error maps match on it. The SDK message is not part of its contract.
+ raise IntegrationNotFound(f"Integration not found: {integration_id}") from error
except OperationalError:
attempt += 1
if attempt >= _MAX_INTEGRATION_FETCH_ATTEMPTS:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/linkedin_ads/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/linkedin_ads/source.py
index 2d914af98e0b..52aa9669230c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/linkedin_ads/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/linkedin_ads/source.py
@@ -2,7 +2,7 @@ from datetime import date
from typing import Optional, cast
from posthog.exceptions_capture import capture_exception
-from posthog.models.integration import Integration
+from posthog.integrations import IntegrationManager, IntegrationNotFound
from products.warehouse_sources.backend.facade.codebase_detection import detect
from products.warehouse_sources.backend.facade.source_config import (
@@ -156,6 +156,8 @@ class LinkedInAdsSource(ResumableSource[LinkedinAdsSourceConfig, LinkedInAdsResu
# user must re-authorize. Model-specific so we don't swallow unrelated `DoesNotExist`
# errors from other models, which may be real bugs.
"Integration matching query does not exist": "Your LinkedIn Ads connection is no longer available — it may have been disconnected. Please re-authorize the LinkedIn Ads integration.",
+ # `_get_integration` raises `IntegrationNotFound("Integration not found: <id>")` for the same case.
+ "Integration not found": "Your LinkedIn Ads connection is no longer available — it may have been disconnected. Please re-authorize the LinkedIn Ads integration.",
# LinkedIn rejects a deprecated/sunset version header with this stable code (see
# `deprecated_versions`). It happens on every call under that pin regardless of resource
# or account, so retrying can never succeed — only repinning to a supported version does.
@@ -226,7 +228,7 @@ class LinkedInAdsSource(ResumableSource[LinkedinAdsSourceConfig, LinkedInAdsResu
client = linkedin_ads_client_for_integration(
integration_id, team_id, api_version=_API_HEADER_BY_VERSION[self.default_version]
)
- except Integration.DoesNotExist as e:
+ except IntegrationNotFound as e:
raise IntegrationAccountListingError(
"Your LinkedIn Ads connection is no longer available — it may have been disconnected. "
"Please re-authorize the LinkedIn Ads integration."
@@ -283,9 +285,9 @@ class LinkedInAdsSource(ResumableSource[LinkedinAdsSourceConfig, LinkedInAdsResu
)
try:
- Integration.objects.get(id=config.linkedin_ads_integration_id, team_id=team_id)
+ IntegrationManager.linkedin_ads.oauth.get(team_id, config.linkedin_ads_integration_id)
return True, None
- except Integration.DoesNotExist:
+ except IntegrationNotFound:
return False, _MISSING_INTEGRATION_ERROR
except Exception as e:
capture_exception(e)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/linkedin_ads/tests/test_linkedin_ads.py b/products/warehouse_sources/backend/temporal/data_imports/sources/linkedin_ads/tests/test_linkedin_ads.py
index d797f6d065c0..293fc43069cf 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/linkedin_ads/tests/test_linkedin_ads.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/linkedin_ads/tests/test_linkedin_ads.py
@@ -11,7 +11,8 @@ from django.db import OperationalError
import pyarrow as pa
from parameterized import parameterized
-from posthog.models.integration import ERROR_TOKEN_REFRESH_FAILED, Integration
+from posthog.integrations import IntegrationNotFound
+from posthog.models.integration import ERROR_TOKEN_REFRESH_FAILED
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import (
evolve_pyarrow_schema,
@@ -494,7 +495,9 @@ class TestFlattenLinkedinRecord:
assert result["created_time"] is None
[email protected]("products.warehouse_sources.backend.temporal.data_imports.sources.linkedin_ads.linkedin_ads.Integration")
+ "products.warehouse_sources.backend.temporal.data_imports.sources.linkedin_ads.linkedin_ads.IntegrationManager"
+)
class TestLinkedinAdsClientFunction:
"""Test linkedin_ads_client function."""
@@ -502,7 +505,7 @@ class TestLinkedinAdsClientFunction:
mock_integration = mock.MagicMock()
mock_integration.access_token = None
mock_integration.sensitive_config = {} # no refresh token → not expired, skips refresh
- mock_integration_model.objects.get.return_value = mock_integration
+ mock_integration_model.linkedin_ads.oauth.get.return_value = mock_integration
config = LinkedinAdsSourceConfig(linkedin_ads_integration_id=123, account_id="456")
@@ -526,20 +529,18 @@ class TestLinkedinAdsClientFunction:
mock_integration.sensitive_config = {} # no refresh token → not expired, skips refresh
def fake_get(*args, **kwargs):
- calls.append("Integration.objects.get")
+ calls.append("IntegrationManager.get")
return mock_integration
- mock_integration_model.objects.get.side_effect = fake_get
+ mock_integration_model.linkedin_ads.oauth.get.side_effect = fake_get
config = LinkedinAdsSourceConfig(linkedin_ads_integration_id=123, account_id="456")
linkedin_ads_client(config, team_id=789)
- assert calls == ["close_old_connections", "Integration.objects.get"]
+ assert calls == ["close_old_connections", "IntegrationManager.get"]
-_INTEGRATION_GET_PATH = (
- "products.warehouse_sources.backend.temporal.data_imports.sources.linkedin_ads.linkedin_ads.Integration.objects.get"
-)
+_INTEGRATION_GET_PATH = "products.warehouse_sources.backend.temporal.data_imports.sources.linkedin_ads.linkedin_ads.IntegrationManager.linkedin_ads.oauth.get"
_CLOSE_CONNECTIONS_PATH = (
"products.warehouse_sources.backend.temporal.data_imports.sources.linkedin_ads.linkedin_ads.close_old_connections"
)
@@ -602,10 +603,10 @@ class TestGetIntegrationDbResilience:
assert sleep.call_args_list == [mock.call(2), mock.call(4), mock.call(6)]
def test_missing_integration_is_not_retried(self):
- get = mock.Mock(side_effect=Integration.DoesNotExist())
+ get = mock.Mock(side_effect=IntegrationNotFound("SDK text"))
with mock.patch(_INTEGRATION_GET_PATH, get), mock.patch(_CLOSE_CONNECTIONS_PATH), mock.patch(_SLEEP_PATH):
- with pytest.raises(Integration.DoesNotExist):
+ with pytest.raises(IntegrationNotFound, match="^Integration not found: 1$"):
_get_integration(integration_id=1, team_id=2)
# A deleted integration row is non-retryable — don't mask it as a transient drop.
@@ -615,7 +616,9 @@ class TestGetIntegrationDbResilience:
@mock.patch(
"products.warehouse_sources.backend.temporal.data_imports.sources.linkedin_ads.linkedin_ads.OauthIntegration"
)
[email protected]("products.warehouse_sources.backend.temporal.data_imports.sources.linkedin_ads.linkedin_ads.Integration")
+ "products.warehouse_sources.backend.temporal.data_imports.sources.linkedin_ads.linkedin_ads.IntegrationManager"
+)
class TestLinkedinAdsClientTokenRefresh:
"""Token refresh behaviour of linkedin_ads_client."""
@@ -631,7 +634,7 @@ class TestLinkedinAdsClientTokenRefresh:
integration = mock.MagicMock()
integration.access_token = "refreshed-token"
integration.errors = ""
- mock_integration_model.objects.get.return_value = integration
+ mock_integration_model.linkedin_ads.oauth.get.return_value = integration
oauth = mock_oauth_cls.return_value
oauth.access_token_expired.return_value = token_expired
@@ -648,7 +651,7 @@ class TestLinkedinAdsClientTokenRefresh:
integration = mock.MagicMock()
integration.access_token = "stale-token"
integration.errors = ERROR_TOKEN_REFRESH_FAILED
- mock_integration_model.objects.get.return_value = integration
+ mock_integration_model.linkedin_ads.oauth.get.return_value = integration
oauth = mock_oauth_cls.return_value
oauth.access_token_expired.return_value = True
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/linkedin_ads/tests/test_linkedin_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/linkedin_ads/tests/test_linkedin_source.py
index 3d348ba03c80..d42cad329d1c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/linkedin_ads/tests/test_linkedin_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/linkedin_ads/tests/test_linkedin_source.py
@@ -4,6 +4,8 @@ from datetime import date
import pytest
from unittest import mock
+from posthog.integrations import IntegrationNotFound
+
from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import VersionDeprecation
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.linkedinads import (
LinkedinAdsSourceConfig,
@@ -202,31 +204,25 @@ class TestLinkedInAdsSource:
assert error_message is not None
assert "numeric account ID" in error_message
- @mock.patch("products.warehouse_sources.backend.temporal.data_imports.sources.linkedin_ads.source.Integration")
+ @mock.patch(
+ "products.warehouse_sources.backend.temporal.data_imports.sources.linkedin_ads.source.IntegrationManager"
+ )
def test_validate_credentials_integration_not_found(self, mock_integration_model):
- # Mock DoesNotExist exception
- class MockDoesNotExist(Exception):
- pass
-
- mock_integration_model.DoesNotExist = MockDoesNotExist
- mock_integration_model.objects.get.side_effect = MockDoesNotExist()
+ mock_integration_model.linkedin_ads.oauth.get.side_effect = IntegrationNotFound()
is_valid, error_message = self.source.validate_credentials(self.config, self.team_id)
assert is_valid is False
assert error_message == _MISSING_INTEGRATION_ERROR
- @mock.patch("products.warehouse_sources.backend.temporal.data_imports.sources.linkedin_ads.source.Integration")
+ @mock.patch(
+ "products.warehouse_sources.backend.temporal.data_imports.sources.linkedin_ads.source.IntegrationManager"
+ )
@mock.patch(
"products.warehouse_sources.backend.temporal.data_imports.sources.linkedin_ads.source.capture_exception"
)
def test_validate_credentials_unexpected_error(self, mock_capture_exception, mock_integration_model):
- # Mock DoesNotExist exception
- class MockDoesNotExist(Exception):
- pass
-
- mock_integration_model.DoesNotExist = MockDoesNotExist
- mock_integration_model.objects.get.side_effect = Exception("Database error")
+ mock_integration_model.linkedin_ads.oauth.get.side_effect = Exception("Database error")
is_valid, error_message = self.source.validate_credentials(self.config, self.team_id)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/meta_ads.py b/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/meta_ads.py
index 1f6b9b8d7e44..874f098c1de7 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/meta_ads.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/meta_ads.py
@@ -19,7 +19,8 @@ from requests.exceptions import (
)
from posthog.dataclasses import frozen
-from posthog.models.integration import ERROR_TOKEN_REFRESH_FAILED, Integration, MetaAdsIntegration
+from posthog.integrations import IntegrationManager, IntegrationNotFound, IntegrationRecord
+from posthog.models.integration import ERROR_TOKEN_REFRESH_FAILED, MetaAdsIntegration
from products.warehouse_sources.backend.temporal.data_imports.naming_convention import NamingConvention
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
@@ -145,7 +146,7 @@ def _backoff_sleep(attempt: int) -> None:
_MAX_INTEGRATION_FETCH_ATTEMPTS = 4
-def _fetch_integration_row(integration_id: int, team_id: int) -> Integration:
+def _fetch_integration_row(integration_id: int, team_id: int) -> IntegrationRecord:
"""Fetch the OAuth ``Integration`` row, retrying a transient DB failure with backoff.
Temporal activities run in a long-lived worker outside Django's request cycle, so a pooled
@@ -156,13 +157,16 @@ def _fetch_integration_row(integration_id: int, team_id: int) -> Integration:
to be stale (and, after a failed query marks one unusable, drops it), so each attempt runs on a
fresh connection; the short backoff also gives a saturated pool time to drain rather than
retrying straight back into the same wait timeout. This read is idempotent, so it is safe to
- repeat. ``Integration.DoesNotExist`` is left to propagate.
+ repeat. A missing row raises ``IntegrationNotFound`` with the text "Integration not found: <id>".
"""
attempt = 0
while True:
close_old_connections()
try:
- return Integration.objects.get(id=integration_id, team_id=team_id)
+ return IntegrationManager.meta_ads.oauth.get(team_id, integration_id)
+ except IntegrationNotFound as error:
+ # A fixed text, because the error maps match on it. The SDK message is not part of its contract.
+ raise IntegrationNotFound(f"Integration not found: {integration_id}") from error
except OperationalError:
attempt += 1
if attempt >= _MAX_INTEGRATION_FETCH_ATTEMPTS:
@@ -179,7 +183,7 @@ class MetaAdsTokenRefreshError(Exception):
"""Meta refused to refresh the integration's access token — only re-authorization fixes it."""
-def get_integration_by_id(integration_id: int, team_id: int) -> Integration:
+def get_integration_by_id(integration_id: int, team_id: int) -> IntegrationRecord:
"""Get a Meta Ads integration by row id, with a freshly refreshed access token."""
integration = _fetch_integration_row(integration_id, team_id)
meta_ads_integration = MetaAdsIntegration(integration)
@@ -191,7 +195,7 @@ def get_integration_by_id(integration_id: int, team_id: int) -> Integration:
return meta_ads_integration.integration
-def get_integration(config: MetaAdsSourceConfig, team_id: int) -> Integration:
+def get_integration(config: MetaAdsSourceConfig, team_id: int) -> IntegrationRecord:
return get_integration_by_id(config.meta_ads_integration_id, team_id)
@@ -734,7 +738,7 @@ def _get_ad_account_page(session: Session, url: str, access_token: str, params:
raise IntegrationAccountListingError(META_AD_ACCOUNT_LISTING_UNAVAILABLE_MESSAGE) from e
-def list_ad_accounts(integration: Integration) -> list[dict]:
+def list_ad_accounts(integration: IntegrationRecord) -> list[dict]:
"""Every ad account the connected Meta user can access."""
access_token = integration.sensitive_config["access_token"]
# `redact_values` masks the access token wherever it lands in telemetry — Meta takes it as a query
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/source.py
index c67ce9a8f66e..2d4967d86c31 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/source.py
@@ -1,7 +1,7 @@
from datetime import date
from typing import cast
-from posthog.models.integration import Integration
+from posthog.integrations import IntegrationNotFound
from products.warehouse_sources.backend.facade.codebase_detection import detect
from products.warehouse_sources.backend.facade.source_config import (
@@ -144,6 +144,10 @@ class MetaAdsSource(ResumableSource[MetaAdsSourceConfig, MetaAdsResumeConfig], O
"Integration matching query does not exist.": (
"The Meta Ads integration for this source no longer exists. Please reconnect the Meta Ads integration."
),
+ # `_fetch_integration_row` raises `IntegrationNotFound("Integration not found: <id>")` for the same case.
+ "Integration not found": (
+ "The Meta Ads integration for this source no longer exists. Please reconnect the Meta Ads integration."
+ ),
META_CHECKPOINT_ERROR_MESSAGE: META_CHECKPOINT_ERROR_MESSAGE,
# Graph API code 200: "Ad account owner has NOT granted ads_management or ads_read
# permission." The connected Meta user can't read this ad account's data. Retrying
@@ -397,7 +401,7 @@ class MetaAdsSource(ResumableSource[MetaAdsSourceConfig, MetaAdsResumeConfig], O
# A Meta user's ad accounts are few, so `search` is ignored here and the endpoint filters the list.
try:
integration = get_integration_by_id(integration_id, team_id)
- except Integration.DoesNotExist as e:
+ except IntegrationNotFound as e:
raise IntegrationAccountListingError(
"The linked Meta Ads integration could not be found. Please reconnect your Meta Ads integration."
) from e
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/test_meta_ads.py b/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/test_meta_ads.py
index c9a02fa88022..c86f6bb8653a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/test_meta_ads.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/test_meta_ads.py
@@ -13,6 +13,8 @@ from requests.exceptions import (
JSONDecodeError as RequestsJSONDecodeError,
)
+from posthog.integrations import IntegrationNotFound
+
from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
VersionDeprecation,
error_message_matches,
@@ -1753,8 +1755,10 @@ class TestNonRetryableErrors:
# Token refresh failure raised by get_integration.
META_TOKEN_REFRESH_ERROR_MESSAGE,
# Integration row deleted/de-authorized while the source still references it —
- # get_integration raises Django's Integration.DoesNotExist with this message.
+ # get_integration raises Django's IntegrationNotFound with this message.
"Integration matching query does not exist.",
+ # `_fetch_integration_row` raises `IntegrationNotFound` with this text.
+ "Integration not found: 42",
# 400 from Meta when the ad account no longer belongs to the authorised user.
'Meta API request failed: 400 - {"error":{"message":"(#200) Ad account owner has NOT granted ads_management or ads_read permission.","type":"OAuthException","code":200}}',
# 400 when a specific endpoint cannot be accessed with the granted permissions.
@@ -2113,10 +2117,10 @@ class TestGetIntegration:
integration = mock.MagicMock()
def fake_get(*args: Any, **kwargs: Any) -> mock.MagicMock:
- calls.append("Integration.objects.get")
+ calls.append("IntegrationManager.get")
return integration
- monkeypatch.setattr(meta_ads_module.Integration.objects, "get", fake_get)
+ monkeypatch.setattr(meta_ads_module.IntegrationManager.meta_ads.oauth, "get", fake_get)
meta_ads_integration = mock.MagicMock()
meta_ads_integration.integration = integration
@@ -2126,7 +2130,7 @@ class TestGetIntegration:
config = MetaAdsSourceConfig(account_id="act_123", meta_ads_integration_id=42)
result = get_integration(config, team_id=1)
- assert calls == ["close_old_connections", "Integration.objects.get"]
+ assert calls == ["close_old_connections", "IntegrationManager.get"]
meta_ads_integration.refresh_access_token.assert_called_once()
assert result is integration
@@ -2136,7 +2140,7 @@ class TestFetchIntegrationRowDbResilience:
integration = object()
get = mock.Mock(side_effect=[OperationalError("server closed the connection unexpectedly"), integration])
close = mock.Mock()
- monkeypatch.setattr(meta_ads_module.Integration.objects, "get", get)
+ monkeypatch.setattr(meta_ads_module.IntegrationManager.meta_ads.oauth, "get", get)
monkeypatch.setattr(meta_ads_module, "close_old_connections", close)
monkeypatch.setattr(meta_ads_module.time, "sleep", lambda _s: None)
@@ -2157,7 +2161,7 @@ class TestFetchIntegrationRowDbResilience:
]
)
sleeps: list[int] = []
- monkeypatch.setattr(meta_ads_module.Integration.objects, "get", get)
+ monkeypatch.setattr(meta_ads_module.IntegrationManager.meta_ads.oauth, "get", get)
monkeypatch.setattr(meta_ads_module, "close_old_connections", lambda: None)
monkeypatch.setattr(meta_ads_module.time, "sleep", lambda s: sleeps.append(s))
@@ -2171,7 +2175,7 @@ class TestFetchIntegrationRowDbResilience:
def test_reraises_after_exhausting_attempts(self, monkeypatch) -> None:
get = mock.Mock(side_effect=OperationalError("query_wait_timeout"))
sleeps: list[int] = []
- monkeypatch.setattr(meta_ads_module.Integration.objects, "get", get)
+ monkeypatch.setattr(meta_ads_module.IntegrationManager.meta_ads.oauth, "get", get)
monkeypatch.setattr(meta_ads_module, "close_old_connections", lambda: None)
monkeypatch.setattr(meta_ads_module.time, "sleep", lambda s: sleeps.append(s))
@@ -2184,12 +2188,12 @@ class TestFetchIntegrationRowDbResilience:
assert sleeps == [2, 4, 6]
def test_missing_integration_is_not_retried(self, monkeypatch) -> None:
- get = mock.Mock(side_effect=meta_ads_module.Integration.DoesNotExist())
- monkeypatch.setattr(meta_ads_module.Integration.objects, "get", get)
+ get = mock.Mock(side_effect=IntegrationNotFound("SDK text"))
+ monkeypatch.setattr(meta_ads_module.IntegrationManager.meta_ads.oauth, "get", get)
monkeypatch.setattr(meta_ads_module, "close_old_connections", lambda: None)
monkeypatch.setattr(meta_ads_module.time, "sleep", lambda _s: None)
- with pytest.raises(meta_ads_module.Integration.DoesNotExist):
+ with pytest.raises(IntegrationNotFound, match="^Integration not found: 1$"):
_fetch_integration_row(integration_id=1, team_id=2)
# A deleted integration row is non-retryable — don't mask it as a transient drop.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/resend/oauth.py b/products/warehouse_sources/backend/temporal/data_imports/sources/resend/oauth.py
index b8c0d7501bc4..c99aba61c39e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/resend/oauth.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/resend/oauth.py
@@ -1,10 +1,8 @@
from datetime import UTC, datetime, timedelta
-from django.db import transaction
-
from requests import PreparedRequest
-from posthog.models.integration import Integration, OauthIntegration
+from posthog.integrations import IntegrationManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.auth import AuthConfigBase
@@ -19,20 +17,17 @@ def resolve_resend_oauth_token(integration_id: int, team_id: int, current_token:
Resend access tokens are short-lived (~15m) and its refresh tokens rotate on every use, with
reuse-detection that revokes the whole grant if a rotated token is dropped or spent twice. So a
- refresh MUST (a) persist the newly rotated refresh token — ``refresh_access_token`` does — and
- (b) be serialized against the parallel schema syncs that share one integration. ``select_for_update``
- gives us that Postgres row lock, and we reload inside it so a sync that lost the race picks up the
- token another already rotated instead of re-spending the old one.
+ refresh MUST (a) persist the newly rotated refresh token and (b) be serialized against the parallel
+ schema syncs that share one integration. The SDK's ``refresh_access_token`` does both: it holds a
+ row lock and reads the row inside it, so a sync that lost the race picks up the token another
+ already rotated instead of re-spending the old one.
``current_token`` is the token the caller just refreshed from; when it still matches the row
(nobody else rotated past it) we refresh, otherwise we hand back the fresher token already on the row.
"""
- with transaction.atomic():
- integration = Integration.objects.select_for_update().get(id=integration_id, team_id=team_id, kind="resend")
- oauth = OauthIntegration(integration)
- if oauth.access_token_expired() or (current_token is not None and integration.access_token == current_token):
- oauth.refresh_access_token()
- token = integration.access_token
+ token = IntegrationManager.resend.oauth.refresh_access_token(
+ team_id, integration_id, stale_access_token=current_token
+ ).access_token
if not token:
raise ValueError("Resend access token not found")
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/salesforce/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/salesforce/source.py
index d4a2202c69ab..c21a0d64d55a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/salesforce/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/salesforce/source.py
@@ -264,7 +264,7 @@ class SalesforceSource(ResumableSource[SalesforceSourceConfig, SalesforceResumeC
raise ValueError(f"Salesforce refresh token not found for job {inputs.job_id}")
salesforce_access_token = integration.access_token
- salesforce_instance_url = integration.config.get("instance_url")
+ salesforce_instance_url = cast(str, integration.config.get("instance_url"))
if not salesforce_access_token:
salesforce_access_token = salesforce_refresh_access_token(salesforce_refresh_token, salesforce_instance_url)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/slack/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/slack/source.py
index 96b767ee9493..4a8eac010ee4 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/slack/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/slack/source.py
@@ -4,7 +4,7 @@ from typing import TYPE_CHECKING, Optional, cast
from products.warehouse_sources.backend.facade.codebase_detection import detect
if TYPE_CHECKING:
- from posthog.models.integration import Integration
+ from posthog.integrations import IntegrationRecord
from products.cdp.backend.templates.hog_function_template import HogFunctionTemplateDC
@@ -162,7 +162,7 @@ class SlackSource(ResumableSource[SlackSourceConfig, SlackResumeConfig], Webhook
return WebhookSourceManager(inputs, inputs.logger)
@staticmethod
- def _get_authed_user_id(integration: "Integration") -> str | None:
+ def _get_authed_user_id(integration: "IntegrationRecord") -> str | None:
return (integration.config or {}).get("authed_user", {}).get("id")
def _resolve_access_token(self, config: SlackSourceConfig, team_id: int) -> SlackAccess:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/snapchat_ads/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/snapchat_ads/source.py
index c92cf0f753a2..ba7b7282aacb 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/snapchat_ads/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/snapchat_ads/source.py
@@ -244,7 +244,7 @@ class SnapchatAdsSource(ResumableSource[SnapchatAdsSourceConfig, SnapchatResumeC
# The refresh sweep can fall behind, and an expired token gets a 401 on every request. A
# failed refresh is recorded on the integration and the run continues with the old token.
oauth = OauthIntegration(integration)
- if not refresh_backoff_active(integration) and oauth.access_token_expired():
+ if not refresh_backoff_active(oauth.integration) and oauth.access_token_expired():
oauth.refresh_access_token()
if integration.config.get("refresh_terminal"):
raise ValueError(f"{SNAPCHAT_REAUTHORIZATION_REQUIRED_ERROR} for job {inputs.job_id}")
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/twitter_ads/twitter_ads.py b/products/warehouse_sources/backend/temporal/data_imports/sources/twitter_ads/twitter_ads.py
index c6b9596ffb6d..e5f9bd76fefe 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/twitter_ads/twitter_ads.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/twitter_ads/twitter_ads.py
@@ -8,7 +8,7 @@ import requests
from requests_oauthlib import OAuth1
from posthog.dataclasses import frozen
-from posthog.models.integration.model import Integration
+from posthog.integrations import IntegrationRecord
from products.warehouse_sources.backend.temporal.data_imports.sources.common import credentials_service
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
@@ -85,7 +85,7 @@ def _error_detail(response: requests.Response) -> str:
class TwitterAdsClient:
- def __init__(self, integration: Integration, api_version: str = API_VERSION) -> None:
+ def __init__(self, integration: IntegrationRecord, api_version: str = API_VERSION) -> None:
if api_version != API_VERSION:
raise ValueError("Unsupported X Ads API version")
if integration.kind != "twitter-ads":
diff --git a/products/warehouse_sources/backend/temporal/data_imports/tests/e2e/test_google_ads_source.py b/products/warehouse_sources/backend/temporal/data_imports/tests/e2e/test_google_ads_source.py
index 285678eb86d5..9df2e74a8046 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/tests/e2e/test_google_ads_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/tests/e2e/test_google_ads_source.py
@@ -527,35 +527,38 @@ class TestGoogleAdsClientStaleConnection:
When an import streams rows for many minutes, the Django connection that the
Temporal worker thread holds can be dropped server-side. The next call into
- `Integration.objects.get(...)` from `get_rows` then raises
+ `IntegrationManager.google_ads.oauth.get(...)` from `get_rows` then raises
`OperationalError: server closed the connection unexpectedly`. We close stale
- connections before the ORM query so the next lookup grabs a fresh one.
+ connections before the integration read so the next lookup grabs a fresh one.
"""
@mock.patch(
"products.warehouse_sources.backend.temporal.data_imports.sources.google_ads.google_ads.GoogleAdsClient"
)
- @mock.patch("products.warehouse_sources.backend.temporal.data_imports.sources.google_ads.google_ads.Integration")
+ @mock.patch(
+ "products.warehouse_sources.backend.temporal.data_imports.sources.google_ads.google_ads.IntegrationManager"
+ )
@mock.patch(
"products.warehouse_sources.backend.temporal.data_imports.sources.google_ads.google_ads.close_old_connections"
)
def test_oauth_client_closes_stale_connections_before_integration_lookup(
- self, mock_close_old_connections, mock_integration_model, mock_client_cls
+ self, mock_close_old_connections, mock_integration_manager, mock_client_cls
):
+ integration_get = mock_integration_manager.google_ads.oauth.get
manager = mock.MagicMock()
manager.attach_mock(mock_close_old_connections, "close_old_connections")
- manager.attach_mock(mock_integration_model.objects.get, "integration_get")
+ manager.attach_mock(integration_get, "integration_get")
mock_integration = mock.MagicMock()
mock_integration.refresh_token = "fake-refresh-token"
- mock_integration_model.objects.get.return_value = mock_integration
+ integration_get.return_value = mock_integration
config = GoogleAdsSourceConfig(customer_id="123-456-7890", google_ads_integration_id=42)
google_ads_client(config, team_id=7)
mock_close_old_connections.assert_called_once_with()
- mock_integration_model.objects.get.assert_called_once_with(id=42, team_id=7)
+ integration_get.assert_called_once_with(7, 42)
# Order matters: closing stale connections AFTER the failing query would
# not prevent the OperationalError we are guarding against.
call_order = [name for name, _, _ in manager.mock_calls]
diff --git a/products/warehouse_sources/backend/tests/api/test_external_data_source.py b/products/warehouse_sources/backend/tests/api/test_external_data_source.py
index 0e0ab2ebfe9b..729f57525e1c 100644
--- a/products/warehouse_sources/backend/tests/api/test_external_data_source.py
+++ b/products/warehouse_sources/backend/tests/api/test_external_data_source.py
@@ -23,6 +23,7 @@ from rest_framework import status
from rest_framework.exceptions import PermissionDenied, ValidationError
from sshtunnel import BaseSSHTunnelForwarderError
+from posthog.integrations import IntegrationNotFound
from posthog.models import OrganizationMembership, Team
from posthog.models.integration import ERROR_TOKEN_REFRESH_FAILED, Integration, OauthIntegration
from posthog.models.personal_api_key import PersonalAPIKey, hash_key_value
@@ -14568,13 +14569,13 @@ class TestOAuthAccountsEndpoint(APIBaseTest):
assert "reconnect the integration" in str(response.json()).lower()
def test_gsc_missing_integration_returns_400(self):
- # google_search_console_session raises Integration.DoesNotExist for a missing/foreign id — it
+ # google_search_console_session raises IntegrationNotFound for a missing/foreign id — it
# must surface as an actionable 400, not a 500. (Mocked rather than hitting the real session,
# whose lazy credential load calls close_old_connections and would drop the test connection.)
integration = self._gsc_integration()
with patch(
f"{self._GSC_MODULE}.google_search_console_session",
- side_effect=Integration.DoesNotExist(),
+ side_effect=IntegrationNotFound(),
):
response = self.client.get(self._url("GoogleSearchConsole", integration.id))