Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(warehouse-sources): verify tls certificates for database sources (#116576)

Daniel RC59e7c8701b20
diff --git a/posthog/hogql/direct_sql/postgres_adapter.py b/posthog/hogql/direct_sql/postgres_adapter.py
index 45152083db52..4fad8c571e2b 100644
--- a/posthog/hogql/direct_sql/postgres_adapter.py
+++ b/posthog/hogql/direct_sql/postgres_adapter.py
@@ -171,8 +171,10 @@ class PostgresAdapter:
HostNotAllowedError,
TemporaryHostResolutionError,
_get_sslmode,
+ libpq_tls_kwargs,
pinned_host_kwargs,
source_requires_ssl,
+ tls_options,
)
postgres_source, source_config = self.validate_source_config(source, request.team, request.timings)
@@ -239,6 +241,9 @@ class PostgresAdapter:
with request.timings.measure("postgres_connect", emit_span=True):
try:
+ # The files must outlast the connect, because a cancel request reads them again.
+ tls_kwargs = tunnel_stack.enter_context(libpq_tls_kwargs(tls_options(source_config)))
+ connection_kwargs.update(cast(PostgresConnectionKwargs, tls_kwargs))
connection_context = psycopg.connect(**connection_kwargs)
except (psycopg.Error, RuntimeError, ValueError) as error:
if source.is_managed_warehouse_ready:
diff --git a/posthog/hogql/test/test_direct_postgres_query.py b/posthog/hogql/test/test_direct_postgres_query.py
index 211cde78197a..da27cdf2ea56 100644
--- a/posthog/hogql/test/test_direct_postgres_query.py
+++ b/posthog/hogql/test/test_direct_postgres_query.py
@@ -1920,6 +1920,12 @@ class TestDirectPostgresQuery(APIBaseTest):
("tunnel_tls_on", _SSH_TUNNEL_CONFIG, "require"),
("tunnel_tls_off", _SSH_TUNNEL_TLS_OFF, "prefer"),
("disabled_tunnel_tls_off_ignored", _SSH_TUNNEL_DISABLED_TLS_OFF, "require"),
+ ("verify_certificate", {"verify_server_certificate": "true"}, "verify-full"),
+ (
+ "verify_certificate_through_tunnel",
+ {**_SSH_TUNNEL_CONFIG, "verify_server_certificate": "true"},
+ "verify-ca",
+ ),
]
)
@override_settings(DEBUG=False, TEST=False)
diff --git a/products/warehouse_sources/backend/facade/source_management.py b/products/warehouse_sources/backend/facade/source_management.py
index ed822b52ee95..f3ef8ef4d5e4 100644
--- a/products/warehouse_sources/backend/facade/source_management.py
+++ b/products/warehouse_sources/backend/facade/source_management.py
@@ -55,6 +55,8 @@ _LAZY = {
"OAuthMixin": "sources.common.mixins",
"pinned_host_kwargs": "sources.common.mixins",
"TemporaryHostResolutionError": "sources.common.mixins",
+ "libpq_tls_kwargs": "sources.common.tls",
+ "tls_options": "sources.common.tls",
"SourceSchema": "sources.common.schema",
"discover_trino_catalog": "sources.trino.trino",
"build_default_schemas": "sources.common.schema",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/errors.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/errors.py
index c2f273438bfc..291222b0d500 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/errors.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/errors.py
@@ -25,6 +25,7 @@ MAX_FRIENDLY_MESSAGE_LENGTH = 500
class CDCErrorCategory(enum.StrEnum):
AUTH_FAILED = "auth_failed"
SSL_REQUIRED = "ssl_required"
+ CERTIFICATE_REJECTED = "certificate_rejected"
CONNECTION_FAILED = "connection_failed"
HOST_UNREACHABLE = "host_unreachable"
QUOTA_EXCEEDED = "quota_exceeded"
@@ -97,6 +98,12 @@ _CATEGORY_DEFAULTS: dict[CDCErrorCategory, tuple[str, bool]] = {
"that the database accepts SSL connections, then re-enable change data capture.",
False,
),
+ CDCErrorCategory.CERTIFICATE_REJECTED: (
+ "The TLS certificate check failed. Either the source database's certificate did not pass "
+ "verification, or the database rejected the client certificate. Check the TLS settings of the "
+ "source, then re-enable change data capture.",
+ False,
+ ),
CDCErrorCategory.CONNECTION_FAILED: (
"Could not connect to the source database. PostHog will keep retrying — if this persists, "
"check that the database is reachable and accepting connections.",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_errors.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_errors.py
index 5f22e67019dd..2c9175f2ce63 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_errors.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_errors.py
@@ -31,6 +31,7 @@ class TestCDCErrorInfo:
[
(CDCErrorCategory.AUTH_FAILED, False),
(CDCErrorCategory.SSL_REQUIRED, False),
+ (CDCErrorCategory.CERTIFICATE_REJECTED, False),
(CDCErrorCategory.CONNECTION_FAILED, True),
(CDCErrorCategory.HOST_UNREACHABLE, False),
(CDCErrorCategory.QUOTA_EXCEEDED, False),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/test/test_tls.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/test/test_tls.py
new file mode 100644
index 000000000000..c7e73cbecc79
--- /dev/null
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/test/test_tls.py
@@ -0,0 +1,81 @@
+import os
+import stat
+from pathlib import Path
+from types import SimpleNamespace
+
+from django.test import SimpleTestCase
+
+from parameterized import parameterized
+
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.tls import (
+ TLSConfigurationError,
+ libpq_tls_kwargs,
+ system_ca_bundle,
+ tls_options,
+)
+
+_TUNNEL = SimpleNamespace(enabled=True)
+
+
+class TestLibpqTLSKwargs(SimpleTestCase):
+ @parameterized.expand(
+ [
+ ("no_tls_fields", {}, None, None, None),
+ ("blank_fields", {"verify_server_certificate": False, "ca_certificate": " "}, None, None, None),
+ ("verify", {"verify_server_certificate": True}, "verify-full", "system", None),
+ (
+ "verify_through_tunnel",
+ {"verify_server_certificate": True, "ssh_tunnel": _TUNNEL},
+ "verify-ca",
+ "system",
+ None,
+ ),
+ ("ca_alone_turns_verification_on", {"ca_certificate": "CA PEM"}, "verify-full", "CA PEM", None),
+ (
+ "client_certificate_alone",
+ {"client_certificate": "CERT PEM", "client_key": "KEY PEM"},
+ "require",
+ None,
+ ("CERT PEM", "KEY PEM"),
+ ),
+ ("cleared_certificate_ignores_the_stored_key", {"client_key": "KEY PEM"}, None, None, None),
+ ]
+ )
+ def test_overrides(
+ self,
+ _name: str,
+ fields: dict[str, object],
+ expected_sslmode: str | None,
+ expected_root: str | None,
+ expected_client: tuple[str, str] | None,
+ ) -> None:
+ with libpq_tls_kwargs(tls_options(SimpleNamespace(**fields))) as kwargs:
+ written = [Path(value) for key, value in kwargs.items() if key != "sslmode" and value != system_ca_bundle()]
+ root = kwargs.get("sslrootcert")
+ actual_root = "system" if root == system_ca_bundle() else root and Path(root).read_text()
+ actual_client = (
+ (Path(kwargs["sslcert"]).read_text(), Path(kwargs["sslkey"]).read_text())
+ if "sslcert" in kwargs
+ else None
+ )
+ assert (kwargs.get("sslmode"), actual_root, actual_client) == (
+ expected_sslmode,
+ expected_root,
+ expected_client,
+ )
+ assert all(stat.S_IMODE(os.stat(path).st_mode) == 0o600 for path in written if path.name == "client.key")
+
+ assert not any(path.exists() for path in written)
+
+ @parameterized.expand(
+ [
+ ("certificate_without_key", {"client_certificate": "CERT PEM"}),
+ (
+ "encrypted_key",
+ {"client_certificate": "CERT PEM", "client_key": "-----BEGIN ENCRYPTED PRIVATE KEY-----"},
+ ),
+ ]
+ )
+ def test_unusable_client_certificate_is_refused(self, _name: str, fields: dict[str, str]) -> None:
+ with self.assertRaises(TLSConfigurationError):
+ tls_options(SimpleNamespace(**fields))
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/tls.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/tls.py
new file mode 100644
index 000000000000..4655a73e905b
--- /dev/null
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/tls.py
@@ -0,0 +1,237 @@
+import ssl
+import tempfile
+from collections.abc import Iterator
+from contextlib import contextmanager
+from dataclasses import field
+from functools import cache
+from pathlib import Path
+
+from posthog.dataclasses import frozen
+
+from products.warehouse_sources.backend.facade.source_config import (
+ SourceFieldInputConfig,
+ SourceFieldInputConfigType,
+ SourceFieldSelectConfig,
+ SourceFieldSelectConfigConverter,
+ SourceFieldSelectConfigOption,
+)
+
+CLIENT_CERTIFICATE_INCOMPLETE_ERROR = (
+ "A client certificate needs its private key. Add the key, or clear the certificate."
+)
+
+ENCRYPTED_CLIENT_KEY_ERROR = "The client key has a passphrase. Remove the passphrase and paste the key again."
+
+# The user has to change the source to clear either one, so a sync stops on them.
+TLS_CONFIGURATION_ERRORS: dict[str, str | None] = {
+ CLIENT_CERTIFICATE_INCOMPLETE_ERROR: None,
+ ENCRYPTED_CLIENT_KEY_ERROR: None,
+}
+
+_CLIENT_CERTIFICATE_REJECTED_ERROR = (
+ "Your database rejected the client certificate, or requires one. Add a client certificate and key that "
+ "your database trusts."
+)
+
+# Keyed by the stable part of the client library's message, in lower case. Each failure stays the same
+# on every attempt until the user changes the source.
+CERTIFICATE_ERRORS: dict[str, str] = {
+ "certificate verify failed": (
+ "PostHog could not verify your database's TLS certificate. Either an authority PostHog trusts did not "
+ "sign it, or it does not match the host. Add the authority that signed it, check the host, or turn "
+ "off certificate verification and remove the certificate authority."
+ ),
+ "does not match host name": (
+ "Your database's TLS certificate does not match the host you entered. Use the hostname the "
+ "certificate was issued for, or turn off certificate verification and remove the certificate authority."
+ ),
+ "alert certificate required": _CLIENT_CERTIFICATE_REJECTED_ERROR,
+ "alert bad certificate": _CLIENT_CERTIFICATE_REJECTED_ERROR,
+ "alert unknown ca": _CLIENT_CERTIFICATE_REJECTED_ERROR,
+ # Postgres completes the TLS handshake and then rejects the certificate at its pg_hba check.
+ "certificate authentication failed": _CLIENT_CERTIFICATE_REJECTED_ERROR,
+ "connection requires a valid client certificate": _CLIENT_CERTIFICATE_REJECTED_ERROR,
+}
+
+
+class TLSConfigurationError(Exception):
+ """The TLS fields of a source contradict each other."""
+
+
+@frozen
+class TLSOptions:
+ """The TLS settings a database source reads from its config.
+
+ `verify_hostname` is False through an SSH tunnel. The tunnel presents the database on a
+ loopback address, so the certificate's hostname cannot match the address the client dials.
+ """
+
+ verify: bool = False
+ verify_hostname: bool = False
+ ca_certificate: str | None = None
+ client_certificate: str | None = None
+ client_key: str | None = field(default=None, repr=False)
+
+ def __post_init__(self) -> None:
+ if self.client_certificate and not self.client_key:
+ raise TLSConfigurationError(CLIENT_CERTIFICATE_INCOMPLETE_ERROR)
+ # OpenSSL prompts on the terminal for the passphrase of an encrypted key, which can block
+ # a worker. Both PEM forms of an encrypted key carry this word.
+ if self.client_key and "ENCRYPTED" in self.client_key:
+ raise TLSConfigurationError(ENCRYPTED_CLIENT_KEY_ERROR)
+
+ @property
+ def is_set(self) -> bool:
+ return self.verify or bool(self.client_certificate)
+
+
+@frozen
+class TLSFiles:
+ ca_certificate: str | None
+ client_certificate: str | None
+ client_key: str | None
+
+
+def tls_options(config: object) -> TLSOptions:
+ """Read the TLS fields from a source config. A config without them gets the no-op options."""
+ ca_certificate = _pem(config, "ca_certificate")
+ client_certificate = _pem(config, "client_certificate")
+ # A certificate authority has no use except to verify against, so setting one turns verification on.
+ verify = bool(getattr(config, "verify_server_certificate", False)) or ca_certificate is not None
+ ssh_tunnel = getattr(config, "ssh_tunnel", None)
+ return TLSOptions(
+ verify=verify,
+ verify_hostname=verify and not (ssh_tunnel is not None and ssh_tunnel.enabled),
+ ca_certificate=ca_certificate,
+ client_certificate=client_certificate,
+ # The form cannot blank a stored secret, so clearing the certificate is how a user turns the
+ # client certificate off. The stored key stays, and nothing uses it without a certificate.
+ client_key=_pem(config, "client_key") if client_certificate else None,
+ )
+
+
+# Debian and Ubuntu first, the production images. macOS and Alpine second.
+_CA_BUNDLE_PATHS = ("/etc/ssl/certs/ca-certificates.crt", "/etc/ssl/cert.pem")
+
+
+@cache
+def system_ca_bundle() -> str:
+ """Return the first CA bundle that exists on this host."""
+ defaults = ssl.get_default_verify_paths()
+ for path in (*_CA_BUNDLE_PATHS, defaults.cafile, defaults.openssl_cafile):
+ if path and Path(path).is_file():
+ return path
+ return _CA_BUNDLE_PATHS[0]
+
+
+@contextmanager
+def tls_files(options: TLSOptions) -> Iterator[TLSFiles]:
+ """Write the PEM values to files for the duration of the context.
+
+ The database clients take certificate paths, not contents. The directory is private to this
+ process user, which libpq requires for the private key.
+ """
+ if not (options.ca_certificate or options.client_certificate):
+ yield TLSFiles(ca_certificate=None, client_certificate=None, client_key=None)
+ return
+ with tempfile.TemporaryDirectory(prefix="source-tls-") as directory:
+
+ def write(name: str, pem: str | None) -> str | None:
+ if not pem:
+ return None
+ path = Path(directory) / name
+ path.touch(mode=0o600)
+ path.write_text(pem)
+ return str(path)
+
+ yield TLSFiles(
+ ca_certificate=write("ca.crt", options.ca_certificate),
+ client_certificate=write("client.crt", options.client_certificate),
+ client_key=write("client.key", options.client_key),
+ )
+
+
+@contextmanager
+def libpq_tls_kwargs(options: TLSOptions | None) -> Iterator[dict[str, str]]:
+ """Yield the libpq connection parameters that `options` overrides.
+
+ Yields nothing when no TLS option is set, so the caller's own `sslmode` stays in effect.
+ The files exist only inside the context, and libpq reads them when it connects.
+ """
+ if options is None or not options.is_set:
+ yield {}
+ return
+ with tls_files(options) as files:
+ kwargs: dict[str, str] = {}
+ if options.verify:
+ kwargs["sslmode"] = "verify-full" if options.verify_hostname else "verify-ca"
+ kwargs["sslrootcert"] = files.ca_certificate or system_ca_bundle()
+ if files.client_certificate and files.client_key:
+ # With `prefer`, libpq falls back to plaintext and never presents the certificate.
+ kwargs.setdefault("sslmode", "require")
+ kwargs["sslcert"] = files.client_certificate
+ kwargs["sslkey"] = files.client_key
+ yield kwargs
+
+
+def _pem(config: object, name: str) -> str | None:
+ value = getattr(config, name, None)
+ return value.strip() or None if isinstance(value, str) else None
+
+
+def verify_server_certificate_field() -> SourceFieldSelectConfig:
+ return SourceFieldSelectConfig(
+ name="verify_server_certificate",
+ label="Verify the server certificate?",
+ required=True,
+ defaultValue="false",
+ converter=SourceFieldSelectConfigConverter.STR_TO_BOOL,
+ caption=(
+ "Check that your database's TLS certificate comes from a trusted authority and matches the host. "
+ "Through an SSH tunnel we check the certificate chain but not the hostname, because the tunnel "
+ "presents your database on a local address."
+ ),
+ options=[
+ SourceFieldSelectConfigOption(label="Yes", value="true"),
+ SourceFieldSelectConfigOption(label="No", value="false"),
+ ],
+ )
+
+
+def ca_certificate_field() -> SourceFieldInputConfig:
+ return SourceFieldInputConfig(
+ name="ca_certificate",
+ label="Certificate authority (optional)",
+ type=SourceFieldInputConfigType.TEXTAREA,
+ required=False,
+ placeholder="-----BEGIN CERTIFICATE-----",
+ caption=(
+ "The PEM certificate of the authority that signed your database's certificate. Add it when your "
+ "database uses a private authority or a self-signed certificate. Adding one turns on certificate "
+ "verification."
+ ),
+ secret=False,
+ )
+
+
+def client_certificate_fields() -> list[SourceFieldInputConfig]:
+ return [
+ SourceFieldInputConfig(
+ name="client_certificate",
+ label="Client certificate (optional)",
+ type=SourceFieldInputConfigType.TEXTAREA,
+ required=False,
+ placeholder="-----BEGIN CERTIFICATE-----",
+ caption="A PEM certificate that PostHog presents to your database. Add it when your database requires one.",
+ secret=False,
+ ),
+ SourceFieldInputConfig(
+ name="client_key",
+ label="Client key (optional)",
+ type=SourceFieldInputConfigType.TEXTAREA,
+ required=False,
+ placeholder="-----BEGIN PRIVATE KEY-----",
+ caption="The private key of the client certificate, in PEM format and without a passphrase.",
+ secret=True,
+ ),
+ ]
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/mssql.py b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/mssql.py
index 301be94f8497..a0b3141ef2e1 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/mssql.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/mssql.py
@@ -14,4 +14,5 @@ class MSSQLSourceConfig(config.Config):
port: int = config.value(converter=int)
connection_string: str | None = None
schema: str | None = None
+ require_encryption: bool = config.value(default=config.str_to_bool("false"), converter=config.str_to_bool)
ssh_tunnel: SSHTunnelConfig | None = None
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/mysql.py b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/mysql.py
index 76406ed725e0..9d97db733baa 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/mysql.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/mysql.py
@@ -15,4 +15,7 @@ class MySQLSourceConfig(config.Config):
schema: str | None = None
using_ssl: bool = config.value(default=config.str_to_bool("true"), converter=config.str_to_bool)
verify_server_certificate: bool = config.value(default=config.str_to_bool("false"), converter=config.str_to_bool)
+ ca_certificate: str | None = None
+ client_certificate: str | None = None
+ client_key: str | None = None
ssh_tunnel: SSHTunnelConfig | None = None
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/neon.py b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/neon.py
index bbde6290afa1..f8a099c08b16 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/neon.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/neon.py
@@ -14,4 +14,8 @@ class NeonSourceConfig(config.Config):
port: int = config.value(converter=int)
connection_string: str | None = None
schema: str | None = None
+ verify_server_certificate: bool = config.value(default=config.str_to_bool("false"), converter=config.str_to_bool)
+ ca_certificate: str | None = None
+ client_certificate: str | None = None
+ client_key: str | None = None
ssh_tunnel: SSHTunnelConfig | None = None
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/planetscalepostgres.py b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/planetscalepostgres.py
index bee481a9bb3e..d04aa946055a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/planetscalepostgres.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/planetscalepostgres.py
@@ -13,3 +13,7 @@ class PlanetScalePostgresSourceConfig(config.Config):
port: int = config.value(converter=int)
connection_string: str | None = None
schema: str | None = None
+ verify_server_certificate: bool = config.value(default=config.str_to_bool("false"), converter=config.str_to_bool)
+ ca_certificate: str | None = None
+ client_certificate: str | None = None
+ client_key: str | None = None
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/postgres.py b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/postgres.py
index 96ab3e4aa230..9b0b94704d51 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/postgres.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/postgres.py
@@ -14,4 +14,8 @@ class PostgresSourceConfig(config.Config):
port: int = config.value(converter=int)
connection_string: str | None = None
schema: str | None = None
+ verify_server_certificate: bool = config.value(default=config.str_to_bool("false"), converter=config.str_to_bool)
+ ca_certificate: str | None = None
+ client_certificate: str | None = None
+ client_key: str | None = None
ssh_tunnel: SSHTunnelConfig | None = None
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/redshift.py b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/redshift.py
index 364dd41de841..7bc7e8c68e56 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/redshift.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/redshift.py
@@ -14,4 +14,6 @@ class RedshiftSourceConfig(config.Config):
port: int = config.value(converter=int)
connection_string: str | None = None
schema: str | None = None
+ verify_server_certificate: bool = config.value(default=config.str_to_bool("false"), converter=config.str_to_bool)
+ ca_certificate: str | None = None
ssh_tunnel: SSHTunnelConfig | None = None
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/supabase.py b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/supabase.py
index b470ac8d2ace..0ae5f66964db 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/supabase.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/supabase.py
@@ -14,4 +14,8 @@ class SupabaseSourceConfig(config.Config):
port: int = config.value(converter=int)
connection_string: str | None = None
schema: str | None = None
+ verify_server_certificate: bool = config.value(default=config.str_to_bool("false"), converter=config.str_to_bool)
+ ca_certificate: str | None = None
+ client_certificate: str | None = None
+ client_key: str | None = None
ssh_tunnel: SSHTunnelConfig | None = None
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mssql/mssql.py b/products/warehouse_sources/backend/temporal/data_imports/sources/mssql/mssql.py
index e912006b8de3..833dc785e1e7 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mssql/mssql.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/mssql/mssql.py
@@ -692,6 +692,8 @@ class MSSQLImplementation(SQLSourceImplementation[MSSQLSourceConfig, pymssql.Con
user=config.user,
password=config.password,
login_timeout=MSSQL_LOGIN_TIMEOUT_SECONDS,
+ # None keeps the FreeTDS default, which encrypts only when the server offers it.
+ encryption="require" if config.require_encryption else None,
) as conn:
with conn.cursor() as cursor:
cursor.execute(f"SET LOCK_TIMEOUT {int(lock_timeout_ms)}")
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mssql/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/mssql/source.py
index f4feb3a2210a..497e4e07443b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mssql/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/mssql/source.py
@@ -10,6 +10,9 @@ from products.warehouse_sources.backend.facade.source_config import (
SourceConfig,
SourceFieldInputConfig,
SourceFieldInputConfigType,
+ SourceFieldSelectConfig,
+ SourceFieldSelectConfigConverter,
+ SourceFieldSelectConfigOption,
SourceFieldSSHTunnelConfig,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import FieldType, ResumableSource
@@ -356,6 +359,21 @@ class MSSQLSource(
placeholder="Leave blank to import all schemas",
secret=False,
),
+ SourceFieldSelectConfig(
+ name="require_encryption",
+ label="Require encryption?",
+ required=True,
+ defaultValue="false",
+ converter=SourceFieldSelectConfigConverter.STR_TO_BOOL,
+ caption=(
+ "Refuse to connect unless the connection to your database is encrypted. "
+ "PostHog does not verify the SQL Server certificate."
+ ),
+ options=[
+ SourceFieldSelectConfigOption(label="Yes", value="true"),
+ SourceFieldSelectConfigOption(label="No", value="false"),
+ ],
+ ),
SourceFieldSSHTunnelConfig(name="ssh_tunnel", label="Use SSH tunnel?"),
],
),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mssql/tests/test_mssql.py b/products/warehouse_sources/backend/temporal/data_imports/sources/mssql/tests/test_mssql.py
index 66b05e3eb94f..1ae91fc7c543 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mssql/tests/test_mssql.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/mssql/tests/test_mssql.py
@@ -532,6 +532,16 @@ class TestConnectTimeouts:
assert "timeout" not in mock_connect.call_args.kwargs
assert [call.args[0] for call in cursor.execute.call_args_list] == expected_statements
+ @pytest.mark.parametrize("overrides,expected", [({}, None), ({"require_encryption": "true"}, "require")])
+ def test_connect_requires_encryption_only_when_asked(self, mocker, overrides, expected):
+ mocker.patch(f"{_MSSQL_MODULE}.open_ssh_tunnel").return_value.__enter__.return_value = ("localhost", 1433)
+ mock_connect = mocker.patch(f"{_MSSQL_MODULE}.pymssql.connect")
+
+ with MSSQLImplementation().connect(_make_config(**overrides)):
+ pass
+
+ assert mock_connect.call_args.kwargs["encryption"] == expected
+
class _RaisingTunnel:
"""Context manager whose `__enter__` raises, standing in for paramiko's handshake EOFError."""
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mysql/mysql.py b/products/warehouse_sources/backend/temporal/data_imports/sources/mysql/mysql.py
index 06e989bf6acc..d329766a17a3 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mysql/mysql.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/mysql/mysql.py
@@ -24,8 +24,6 @@ from collections.abc import Callable, Iterator, Sequence
from contextlib import ExitStack, contextmanager
from typing import Any, TypeVar
-from django.conf import settings
-
import psycopg
import pyarrow as pa
import pymysql
@@ -99,6 +97,11 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.sql
normalize_namespace,
resolve_source_location,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.tls import (
+ system_ca_bundle,
+ tls_files,
+ tls_options,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.mysql import MySQLSourceConfig
from products.warehouse_sources.backend.types import IncrementalFieldType, PartitionSettings
@@ -915,8 +918,9 @@ class _TLSRequiredConnection(_SourceConnection):
if self.ssl and not offers_tls:
raise pymysql.err.OperationalError(
CR.CR_SSL_CONNECTION_ERROR,
- "The MySQL server did not offer a TLS connection. Turn off certificate "
- "verification for this source, or enable TLS on the server.",
+ "The MySQL server did not offer a TLS connection. Enable TLS on the server, or turn off "
+ "certificate verification and remove the certificate authority and client certificate "
+ "for this source.",
)
super()._request_authentication()
@@ -935,8 +939,8 @@ def connection_timeouts(read_timeout: int) -> dict[str, int]:
def _new_connection(kwargs: dict[str, Any], **extra: Any) -> pymysql.Connection:
- """Build the connection, refusing plaintext when this source verifies the certificate."""
- if kwargs.get("ssl_verify_cert"):
+ """Build the connection, refusing plaintext when this source verifies or presents a certificate."""
+ if kwargs.get("ssl_verify_cert") or kwargs.get("ssl_cert"):
return _TLSRequiredConnection(**kwargs, **extra)
return _SourceConnection(**kwargs, **extra)
@@ -1329,19 +1333,12 @@ class MySQLImplementation(SQLSourceImplementation[MySQLSourceConfig, pymysql.Con
first rows are ready, and the keyset path sets `autocommit` so each
page is its own transaction (see `_keyset_get_rows`).
"""
- verify_certificate = config.verify_server_certificate
- ssl_ca: str | None = None
- # Verification implies TLS, so a user who asks us to check the certificate gets the
- # encrypted connection that check needs, whatever `using_ssl` says.
- if config.using_ssl or verify_certificate:
- ssl_ca = "/etc/ssl/cert.pem" if settings.DEBUG else "/etc/ssl/certs/ca-certificates.crt"
-
- # The tunnel presents the database on a loopback address, so the certificate's hostname
- # cannot match the address pymysql dials. The chain check still applies there.
- tunnel = config.ssh_tunnel
- verify_hostname = verify_certificate and not (tunnel is not None and tunnel.enabled)
-
- with self._ssh_tunnel_endpoint(config, team_id) as (host, port):
+ tls = tls_options(config)
+ # Any TLS option implies TLS, so a user who sets one gets the encrypted connection it needs,
+ # whatever `using_ssl` says.
+ use_tls = config.using_ssl or tls.is_set
+
+ with self._ssh_tunnel_endpoint(config, team_id) as (host, port), tls_files(tls) as files:
kwargs: dict[str, Any] = {
"host": host,
# pymysql rejects a non-int port; config.port can arrive as a string when the
@@ -1351,11 +1348,13 @@ class MySQLImplementation(SQLSourceImplementation[MySQLSourceConfig, pymysql.Con
"user": config.user,
"password": config.password,
**connection_timeouts(read_timeout),
- "ssl_ca": ssl_ca,
+ "ssl_ca": (files.ca_certificate or system_ca_bundle()) if use_tls else None,
+ "ssl_cert": files.client_certificate,
+ "ssl_key": files.client_key,
# pymysql resolves `ssl_ca` on its own to `verify_mode=CERT_NONE`, so only an
# explicit True verifies. None leaves that prior behavior untouched.
- "ssl_verify_cert": True if verify_certificate else None,
- "ssl_verify_identity": True if verify_hostname else None,
+ "ssl_verify_cert": True if tls.verify else None,
+ "ssl_verify_identity": True if tls.verify_hostname else None,
"conv": _MYSQL_SAFE_CONVERSIONS,
"autocommit": autocommit,
"init_command": "SET workload = 'OLAP';" if host.endswith("psdb.cloud") else None,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mysql/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/mysql/source.py
index 97666bcfa7ef..bcae01a80311 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mysql/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/mysql/source.py
@@ -32,6 +32,13 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res
from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import SourceSchema
from products.warehouse_sources.backend.temporal.data_imports.sources.common.sql.base import SQLSource
from products.warehouse_sources.backend.temporal.data_imports.sources.common.sql.keyset import KeysetResumeState
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.tls import (
+ CERTIFICATE_ERRORS,
+ TLS_CONFIGURATION_ERRORS,
+ TLSConfigurationError,
+ ca_certificate_field,
+ client_certificate_fields,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.mysql import MySQLSourceConfig
from products.warehouse_sources.backend.temporal.data_imports.sources.mysql.mysql import (
@@ -54,6 +61,7 @@ _MYSQL_IMPLEMENTATION = MySQLImplementation()
# get_non_retryable_errors — which the sync path also consults for retry
# classification — so connection-error retry behaviour is unchanged.
_VALIDATE_CONNECTION_HINTS: list[tuple[str, str]] = [
+ *CERTIFICATE_ERRORS.items(),
(
"Name or service not known",
"Host could not be resolved. Check the host is spelled correctly and reachable from PostHog.",
@@ -275,8 +283,8 @@ class MySQLSource(
converter=SourceFieldSelectConfigConverter.STR_TO_BOOL,
caption=(
"Check that your database's TLS certificate comes from a trusted authority. "
- "A self-signed certificate or a private authority does not pass, so leave this off "
- "if you use one. Through an SSH tunnel we check the certificate chain but not the "
+ "A self-signed certificate or a private authority passes only when you add its "
+ "certificate authority below. Through an SSH tunnel we check the certificate chain but not the "
"hostname, because the tunnel presents your database on a local address."
),
options=[
@@ -284,6 +292,8 @@ class MySQLSource(
SourceFieldSelectConfigOption(label="No", value="false"),
],
),
+ ca_certificate_field(),
+ *client_certificate_fields(),
SourceFieldSSHTunnelConfig(name="ssh_tunnel", label="Use SSH tunnel?"),
],
),
@@ -291,6 +301,9 @@ class MySQLSource(
def get_non_retryable_errors(self) -> dict[str, str | None]:
return {
+ # pymysql wraps a TLS failure in the generic 2003 error below, so these must come first.
+ **CERTIFICATE_ERRORS,
+ **TLS_CONFIGURATION_ERRORS,
# pymysql collapses every connect-level failure (wrong host/port, closed firewall,
# unreachable host, a connect timeout that outlasts the in-process retry budget) into
# error 2003, "Can't connect to MySQL server on '<host>' (<os detail>)". Non-retryable
@@ -649,9 +662,10 @@ class MySQLSource(
try:
self.get_schemas(config, team_id, api_version=api_version)
- except (HostNotAllowedError, TemporaryHostResolutionError) as e:
- # The host policy refused the host, or its lookup never answered. Both carry their own
- # user-facing wording and neither is a PostHog defect, so they are not captured.
+ except (HostNotAllowedError, TemporaryHostResolutionError, TLSConfigurationError) as e:
+ # The host policy refused the host, its lookup never answered, or the TLS fields contradict
+ # each other. Each carries its own user-facing wording and none is a PostHog defect, so they
+ # are not captured.
return False, str(e)
except BaseSSHTunnelForwarderError as e:
# sshtunnel surfaces raw library strings (e.g. "Could not establish session to SSH
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mysql/tests/test_mysql.py b/products/warehouse_sources/backend/temporal/data_imports/sources/mysql/tests/test_mysql.py
index c14f6f3b81ba..fa7b724a92fa 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mysql/tests/test_mysql.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/mysql/tests/test_mysql.py
@@ -3,6 +3,7 @@ import datetime
from collections.abc import Generator, Iterator
from contextlib import contextmanager
from decimal import Decimal
+from pathlib import Path
from typing import cast
import pytest
@@ -27,6 +28,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.sql
ValidatedRowFilter,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.tests.resolver import addrinfo
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.tls import CERTIFICATE_ERRORS
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.mysql import MySQLSourceConfig
from products.warehouse_sources.backend.temporal.data_imports.sources.mysql.mysql import (
@@ -3099,6 +3101,15 @@ class TestMySQLSourceValidateCredentials:
),
"Could not reach the host. Check the host is correct and that PostHog's IP addresses are allowed through your firewall.",
),
+ # pymysql wraps the TLS failure in the generic 2003 error, so it must not get the firewall advice.
+ (
+ pymysql.err.OperationalError(
+ 2003,
+ "Can't connect to MySQL server on 'db.example.com' ([SSL: CERTIFICATE_VERIFY_FAILED] "
+ "certificate verify failed: unable to get local issuer certificate (_ssl.c:1032))",
+ ),
+ CERTIFICATE_ERRORS["certificate verify failed"],
+ ),
# Server error 1049: the host/port/credentials are fine but the named database
# doesn't exist. Previously fell through to capture as an unexpected error.
(
@@ -3398,6 +3409,33 @@ class TestConnectCertificateVerification:
assert kwargs["ssl_verify_identity"] is expected_identity
assert refuses_plaintext is (expected_cert is True)
+ def test_custom_authority_and_client_certificate_reach_pymysql(self, mocker):
+ seen: dict[str, str] = {}
+
+ def read_files(**kwargs):
+ seen.update({name: Path(kwargs[name]).read_text() for name in ("ssl_ca", "ssl_cert", "ssl_key")})
+ seen["verify"] = kwargs["ssl_verify_cert"]
+ return MagicMock()
+
+ mocker.patch(f"{_MYSQL_MODULE}._TLSRequiredConnection", side_effect=read_files)
+ config = _make_config(
+ using_ssl="false", ca_certificate="CA PEM", client_certificate="CERT PEM", client_key="KEY PEM"
+ )
+
+ with MySQLImplementation().connect(config):
+ pass
+
+ assert seen == {"ssl_ca": "CA PEM", "ssl_cert": "CERT PEM", "ssl_key": "KEY PEM", "verify": True}
+
+ def test_client_certificate_without_verification_still_refuses_plaintext(self, mocker):
+ plain = mocker.patch(f"{_MYSQL_MODULE}._SourceConnection", return_value=MagicMock())
+ refusing = mocker.patch(f"{_MYSQL_MODULE}._TLSRequiredConnection", return_value=MagicMock())
+
+ with MySQLImplementation().connect(_make_config(client_certificate="CERT PEM", client_key="KEY PEM")):
+ pass
+
+ assert refusing.called and not plain.called
+
@pytest.mark.parametrize("capabilities,refused", [(0, True), (CLIENT.SSL, False)])
def test_authentication_against_a_server_that_advertises_tls_or_not(self, mocker, capabilities, refused):
# The credentials go out during `_request_authentication`, and every reconnect runs it
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/adapter.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/adapter.py
index 5d7b0314c7fb..7ae8a92befe2 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/adapter.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/adapter.py
@@ -13,6 +13,7 @@ from sshtunnel import BaseSSHTunnelForwarderError
from products.warehouse_sources.backend.temporal.data_imports.cdc.errors import cdc_error_info
from products.warehouse_sources.backend.temporal.data_imports.sources.common.mixins import HostNotAllowedError
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.tls import tls_options
from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.cdc.config import PostgresCDCConfig
from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.cdc.errors import (
classify_postgres_cdc_error,
@@ -135,6 +136,7 @@ class PostgresCDCAdapter:
user=inputs.get("user", ""),
password=inputs.get("password", ""),
require_ssl=source_requires_ssl(source, config),
+ tls=tls_options(config),
slot_name=cdc_config.slot_name,
publication_name=cdc_config.publication_name,
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/errors.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/errors.py
index 0e35b61b11eb..0d41ed45fd5f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/errors.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/errors.py
@@ -19,6 +19,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.mix
SSH_TUNNEL_HOST_NOT_ALLOWED_ERROR,
HostNotAllowedError,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.tls import CERTIFICATE_ERRORS
# Server requires (or rejects) an encrypted connection. Kept specific so a transient
# "SSL connection has been closed unexpectedly" stays a retryable connection failure.
@@ -128,6 +129,11 @@ def classify_postgres_cdc_error(exc: BaseException) -> CDCErrorCategory | None:
if "publication" in message:
return CDCErrorCategory.PUBLICATION_MISSING
+ # Before the authentication checks: Postgres reports a rejected client certificate as
+ # "certificate authentication failed", which the auth markers would otherwise claim.
+ if any(marker in message for marker in CERTIFICATE_ERRORS):
+ return CDCErrorCategory.CERTIFICATE_REJECTED
+
# Authentication (SQLSTATE class 28) surfaces either as a mapped subclass (server-side)
# or a bare OperationalError with a libpq message (connection-time). Check before the
# generic OperationalError fallback, which these subclass.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/slot_manager.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/slot_manager.py
index 181771106b93..addbda47b05a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/slot_manager.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/slot_manager.py
@@ -23,6 +23,7 @@ def cdc_pg_connection(source: ExternalDataSource, connect_timeout: int = 15) ->
Handles SSH tunnel and delegates the actual connection
to _connect_to_postgres which owns SSL cert overrides.
"""
+ from products.warehouse_sources.backend.temporal.data_imports.sources.common.tls import tls_options
from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.postgres import (
_connect_to_postgres,
source_requires_ssl,
@@ -45,6 +46,7 @@ def cdc_pg_connection(source: ExternalDataSource, connect_timeout: int = 15) ->
password=config.password,
require_ssl=require_ssl,
connect_timeout=connect_timeout,
+ tls=tls_options(config),
team_id=source.team_id,
)
try:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/stream_reader.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/stream_reader.py
index 059bbe9ac7ae..23e6d8c86584 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/stream_reader.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/stream_reader.py
@@ -19,6 +19,7 @@ import psycopg.errors
from psycopg import sql
from products.warehouse_sources.backend.temporal.data_imports.cdc.types import ChangeEvent
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.tls import TLSOptions
from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.cdc.decoder import PgOutputDecoder
from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.postgres import (
_connect_to_postgres,
@@ -66,6 +67,7 @@ class PgCDCConnectionParams:
user: str
password: str = field(repr=False)
require_ssl: bool = False
+ tls: TLSOptions | None = None
slot_name: str = ""
publication_name: str = ""
@@ -128,6 +130,7 @@ class PgCDCStreamReader:
user=self._params.user,
password=self._params.password,
require_ssl=self._params.require_ssl,
+ tls=self._params.tls,
team_id=self._source.team_id if self._source is not None else None,
# statement_timeout (30m) is a server-side ceiling so a stalled WAL decode can't
# hang the streaming connection indefinitely; it bounds a single peek, the caller's
@@ -293,6 +296,7 @@ class PgCDCStreamReader:
user=self._params.user,
password=self._params.password,
require_ssl=self._params.require_ssl,
+ tls=self._params.tls,
team_id=self._source.team_id if self._source is not None else None,
),
logger,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/tests/test_errors.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/tests/test_errors.py
index ca202fb44854..5b074f276173 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/tests/test_errors.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/tests/test_errors.py
@@ -34,6 +34,21 @@ class TestClassifyPostgresCDCError:
psycopg.OperationalError("server does not support SSL, but SSL was required"),
CDCErrorCategory.SSL_REQUIRED,
),
+ (
+ "server_certificate_failed_verification",
+ psycopg.OperationalError("connection failed: SSL error: certificate verify failed"),
+ CDCErrorCategory.CERTIFICATE_REJECTED,
+ ),
+ (
+ "client_certificate_rejected_by_pg_hba",
+ psycopg.OperationalError('connection failed: FATAL: certificate authentication failed for user "u"'),
+ CDCErrorCategory.CERTIFICATE_REJECTED,
+ ),
+ (
+ "client_certificate_rejected",
+ psycopg.OperationalError("connection failed: SSL error: tlsv13 alert certificate required"),
+ CDCErrorCategory.CERTIFICATE_REJECTED,
+ ),
(
"ssl_close_is_connection_not_ssl_required",
psycopg.OperationalError("SSL connection has been closed unexpectedly"),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/postgres.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/postgres.py
index 098dc86d20da..060c731b311a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/postgres.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/postgres.py
@@ -4,6 +4,7 @@ import re
import math
import time
import errno
+import weakref
import collections
import dataclasses
from collections.abc import Callable, Iterator
@@ -95,6 +96,12 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.sql
and_join,
render_psycopg_row_filter_conditions,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.tls import (
+ CERTIFICATE_ERRORS,
+ TLSOptions,
+ libpq_tls_kwargs,
+ tls_options,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.postgres import (
PostgresSourceConfig,
@@ -950,7 +957,16 @@ def _is_invalid_ssl_negotiation_response(error: BaseException) -> bool:
return _INVALID_SSL_NEGOTIATION_RESPONSE_SUBSTRING in " ".join(str(arg) for arg in error.args).lower()
-def _open_connection(*, team_id: int | None = None, **connect_kwargs: Any) -> psycopg.Connection:
+# libpq mentions SSL when it refuses the server certificate or the server refuses ours. The server
+# does speak TLS there, so the "enable SSL" advice of `SSLRequiredError` would mislead.
+def _is_certificate_rejection(error: BaseException) -> bool:
+ message = " ".join(str(arg) for arg in error.args).lower()
+ return any(marker in message for marker in CERTIFICATE_ERRORS)
+
+
+def _open_connection(
+ *, team_id: int | None = None, tls: TLSOptions | None = None, **connect_kwargs: Any
+) -> psycopg.Connection:
"""The one `psycopg.connect` in this module. Every connection to a source database opens
here so that `pinned_host_kwargs` decides what gets dialed.
@@ -958,6 +974,18 @@ def _open_connection(*, team_id: int | None = None, **connect_kwargs: Any) -> ps
`_OPTIONS_STARTUP_PARAM_UNSUPPORTED_SUBSTRINGS` for why transaction-mode poolers reject
`options` and why dropping it is safe.
"""
+ if tls is not None:
+ tls_files = ExitStack()
+ tls_kwargs = tls_files.enter_context(libpq_tls_kwargs(tls))
+ try:
+ connection = _open_connection(team_id=team_id, **{**connect_kwargs, **tls_kwargs})
+ except BaseException:
+ tls_files.close()
+ raise
+ # libpq 17 reads the certificate files again for every `cancel_safe` request, so they must
+ # last as long as the connection.
+ weakref.finalize(connection, tls_files.close)
+ return connection
connect_kwargs.update(
pinned_host_kwargs(
connect_kwargs["host"],
@@ -988,6 +1016,7 @@ def _connect_to_postgres(
require_ssl: bool = False,
connect_timeout: int = 15,
team_id: int | None = None,
+ tls: TLSOptions | None = None,
**kwargs: Any,
) -> psycopg.Connection:
sslmode = _get_sslmode(require_ssl)
@@ -1001,6 +1030,7 @@ def _connect_to_postgres(
try:
return _open_connection(
team_id=team_id,
+ tls=tls,
host=host,
port=port,
dbname=database,
@@ -1020,6 +1050,7 @@ def _connect_to_postgres(
require_ssl
and "SSL" in str(e)
and not _is_invalid_ssl_negotiation_response(e)
+ and not _is_certificate_rejection(e)
and not _is_connection_dropped_error(e)
):
raise SSLRequiredError(
@@ -1039,6 +1070,7 @@ def pg_connection(
password: str,
require_ssl: bool = False,
team_id: int | None = None,
+ tls: TLSOptions | None = None,
) -> Iterator[psycopg.Connection]:
"""Context manager that opens a postgres connection and ensures it is closed on exit."""
conn = _connect_to_postgres(
@@ -1049,6 +1081,7 @@ def pg_connection(
password=password,
require_ssl=require_ssl,
team_id=team_id,
+ tls=tls,
)
try:
yield conn
@@ -1617,6 +1650,7 @@ def get_postgres_row_count(
schema: str | None,
require_ssl: bool = False,
team_id: int | None = None,
+ tls: TLSOptions | None = None,
names: list[str] | None = None,
) -> dict[str, int]:
if _normalize_selected_schema(schema) is None and not names:
@@ -1630,6 +1664,7 @@ def get_postgres_row_count(
password=password,
require_ssl=require_ssl,
team_id=team_id,
+ tls=tls,
) as connection:
return _row_counts_from_conn(connection, schema, names)
except:
@@ -1748,6 +1783,7 @@ def get_schemas(
port: int,
require_ssl: bool = False,
team_id: int | None = None,
+ tls: TLSOptions | None = None,
names: list[str] | None = None,
) -> dict[str, PostgresDiscoveredSchema]:
"""Get all tables from PostgreSQL source schemas to sync."""
@@ -1778,6 +1814,7 @@ def get_schemas(
password=password,
require_ssl=require_ssl,
team_id=team_id,
+ tls=tls,
)
try:
return _schemas_from_conn(connection, schema, names)
@@ -1802,6 +1839,7 @@ def get_primary_keys_for_schemas(
table_names: list[str],
require_ssl: bool = False,
team_id: int | None = None,
+ tls: TLSOptions | None = None,
) -> dict[str, list[str] | None]:
"""Detect primary keys for all tables in a single query."""
result: dict[str, list[str] | None] = dict.fromkeys(table_names)
@@ -1815,6 +1853,7 @@ def get_primary_keys_for_schemas(
password=password,
require_ssl=require_ssl,
team_id=team_id,
+ tls=tls,
) as connection:
pks = get_primary_key_columns(connection, schema, table_names)
for table_name, pk_cols in pks.items():
@@ -1902,6 +1941,7 @@ def get_foreign_keys(
port: int,
require_ssl: bool = False,
team_id: int | None = None,
+ tls: TLSOptions | None = None,
names: list[str] | None = None,
) -> dict[str, list[tuple[str, str, str]]]:
"""Get foreign keys for tables in the selected PostgreSQL schema."""
@@ -1913,6 +1953,7 @@ def get_foreign_keys(
password=password,
require_ssl=require_ssl,
team_id=team_id,
+ tls=tls,
) as connection:
return _foreign_keys_from_conn(connection, schema, names)
@@ -1937,6 +1978,7 @@ def get_connection_metadata(
port: int,
require_ssl: bool = False,
team_id: int | None = None,
+ tls: TLSOptions | None = None,
) -> dict[str, Any]:
with pg_connection(
host=host,
@@ -1946,6 +1988,7 @@ def get_connection_metadata(
password=password,
require_ssl=require_ssl,
team_id=team_id,
+ tls=tls,
) as connection:
with connection.cursor() as cursor:
cursor.execute("SELECT current_database(), version()")
@@ -3691,6 +3734,7 @@ def postgres_source(
incremental_field: Optional[str] = None,
incremental_field_type: Optional[IncrementalFieldType] = None,
require_ssl: bool = False,
+ tls: TLSOptions | None = None,
is_initial_sync: bool = False,
enabled_columns: Optional[list[str]] = None,
row_filters: Optional[list[ValidatedRowFilter]] = None,
@@ -3722,6 +3766,7 @@ def postgres_source(
try:
conn = _open_connection(
team_id=team_id,
+ tls=tls,
host=host,
port=port,
dbname=database,
@@ -3740,6 +3785,7 @@ def postgres_source(
require_ssl
and "SSL" in str(e)
and not _is_invalid_ssl_negotiation_response(e)
+ and not _is_certificate_rejection(e)
and not _is_connection_dropped_error(e)
):
raise SSLRequiredError(
@@ -4118,6 +4164,7 @@ def postgres_source(
try:
connection = _open_connection(
team_id=team_id,
+ tls=tls,
host=host,
port=port,
dbname=database,
@@ -4137,6 +4184,7 @@ def postgres_source(
require_ssl
and "SSL" in str(e)
and not _is_invalid_ssl_negotiation_response(e)
+ and not _is_certificate_rejection(e)
and not _is_connection_dropped_error(e)
):
raise SSLRequiredError(
@@ -4857,6 +4905,7 @@ class PostgresImplementation(SQLSourceImplementation[PostgresSourceConfig, psyco
password=config.password,
require_ssl=require_ssl,
team_id=team_id,
+ tls=tls_options(config),
) as conn:
yield conn
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py
index 83a5a004c0af..427d1a01f217 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py
@@ -45,6 +45,15 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.sql
from products.warehouse_sources.backend.temporal.data_imports.sources.common.sql.base import SQLSource
from products.warehouse_sources.backend.temporal.data_imports.sources.common.sql.keyset import KeysetResumeState
from products.warehouse_sources.backend.temporal.data_imports.sources.common.sql.location import resolve_source_location
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.tls import (
+ CERTIFICATE_ERRORS,
+ TLS_CONFIGURATION_ERRORS,
+ TLSConfigurationError,
+ ca_certificate_field,
+ client_certificate_fields,
+ tls_options,
+ verify_server_certificate_field,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.postgres import (
PostgresSourceConfig,
@@ -195,6 +204,7 @@ _DATABASE_NOT_FOUND_VALIDATION_ERROR = (
POSTGRES_HOST_CHECK_SOURCE_TYPE = "postgres"
PostgresErrors = {
+ **CERTIFICATE_ERRORS,
"password authentication failed for user": _INVALID_CREDENTIALS_VALIDATION_ERROR,
# A proxy/pooler in front of some providers rejects bad credentials during its own
# database-identification step instead of libpq's "password authentication failed for user",
@@ -729,6 +739,9 @@ class PostgresSource(
placeholder="public",
secret=False,
),
+ verify_server_certificate_field(),
+ ca_certificate_field(),
+ *client_certificate_fields(),
SourceFieldSSHTunnelConfig(name="ssh_tunnel", label="Use SSH tunnel?"),
],
),
@@ -1173,6 +1186,8 @@ class PostgresSource(
"check that the host for this source is the hostname your database provider gave "
"you rather than an IP address, then re-enable the sync."
),
+ **CERTIFICATE_ERRORS,
+ **TLS_CONFIGURATION_ERRORS,
"SSLRequiredError": None,
"SSL/TLS connection is required": None,
_SSH_GATEWAY_SESSION_ERROR: _SSH_GATEWAY_UNREACHABLE_SYNC_MESSAGE,
@@ -1735,6 +1750,7 @@ class PostgresSource(
require_ssl: bool = False,
) -> list[SourceSchema]:
schemas = []
+ tls = tls_options(config)
with self.with_ssh_tunnel(config, team_id, POSTGRES_HOST_CHECK_SOURCE_TYPE) as (host, port):
db_schemas = get_postgres_schemas(
@@ -1747,6 +1763,7 @@ class PostgresSource(
names=names,
require_ssl=require_ssl,
team_id=team_id,
+ tls=tls,
)
# Foreign keys are advisory metadata (they pre-populate relationship hints in the
# table picker). The discovery query joins three `information_schema` views, which
@@ -1764,6 +1781,7 @@ class PostgresSource(
names=names,
require_ssl=require_ssl,
team_id=team_id,
+ tls=tls,
)
except Exception as e:
structlog.get_logger().warning("Failed to detect foreign keys for Postgres schemas", exc_info=e)
@@ -1780,6 +1798,7 @@ class PostgresSource(
names=names,
require_ssl=require_ssl,
team_id=team_id,
+ tls=tls,
)
else:
row_counts = {}
@@ -1814,6 +1833,7 @@ class PostgresSource(
database=config.database,
require_ssl=require_ssl,
team_id=team_id,
+ tls=tls,
) as conn:
# PK lookup powers `supports_cdc`. Wrap in try/except so a permissions
# quirk on `pg_catalog` (rare) only disables CDC advertising for this
@@ -2015,7 +2035,7 @@ class PostgresSource(
if require_ssl:
return False, _SSL_UNSUPPORTED_ERROR
return False, str(e)
- except HostNotAllowedError as e:
+ except (HostNotAllowedError, TLSConfigurationError) as e:
return False, str(e)
except TemporaryHostResolutionError as e:
return False, str(e)
@@ -2071,6 +2091,7 @@ class PostgresSource(
database=config.database,
require_ssl=require_ssl,
team_id=team_id,
+ tls=tls_options(config),
)
def check_cdc_prerequisites(
@@ -2104,6 +2125,7 @@ class PostgresSource(
password=config.password,
require_ssl=require_ssl,
team_id=team_id,
+ tls=tls_options(config),
)
try:
# A blank schema means discovery enumerated every schema and named each table
@@ -2352,6 +2374,7 @@ class PostgresSource(
chunk_size_override=schema.chunk_size_override,
team_id=inputs.team_id,
require_ssl=require_ssl,
+ tls=tls_options(config),
is_initial_sync=not schema.initial_sync_complete,
enabled_columns=inputs.enabled_columns,
row_filters=inputs.row_filters,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py
index 9fff7ac3c29e..9cafd330c234 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py
@@ -5,6 +5,7 @@ import threading
from collections.abc import Generator, Iterable, Iterator
from contextlib import ExitStack, contextmanager
from datetime import UTC, date, datetime, time, timedelta, timezone
+from pathlib import Path
from typing import Any, cast
import pytest
@@ -54,6 +55,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.sql
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.sql.types import Table
from products.warehouse_sources.backend.temporal.data_imports.sources.common.tests.resolver import addrinfo
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.tls import TLSOptions
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.client_deadline import (
CLIENT_DEADLINE_ERROR,
@@ -3587,13 +3589,19 @@ class TestInvalidSSLNegotiationResponse:
def test_detects_invalid_ssl_negotiation_message(self, message, expected):
assert _is_invalid_ssl_negotiation_response(psycopg.OperationalError(message)) is expected
- def test_invalid_negotiation_is_not_wrapped_as_ssl_required(self):
- # require_ssl=True, but the message is a wrong-port/non-Postgres signal — it must surface
- # raw (so get_non_retryable_errors can give an accurate message), not as the misleading
+ @pytest.mark.parametrize(
+ "message",
+ [
+ "received invalid response to SSL negotiation: I",
+ "SSL error: certificate verify failed",
+ "SSL error: tlsv13 alert certificate required",
+ ],
+ )
+ def test_ssl_mentioning_errors_are_not_wrapped_as_ssl_required(self, message):
+ # require_ssl=True, and the message mentions SSL, but the server does speak TLS here. It must
+ # surface raw so get_non_retryable_errors can give an accurate message, not the misleading
# SSLRequiredError "enable SSL on your server".
- connect_mock = mock.MagicMock(
- side_effect=psycopg.OperationalError("received invalid response to SSL negotiation: I")
- )
+ connect_mock = mock.MagicMock(side_effect=psycopg.OperationalError(message))
with patch(
"products.warehouse_sources.backend.temporal.data_imports.sources.postgres.postgres.psycopg.connect",
connect_mock,
@@ -3603,7 +3611,29 @@ class TestInvalidSSLNegotiationResponse:
host="db", port=41667, database="railway", user="postgres", password="x", require_ssl=True
)
assert not isinstance(exc_info.value, SSLRequiredError)
- assert "received invalid response to SSL negotiation" in str(exc_info.value)
+ assert message in str(exc_info.value)
+
+ def test_tls_options_replace_the_unverified_defaults(self):
+ connect_mock = mock.MagicMock()
+ with patch(
+ "products.warehouse_sources.backend.temporal.data_imports.sources.postgres.postgres.psycopg.connect",
+ connect_mock,
+ ):
+ _connect_to_postgres(
+ host="db",
+ port=5432,
+ database="db",
+ user="postgres",
+ password="x",
+ require_ssl=True,
+ tls=TLSOptions(verify=True, verify_hostname=True, ca_certificate="CA PEM"),
+ )
+
+ kwargs = connect_mock.call_args.kwargs
+ assert kwargs["sslmode"] == "verify-full"
+ # A cancel request on the open connection reads the authority file again.
+ assert Path(kwargs["sslrootcert"]).read_text() == "CA PEM"
+ assert "tls" not in kwargs
def test_genuine_unsupported_ssl_still_raises_ssl_required(self):
connect_mock = mock.MagicMock(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/redshift/redshift.py b/products/warehouse_sources/backend/temporal/data_imports/sources/redshift/redshift.py
index 358dc8f6d2fa..f9ab950a1562 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/redshift/redshift.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/redshift/redshift.py
@@ -80,6 +80,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.sql
and_join,
render_psycopg_row_filter_conditions,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.tls import libpq_tls_kwargs, tls_options
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.redshift import (
RedshiftSourceConfig,
@@ -1087,13 +1088,15 @@ class RedshiftImplementation(SQLSourceImplementation[RedshiftSourceConfig, psyco
host, port=port, connect_timeout=_REDSHIFT_CONNECT_OPTS["connect_timeout"], team_id=team_id
),
}
- with psycopg.connect(**connect_kwargs) as conn:
- conn.adapters.register_loader("date", SafeDateLoader)
- conn.adapters.register_loader("timestamp", SafeTimestampLoader)
- conn.adapters.register_loader("timestamptz", SafeTimestamptzLoader)
- if statement_timeout_ms is not None:
- _set_session_statement_timeout(conn, statement_timeout_ms)
- yield conn
+ with libpq_tls_kwargs(tls_options(config)) as tls_kwargs:
+ connection = psycopg.connect(**{**connect_kwargs, **tls_kwargs})
+ with connection as conn:
+ conn.adapters.register_loader("date", SafeDateLoader)
+ conn.adapters.register_loader("timestamp", SafeTimestampLoader)
+ conn.adapters.register_loader("timestamptz", SafeTimestamptzLoader)
+ if statement_timeout_ms is not None:
+ _set_session_statement_timeout(conn, statement_timeout_ms)
+ yield conn
# ------------------------------------------------------------------
# Listing — batch queries run once during `get_schemas`
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/redshift/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/redshift/source.py
index e7ca085f84d7..54b190e110c1 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/redshift/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/redshift/source.py
@@ -28,6 +28,11 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.mix
from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistry
from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import SourceSchema
from products.warehouse_sources.backend.temporal.data_imports.sources.common.sql.base import SQLSource
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.tls import (
+ CERTIFICATE_ERRORS,
+ ca_certificate_field,
+ verify_server_certificate_field,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.redshift import (
RedshiftSourceConfig,
@@ -86,6 +91,7 @@ _CONNECTION_DROPPED_EXHAUSTED_MESSAGE = (
)
RedshiftErrors = {
+ **CERTIFICATE_ERRORS,
"password authentication failed for user": "Invalid user or password",
"could not translate host name": _DNS_RESOLUTION_VALIDATION_ERROR,
"Is the server running on that host and accepting TCP/IP connections": _HOST_UNREACHABLE_VALIDATION_ERROR,
@@ -194,6 +200,8 @@ class RedshiftSource(SQLSource[RedshiftSourceConfig], SSHTunnelMixin, ValidateDa
placeholder="Leave blank to import all schemas",
secret=False,
),
+ verify_server_certificate_field(),
+ ca_certificate_field(),
SourceFieldSSHTunnelConfig(name="ssh_tunnel", label="Use SSH tunnel?"),
],
),
@@ -235,6 +243,7 @@ class RedshiftSource(SQLSource[RedshiftSourceConfig], SSHTunnelMixin, ValidateDa
def get_non_retryable_errors(self) -> dict[str, str | None]:
return {
**self.default_non_retryable_errors(),
+ **CERTIFICATE_ERRORS,
"NoSuchTableError": None,
"is not permitted to log in": None,
"could not translate host name": None,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/redshift/tests/test_redshift.py b/products/warehouse_sources/backend/temporal/data_imports/sources/redshift/tests/test_redshift.py
index 377d12a940f5..dd4b10e0f7c3 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/redshift/tests/test_redshift.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/redshift/tests/test_redshift.py
@@ -1184,6 +1184,20 @@ def build_pipeline_mocks(mocker):
return mock_connect, streaming_cursor
+class TestConnect:
+ @pytest.mark.parametrize(
+ "overrides,expected_sslmode",
+ [({}, "require"), ({"verify_server_certificate": "true"}, "verify-full")],
+ )
+ def test_certificate_verification_is_opt_in(self, mocker, overrides, expected_sslmode):
+ mock_connect = mocker.patch(f"{_REDSHIFT_MODULE}.psycopg.connect")
+
+ with RedshiftImplementation().connect(_make_config(**overrides), statement_timeout_ms=None):
+ pass
+
+ assert mock_connect.call_args.kwargs["sslmode"] == expected_sslmode
+
+
class TestIsTransientConnectionDropError:
def test_does_not_match_non_operational_error(self):
assert _is_transient_connection_drop_error(ValueError("the connection is lost")) is False
diff --git a/products/warehouse_sources/backend/temporal/data_imports/tests/e2e/test_import_data.py b/products/warehouse_sources/backend/temporal/data_imports/tests/e2e/test_import_data.py
index 4e1a26f8a9c5..45ea9fd3fedc 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/tests/e2e/test_import_data.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/tests/e2e/test_import_data.py
@@ -20,6 +20,7 @@ from products.warehouse_sources.backend.facade.models import (
from products.warehouse_sources.backend.facade.types import ExternalDataSourceType
from products.warehouse_sources.backend.models.ssh_tunnel import SSHTunnel
from products.warehouse_sources.backend.temporal.data_imports.settings import import_data_activity_sync
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.tls import TLSOptions
from products.warehouse_sources.backend.temporal.data_imports.workflow_activities.import_data_sync import (
ImportDataActivityInputs,
)
@@ -171,6 +172,7 @@ async def test_job_inputs_with_whitespace(activity_environment, team, **kwargs):
chunk_size_override=None,
team_id=team.id,
require_ssl=True,
+ tls=TLSOptions(),
is_initial_sync=True,
enabled_columns=None,
row_filters=[],
@@ -226,6 +228,7 @@ async def test_postgres_source_without_ssh_tunnel(activity_environment, team, **
chunk_size_override=None,
team_id=team.id,
require_ssl=True,
+ tls=TLSOptions(),
is_initial_sync=True,
enabled_columns=None,
row_filters=[],
@@ -293,6 +296,7 @@ async def test_postgres_source_with_ssh_tunnel_disabled(activity_environment, te
chunk_size_override=None,
team_id=team.id,
require_ssl=True,
+ tls=TLSOptions(),
is_initial_sync=True,
enabled_columns=None,
row_filters=[],
@@ -375,6 +379,7 @@ async def test_postgres_source_with_ssh_tunnel_enabled(activity_environment, tea
chunk_size_override=None,
team_id=team.id,
require_ssl=True,
+ tls=TLSOptions(),
is_initial_sync=True,
enabled_columns=None,
row_filters=[],