:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-imports): replace invalid utf-8 in postgres text columns (#124218)
Tom Owers71a4adc724d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/postgres.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/postgres.pyindex 060c731b311a..d5ef0c93e079 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/postgres.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/postgres.py@@ -27,6 +27,7 @@ import psycopgimport pyarrow as paimport structlogfrom psycopg import sql+from psycopg.abc import AdaptContextfrom psycopg.adapt import Loaderfrom psycopg.types.datetime import TimeLoader, TimestampLoader, TimestamptzLoader, TimetzLoaderfrom structlog.types import FilteringBoundLogger@@ -2080,6 +2081,31 @@ class NetworkAsStringLoader(Loader):return bytes(data).decode("utf-8")+class LenientTextLoader(Loader):+ """Load PostgreSQL text values, replacing bytes that are not valid UTF-8.++ Postgres does not re-validate stored text on output, so a `SQL_ASCII` database or an+ extension that writes raw bytes into a `text` column (e.g. Supabase's `pg_net` storing+ binary HTTP response bodies) can send invalid UTF-8. psycopg's default loader decodes+ strictly, so one such value fails the whole sync with `UnicodeDecodeError`.++ Decodes with the negotiated client encoding, because a pooler that rejects the startup+ `options` leaves the connection on the server's own encoding (e.g. LATIN1). `SQL_ASCII`+ decodes as UTF-8, because psycopg would return `bytes` for it, which Arrow cannot store+ in a string column.+ """++ def __init__(self, oid: int, context: AdaptContext | None = None):+ super().__init__(oid, context)+ encoding = self.connection.info.encoding if self.connection else "utf-8"+ self._encoding = "utf-8" if encoding == "ascii" else encoding++ def load(self, data):+ if data is None:+ return None+ return bytes(data).decode(self._encoding, "replace")++class SafeDateLoader(Loader):"""Load PostgreSQL dates, handling edge cases beyond Python's date range.@@ -4207,6 +4233,10 @@ def postgres_source(connection.adapters.register_loader("timetz", SafeTimetzLoader)connection.adapters.register_loader("inet", NetworkAsStringLoader)connection.adapters.register_loader("cidr", NetworkAsStringLoader)+ connection.adapters.register_loader("text", LenientTextLoader)+ connection.adapters.register_loader("varchar", LenientTextLoader)+ connection.adapters.register_loader("bpchar", LenientTextLoader)+ connection.adapters.register_loader("name", LenientTextLoader)# Bump statement_timeout for the streaming connection. A server# cursor FETCH inherits the session statement_timeout, and on# wide/partitioned scans the source's default (often 30-60s)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.pyindex 391dbb002184..64fbb6dddc27 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py@@ -6,6 +6,7 @@ from collections.abc import Generator, Iterable, Iteratorfrom contextlib import ExitStack, contextmanagerfrom datetime import UTC, date, datetime, time, timedelta, timezonefrom pathlib import Path+from types import SimpleNamespacefrom typing import Any, castimport pytest@@ -98,6 +99,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.pUNFILTERED_COUNT_CLIENT_DEADLINE_SECONDS,XMIN_PROJECTED_COLUMN,JsonAsStringLoader,+ LenientTextLoader,NetworkAsStringLoader,PostgresDiscoveredSchema,PostgresImplementation,@@ -8626,6 +8628,27 @@ class TestNetworkAsStringLoader:assert loader.load(None) is None+class TestLenientTextLoader:+ @pytest.mark.parametrize(+ "encoding,data,expected",+ [+ (None, b"plain text", "plain text"),+ (None, "café 日本".encode(), "café 日本"),+ (None, b"\xff\xd8\xffbinary", "\ufffd\ufffd\ufffdbinary"),+ (None, memoryview(b"ok\xfe"), "ok\ufffd"),+ (None, None, None),+ ("utf-8", b"ok\xff", "ok\ufffd"),+ ("iso8859-1", "café".encode("latin-1"), "café"),+ ("ascii", b"caf\xc3\xa9\xff", "café\ufffd"),+ ],+ )+ def test_load(self, encoding, data, expected):+ context = (+ SimpleNamespace(connection=SimpleNamespace(info=SimpleNamespace(encoding=encoding))) if encoding else None+ )+ assert LenientTextLoader(oid=25, context=context).load(data) == expected++class TestSSLRequiredAfterDate:def test_date_value(self):assert SSL_REQUIRED_AFTER_DATE == datetime(2026, 2, 18, tzinfo=UTC)