Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-imports): replace invalid utf-8 in postgres text columns (#124218)

Tom Owers71a4adc724d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/postgres.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/postgres.py
index 060c731b311a..d5ef0c93e079 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/postgres.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/postgres.py
@@ -27,6 +27,7 @@ import psycopg
import pyarrow as pa
import structlog
from psycopg import sql
+from psycopg.abc import AdaptContext
from psycopg.adapt import Loader
from psycopg.types.datetime import TimeLoader, TimestampLoader, TimestamptzLoader, TimetzLoader
from structlog.types import FilteringBoundLogger
@@ -2080,6 +2081,31 @@ class NetworkAsStringLoader(Loader):
return bytes(data).decode("utf-8")
+class LenientTextLoader(Loader):
+ """Load PostgreSQL text values, replacing bytes that are not valid UTF-8.
+
+ Postgres does not re-validate stored text on output, so a `SQL_ASCII` database or an
+ extension that writes raw bytes into a `text` column (e.g. Supabase's `pg_net` storing
+ binary HTTP response bodies) can send invalid UTF-8. psycopg's default loader decodes
+ strictly, so one such value fails the whole sync with `UnicodeDecodeError`.
+
+ Decodes with the negotiated client encoding, because a pooler that rejects the startup
+ `options` leaves the connection on the server's own encoding (e.g. LATIN1). `SQL_ASCII`
+ decodes as UTF-8, because psycopg would return `bytes` for it, which Arrow cannot store
+ in a string column.
+ """
+
+ def __init__(self, oid: int, context: AdaptContext | None = None):
+ super().__init__(oid, context)
+ encoding = self.connection.info.encoding if self.connection else "utf-8"
+ self._encoding = "utf-8" if encoding == "ascii" else encoding
+
+ def load(self, data):
+ if data is None:
+ return None
+ return bytes(data).decode(self._encoding, "replace")
+
+
class SafeDateLoader(Loader):
"""Load PostgreSQL dates, handling edge cases beyond Python's date range.
@@ -4207,6 +4233,10 @@ def postgres_source(
connection.adapters.register_loader("timetz", SafeTimetzLoader)
connection.adapters.register_loader("inet", NetworkAsStringLoader)
connection.adapters.register_loader("cidr", NetworkAsStringLoader)
+ connection.adapters.register_loader("text", LenientTextLoader)
+ connection.adapters.register_loader("varchar", LenientTextLoader)
+ connection.adapters.register_loader("bpchar", LenientTextLoader)
+ connection.adapters.register_loader("name", LenientTextLoader)
# Bump statement_timeout for the streaming connection. A server
# cursor FETCH inherits the session statement_timeout, and on
# wide/partitioned scans the source's default (often 30-60s)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py
index 391dbb002184..64fbb6dddc27 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py
@@ -6,6 +6,7 @@ from collections.abc import Generator, Iterable, Iterator
from contextlib import ExitStack, contextmanager
from datetime import UTC, date, datetime, time, timedelta, timezone
from pathlib import Path
+from types import SimpleNamespace
from typing import Any, cast
import pytest
@@ -98,6 +99,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.p
UNFILTERED_COUNT_CLIENT_DEADLINE_SECONDS,
XMIN_PROJECTED_COLUMN,
JsonAsStringLoader,
+ LenientTextLoader,
NetworkAsStringLoader,
PostgresDiscoveredSchema,
PostgresImplementation,
@@ -8626,6 +8628,27 @@ class TestNetworkAsStringLoader:
assert loader.load(None) is None
+class TestLenientTextLoader:
+ @pytest.mark.parametrize(
+ "encoding,data,expected",
+ [
+ (None, b"plain text", "plain text"),
+ (None, "café 日本".encode(), "café 日本"),
+ (None, b"\xff\xd8\xffbinary", "\ufffd\ufffd\ufffdbinary"),
+ (None, memoryview(b"ok\xfe"), "ok\ufffd"),
+ (None, None, None),
+ ("utf-8", b"ok\xff", "ok\ufffd"),
+ ("iso8859-1", "café".encode("latin-1"), "café"),
+ ("ascii", b"caf\xc3\xa9\xff", "café\ufffd"),
+ ],
+ )
+ def test_load(self, encoding, data, expected):
+ context = (
+ SimpleNamespace(connection=SimpleNamespace(info=SimpleNamespace(encoding=encoding))) if encoding else None
+ )
+ assert LenientTextLoader(oid=25, context=context).load(data) == expected
+
+
class TestSSLRequiredAfterDate:
def test_date_value(self):
assert SSL_REQUIRED_AFTER_DATE == datetime(2026, 2, 18, tzinfo=UTC)