Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(postgres): stop retrying a source that can't open its own file (#125319)

Tom Owers1901fbdd5205
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py
index f6b6eda9edcd..b9cd818cf4f2 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py
@@ -1493,6 +1493,15 @@ class PostgresSource(
"missing chunk number": _SOURCE_PAGE_CORRUPTION_ERROR,
"unexpected zero page": _SOURCE_PAGE_CORRUPTION_ERROR,
"could not read block": _SOURCE_PAGE_CORRUPTION_ERROR,
+ # The server failed to open a relation's underlying file at all, rather than failing to
+ # read a block from one already open — psycopg's `IoError` (SQLSTATE 58030), raised as
+ # `could not open file "base/<db>/<relfilenode>": <errno message>`. Match only the
+ # "Input/output error" (EIO) errno text, the signal for a failing disk or filesystem,
+ # the same family as the page-level errors above. Matching the bare "could not open
+ # file" prefix instead would also catch "Too many open files" (EMFILE/ENFILE), a
+ # file-descriptor exhaustion that clears once load on the source drops, so that case
+ # must stay retryable. The volatile file path is excluded from the match.
+ "Input/output error": _SOURCE_PAGE_CORRUPTION_ERROR,
# Raised when a Postgres numeric value cannot be represented in any Delta-compatible
# decimal type — the pipeline falls back through the best-fit decimal and
# `decimal256(76, 32)` before giving up. Only triggers when source data genuinely
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py
index ab8f8fdaff15..5f76142a30b3 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py
@@ -1087,6 +1087,9 @@ class TestPostgresSourceNonRetryableErrors:
"missing chunk number 0 for toast value 90210 in pg_toast_16384",
'index "orders_pkey" contains unexpected zero page at block 42',
'could not read block 7 in file "base/16384/16385": read only 0 of 8192 bytes',
+ # The server couldn't open the relation file at all (IoError, SQLSTATE 58030) rather than
+ # failing to read a block from one already open.
+ 'could not open file "base/16384/1259": Input/output error',
# Temporal-wrapped message (what the workflow-level check sees) — carries the class name.
"InternalError_: missing chunk number 0 for toast value 90210 in pg_toast_16384",
],
@@ -1102,6 +1105,16 @@ class TestPostgresSourceNonRetryableErrors:
assert friendly, "Damaged source page should surface an actionable message"
assert "damaged data on disk" in friendly[0]
+ def test_file_descriptor_exhaustion_on_open_is_retryable(self, source):
+ # Same "could not open file" prefix as the damaged-page case above, but a different errno
+ # reason: file-descriptor exhaustion clears once load on the source drops, unlike the
+ # physical "Input/output error" case, so it must not be classified as permanent corruption.
+ error_msg = 'could not open file "base/16384/1259": Too many open files'
+ non_retryable = source.get_non_retryable_errors()
+ assert not error_message_matches(error_msg, non_retryable.keys()), (
+ f"Transient file-descriptor exhaustion should be retryable: {error_msg}"
+ )
+
@pytest.mark.parametrize(
"error_msg",
[