:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(postgres): stop retrying a source that can't open its own file (#125319)
Tom Owers1901fbdd5205
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.pyindex f6b6eda9edcd..b9cd818cf4f2 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py@@ -1493,6 +1493,15 @@ class PostgresSource("missing chunk number": _SOURCE_PAGE_CORRUPTION_ERROR,"unexpected zero page": _SOURCE_PAGE_CORRUPTION_ERROR,"could not read block": _SOURCE_PAGE_CORRUPTION_ERROR,+ # The server failed to open a relation's underlying file at all, rather than failing to+ # read a block from one already open — psycopg's `IoError` (SQLSTATE 58030), raised as+ # `could not open file "base/<db>/<relfilenode>": <errno message>`. Match only the+ # "Input/output error" (EIO) errno text, the signal for a failing disk or filesystem,+ # the same family as the page-level errors above. Matching the bare "could not open+ # file" prefix instead would also catch "Too many open files" (EMFILE/ENFILE), a+ # file-descriptor exhaustion that clears once load on the source drops, so that case+ # must stay retryable. The volatile file path is excluded from the match.+ "Input/output error": _SOURCE_PAGE_CORRUPTION_ERROR,# Raised when a Postgres numeric value cannot be represented in any Delta-compatible# decimal type — the pipeline falls back through the best-fit decimal and# `decimal256(76, 32)` before giving up. Only triggers when source data genuinelydiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.pyindex ab8f8fdaff15..5f76142a30b3 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py@@ -1087,6 +1087,9 @@ class TestPostgresSourceNonRetryableErrors:"missing chunk number 0 for toast value 90210 in pg_toast_16384",'index "orders_pkey" contains unexpected zero page at block 42','could not read block 7 in file "base/16384/16385": read only 0 of 8192 bytes',+ # The server couldn't open the relation file at all (IoError, SQLSTATE 58030) rather than+ # failing to read a block from one already open.+ 'could not open file "base/16384/1259": Input/output error',# Temporal-wrapped message (what the workflow-level check sees) — carries the class name."InternalError_: missing chunk number 0 for toast value 90210 in pg_toast_16384",],@@ -1102,6 +1105,16 @@ class TestPostgresSourceNonRetryableErrors:assert friendly, "Damaged source page should surface an actionable message"assert "damaged data on disk" in friendly[0]+ def test_file_descriptor_exhaustion_on_open_is_retryable(self, source):+ # Same "could not open file" prefix as the damaged-page case above, but a different errno+ # reason: file-descriptor exhaustion clears once load on the source drops, unlike the+ # physical "Input/output error" case, so it must not be classified as permanent corruption.+ error_msg = 'could not open file "base/16384/1259": Too many open files'+ non_retryable = source.get_non_retryable_errors()+ assert not error_message_matches(error_msg, non_retryable.keys()), (+ f"Transient file-descriptor exhaustion should be retryable: {error_msg}"+ )+@pytest.mark.parametrize("error_msg",[