Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(mysql): treat server health-monitor offline mode as non-retryable (#128622)

Tom Owersd9d8360c40b8
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mysql/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/mysql/source.py
index 3cf5d485f219..ed29267bf6a2 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mysql/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/mysql/source.py
@@ -583,6 +583,15 @@ class MySQLSource(
# locked-account (4151) and host-blocked (1129) entries above. Match the locale-independent
# error code (the message text is translated on non-English servers).
"(3032,": "Your MySQL/MariaDB server is in offline mode (error 3032), which a database admin turned on to block non-admin connections, usually ahead of maintenance. Ask your database admin to turn it back off ('SET GLOBAL offline_mode = OFF'), then retry the sync.",
+ # Error 6400 isn't standard MySQL/MariaDB numbering — it's a vendor extension some
+ # MySQL-wire-protocol servers (e.g. SingleStore) use for their own health monitor putting
+ # the server into offline mode automatically, most often because it's low on disk space.
+ # Unlike the admin-triggered (3032,) case above, no person flips this switch, but the
+ # effect on the sync is identical: every retry reconnects to the same offline server and
+ # fails the same way until the triggering resource problem is resolved. Match the
+ # locale-independent code — the reason text and the timestamp that follow it are volatile
+ # and vendor-specific.
+ "(6400,": "Your database server put itself into offline mode (error 6400) because its own health monitor flagged a resource problem, most often low disk space. Free up disk space (or resolve whatever resource issue the server reported) on your database server — it should come back online on its own once the condition clears — then retry the sync.",
}
def get_retryable_errors(self) -> set[str]:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mysql/tests/test_mysql.py b/products/warehouse_sources/backend/temporal/data_imports/sources/mysql/tests/test_mysql.py
index 99471453674a..d538963ccb56 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mysql/tests/test_mysql.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/mysql/tests/test_mysql.py
@@ -2518,11 +2518,13 @@ class TestMySQLSourceNonRetryableErrors:
[
"(3032, 'The server is currently in offline mode')",
"OperationalError: (3032, 'The server is currently in offline mode')",
+ "(6400, 'The server is currently in offline mode since 2026-11-03 14:27:19.007589, reason: (HM) low disk space')",
],
)
def test_server_offline_mode_is_non_retryable(self, source, error_msg):
- # A DB admin put the server into offline mode — only they can turn it back off, and
- # every retry reconnects as the same non-admin user and fails identically until they do.
+ # A DB admin (3032) or the server's own health monitor (6400) put the server into offline
+ # mode — only an admin or a resolved resource problem turns it back off, and every retry
+ # reconnects to the same offline server and fails identically until then.
non_retryable = source.get_non_retryable_errors()
friendly = next(
(message for pattern, message in non_retryable.items() if pattern in error_msg),