:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(mysql): retry a vtgate connection error with no serving endpoints (#117308)
Tom Owers9bb6f0ed120e
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mysql/mysql.py b/products/warehouse_sources/backend/temporal/data_imports/sources/mysql/mysql.pyindex bd3b15856356..f03cd0402c18 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mysql/mysql.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/mysql/mysql.py@@ -625,6 +625,23 @@ def _is_transient_vitess_dial_timeout(e: BaseException) -> bool:return _VITESS_DIAL_TOKEN in args_text and _VITESS_DIAL_TIMEOUT_TOKEN in args_text+# Vitess/PlanetScale vtgate surfaces this 1105 error (gRPC `Unavailable`, the same status as+# `_is_transient_tablet_unavailable`'s `code = Unavailable` wrapping) when its tablet discovery+# has zero serving tablets for the target keyspace/shard/tablet-type at the moment the connection+# is opened — a failover, a restart, or a rollout that briefly leaves no tablet registered as+# healthy. A fresh attempt after a short backoff usually lands once a tablet comes back online.+# Match the stable `vtgate connection error: no endpoints` phrase, excluding the volatile attempt+# count and reqid that follow it.+_VITESS_NO_ENDPOINTS_TOKEN = "vtgate connection error: no endpoints"+++def _is_transient_vitess_no_endpoints(e: BaseException) -> bool:+ """Return True if a Vitess vtgate found no serving tablet endpoints — a transient blip."""+ if not isinstance(e, pymysql.err.OperationalError):+ return False+ return _VITESS_NO_ENDPOINTS_TOKEN in " ".join(str(arg) for arg in e.args)++# MySQL/MariaDB error 1040 (ER_CON_COUNT_ERROR): the server refuses a *new* connection because# `max_connections` is already reached. A transient capacity condition on the customer's database,# not a misconfiguration — a slot frees the moment another connection closes — so a fresh attempt@@ -954,6 +971,7 @@ def _connect_with_transient_retry(kwargs: dict[str, Any], team_id: int | None) -or _is_transient_connect_broken_pipe(e)or _is_transient_packet_sequence_error(e)or _is_transient_vitess_dial_timeout(e)+ or _is_transient_vitess_no_endpoints(e)or _is_transient_tiproxy_unavailable(e)or _is_transient_no_available_tidb_instances(e)or _is_transient_too_many_connections(e)@@ -1076,7 +1094,8 @@ def _retry_on_transient_tablet_unavailable(failing sync setup on the first blip and surfacing it as captured error-trackingnoise. Non-transient errors re-raise immediately — the predicates only match the gRPC`Unavailable` status, a mid-reparent primary, a dial-timeout reaching a backend tablet,- a TiProxy failover, or a plain peer-reset connection drop, all self-healing.+ a vtgate with no serving tablet endpoints, a TiProxy failover, or a plain peer-reset+ connection drop, all self-healing."""attempt = 0while True:@@ -1088,6 +1107,7 @@ def _retry_on_transient_tablet_unavailable(_is_transient_tablet_unavailable(e)or _is_transient_vitess_reparent(e)or _is_transient_vitess_dial_timeout(e)+ or _is_transient_vitess_no_endpoints(e)or _is_transient_tiproxy_unavailable(e)or _is_transient_metadata_query_reset(e)):diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mysql/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/mysql/source.pyindex 5c76c241e7f1..97666bcfa7ef 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mysql/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/mysql/source.py@@ -584,6 +584,13 @@ class MySQLSource(# already retries it in-process (see `_is_transient_no_available_tidb_instances` in# mysql.py); this is the backstop for the rare case where it exhausts that budget."No available TiDB instances, please make sure TiDB is available",+ # A Vitess/PlanetScale vtgate found zero serving tablet endpoints for the target+ # keyspace/shard/tablet-type — the same gRPC `Unavailable` status as `code =+ # Unavailable` above, phrased differently. Both `_connect_with_transient_retry` and+ # `_retry_on_transient_tablet_unavailable` already retry it in-process (see+ # `_is_transient_vitess_no_endpoints` in mysql.py); this is the backstop for the rare+ # case where it exhausts that budget.+ "vtgate connection error: no endpoints",# Vitess/PlanetScale vtgate error 1105 raised while a streaming query is in flight:# vtgate's own gRPC client to the backend vttablet was already closing (a tablet# swap during a failover, reparent, or health-check-triggered pool recycle) when thediff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mysql/tests/test_mysql.py b/products/warehouse_sources/backend/temporal/data_imports/sources/mysql/tests/test_mysql.pyindex 3fd56cb7244c..ee9266403d02 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mysql/tests/test_mysql.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/mysql/tests/test_mysql.py@@ -59,6 +59,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.mysql.mysq_is_transient_tiproxy_unavailable,_is_transient_too_many_connections,_is_transient_vitess_dial_timeout,+ _is_transient_vitess_no_endpoints,_is_transient_vitess_reparent,_reconnect_pinned,_release_streaming_cursor,@@ -1628,6 +1629,39 @@ class TestIsTransientVitessDialTimeout:assert not _is_transient_vitess_dial_timeout(ValueError("dial tcp 10.0.0.1: connection timed out"))+class TestIsTransientVitessNoEndpoints:+ @pytest.mark.parametrize(+ "message",+ [+ # The shape a Vitess/PlanetScale vtgate surfaces when it has zero serving tablet+ # endpoints for the target — the attempt count and reqid both vary, the stable signal+ # is the "vtgate connection error: no endpoints" phrase.+ "unavailable: vtgate connection error: no endpoints, after 1 attempts, reqid=csYTzBMNB2hB8111yzcg4A",+ "unavailable: vtgate connection error: no endpoints",+ ],+ )+ def test_matches_no_endpoints(self, message):+ assert _is_transient_vitess_no_endpoints(pymysql.err.OperationalError(1105, message))++ @pytest.mark.parametrize(+ "code,message",+ [+ # A different 1105 payload must not be absorbed by this predicate.+ (1105, "unknown: target: keyspace.-.primary: code = Unavailable"),+ (2003, "Can't connect to MySQL server on 'db.example.com' (timed out)"),+ (1045, "Access denied for user"),+ ],+ )+ def test_does_not_match_other_errors(self, code, message):+ assert not _is_transient_vitess_no_endpoints(pymysql.err.OperationalError(code, message))++ def test_does_not_match_error_without_args(self):+ assert not _is_transient_vitess_no_endpoints(pymysql.err.OperationalError())++ def test_does_not_match_non_operational_error(self):+ assert not _is_transient_vitess_no_endpoints(ValueError("vtgate connection error: no endpoints"))++class TestIsTransientTooManyConnections:def test_matches_too_many_connections(self):assert _is_transient_too_many_connections(pymysql.err.OperationalError(1040, "Too many connections"))@@ -2956,6 +2990,24 @@ class TestMySQLSourceNonRetryableErrors:is_retryable = any(pattern in error_msg for pattern in retryable)assert is_retryable, f"No-available-TiDB-instances error should be classified retryable: {error_msg}"+ @pytest.mark.parametrize(+ "error_msg",+ [+ "OperationalError: (1105, 'unavailable: vtgate connection error: no endpoints, "+ "after 1 attempts, reqid=csYTzBMNB2hB8111yzcg4A')",+ "vtgate connection error: no endpoints",+ ],+ )+ def test_vitess_no_endpoints_is_classified_retryable(self, source, error_msg):+ # Both `_connect_with_transient_retry` and `_retry_on_transient_tablet_unavailable`+ # already retry this in-process (see `_is_transient_vitess_no_endpoints` in mysql.py);+ # once exhausted it re-raises for Temporal to retry the whole activity. Without this+ # classification `_handle_import_error` logs it at `exception` on every occurrence,+ # flooding error tracking with a self-recovering Vitess/PlanetScale failover.+ retryable = source.get_retryable_errors()+ is_retryable = any(pattern in error_msg for pattern in retryable)+ assert is_retryable, f"Vitess no-endpoints error should be classified retryable: {error_msg}"+@pytest.mark.parametrize("error_msg",[