:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(mssql): stop retrying mssql syncs when sql server runs out of filegroup storage (#124657)
Tom Owers504e35ec84f3
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mssql/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/mssql/source.pyindex dbeca16f6e08..d806b24ea2de 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mssql/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/mssql/source.py@@ -64,6 +64,16 @@ _CONNECTION_TIMED_OUT_ERROR = (_GENERIC_CONNECTION_ERROR = "Could not connect to MS SQL. Please check all connection details are valid."+# SQL Server error 1105 in tempdb. tempdb is shared across sessions, so a concurrent workload that+# filled it can release it; scoped to tempdb so a customer database genuinely out of disk isn't matched.+_TEMPDB_ALLOCATION_FAILED_ERROR = "in database 'tempdb' because the '"+_TEMPDB_ALLOCATION_FAILED_EXHAUSTED_MESSAGE = (+ "Your SQL Server's tempdb ran out of space while sorting rows for this sync (SQL Server error "+ "1105), and it was still out of space after every retry. tempdb is shared across all sessions on "+ "your server, so another workload may be holding the space — check tempdb's free space and "+ "autogrowth settings. This sync is still enabled and will run again on its next schedule."+)+MSSQLErrors = {# SQL Server error 18456 is an authentication failure (wrong username/password, or the login is# disabled), not a problem with the database field. Surface the same wording the sibling SQL@@ -147,6 +157,12 @@ class MSSQLSource(# and then sent nothing, so the next attempt, which continues from the last# checkpoint where the read has one, can succeed.MSSQL_ROW_READ_TIMEOUT_ERROR,+ _TEMPDB_ALLOCATION_FAILED_ERROR,+ }++ def get_retry_exhausted_errors(self) -> dict[str, str]:+ return {+ _TEMPDB_ALLOCATION_FAILED_ERROR: _TEMPDB_ALLOCATION_FAILED_EXHAUSTED_MESSAGE,}def get_non_retryable_errors(self) -> dict[str, str | None]:diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mssql/tests/test_mssql.py b/products/warehouse_sources/backend/temporal/data_imports/sources/mssql/tests/test_mssql.pyindex 1ae91fc7c543..ab6c3d979d60 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mssql/tests/test_mssql.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/mssql/tests/test_mssql.py@@ -661,6 +661,40 @@ class TestMSSQLSourceRetryableErrors:retryable = MSSQLSource().get_retryable_errors()assert any(pattern.lower() in str(error).lower() for pattern in retryable), str(error)+ @pytest.mark.parametrize(+ "error_msg",+ [+ # Real pymssql shape for SQL Server error 1105 during an ORDER BY sort spill into tempdb.+ "(1105, b\"Could not allocate space for object 'dbo.SORT temporary run storage: "+ "140760200970240' in database 'tempdb' because the 'PRIMARY' filegroup is full due to "+ 'lack of storage space or database files reaching the maximum allowed size.")',+ "Could not allocate space for object 'dbo.sysobjvalues' in database 'tempdb' because the "+ "'SECONDARY' filegroup is full due to lack of storage space or database files reaching "+ "the maximum allowed size.",+ ],+ )+ def test_tempdb_storage_exhausted_is_retryable(self, error_msg):+ retryable = MSSQLSource().get_retryable_errors()+ assert any(pattern in error_msg for pattern in retryable), error_msg++ def test_tempdb_storage_exhausted_is_not_non_retryable(self):+ error_msg = (+ "Could not allocate space for object 'dbo.sysobjvalues' in database 'tempdb' because the "+ "'PRIMARY' filegroup is full due to lack of storage space."+ )+ non_retryable = MSSQLSource().get_non_retryable_errors()+ assert not any(pattern in error_msg for pattern in non_retryable.keys()), error_msg++ def test_tempdb_storage_exhausted_has_a_friendly_exhausted_message(self):+ exhausted = MSSQLSource().get_retry_exhausted_errors()+ error_msg = (+ "Could not allocate space for object 'dbo.SORT temporary run storage' in database "+ "'tempdb' because the 'PRIMARY' filegroup is full due to lack of storage space."+ )+ matches = [message for pattern, message in exhausted.items() if pattern in error_msg]+ assert matches, error_msg+ assert "still enabled" in matches[0]+class TestMSSQLSourceValidateCredentials:@pytest.fixture