Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(mssql): stop retrying mssql syncs when sql server runs out of filegroup storage (#124657)

Tom Owers504e35ec84f3
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mssql/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/mssql/source.py
index dbeca16f6e08..d806b24ea2de 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mssql/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/mssql/source.py
@@ -64,6 +64,16 @@ _CONNECTION_TIMED_OUT_ERROR = (
_GENERIC_CONNECTION_ERROR = "Could not connect to MS SQL. Please check all connection details are valid."
+# SQL Server error 1105 in tempdb. tempdb is shared across sessions, so a concurrent workload that
+# filled it can release it; scoped to tempdb so a customer database genuinely out of disk isn't matched.
+_TEMPDB_ALLOCATION_FAILED_ERROR = "in database 'tempdb' because the '"
+_TEMPDB_ALLOCATION_FAILED_EXHAUSTED_MESSAGE = (
+ "Your SQL Server's tempdb ran out of space while sorting rows for this sync (SQL Server error "
+ "1105), and it was still out of space after every retry. tempdb is shared across all sessions on "
+ "your server, so another workload may be holding the space — check tempdb's free space and "
+ "autogrowth settings. This sync is still enabled and will run again on its next schedule."
+)
+
MSSQLErrors = {
# SQL Server error 18456 is an authentication failure (wrong username/password, or the login is
# disabled), not a problem with the database field. Surface the same wording the sibling SQL
@@ -147,6 +157,12 @@ class MSSQLSource(
# and then sent nothing, so the next attempt, which continues from the last
# checkpoint where the read has one, can succeed.
MSSQL_ROW_READ_TIMEOUT_ERROR,
+ _TEMPDB_ALLOCATION_FAILED_ERROR,
+ }
+
+ def get_retry_exhausted_errors(self) -> dict[str, str]:
+ return {
+ _TEMPDB_ALLOCATION_FAILED_ERROR: _TEMPDB_ALLOCATION_FAILED_EXHAUSTED_MESSAGE,
}
def get_non_retryable_errors(self) -> dict[str, str | None]:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mssql/tests/test_mssql.py b/products/warehouse_sources/backend/temporal/data_imports/sources/mssql/tests/test_mssql.py
index 1ae91fc7c543..ab6c3d979d60 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mssql/tests/test_mssql.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/mssql/tests/test_mssql.py
@@ -661,6 +661,40 @@ class TestMSSQLSourceRetryableErrors:
retryable = MSSQLSource().get_retryable_errors()
assert any(pattern.lower() in str(error).lower() for pattern in retryable), str(error)
+ @pytest.mark.parametrize(
+ "error_msg",
+ [
+ # Real pymssql shape for SQL Server error 1105 during an ORDER BY sort spill into tempdb.
+ "(1105, b\"Could not allocate space for object 'dbo.SORT temporary run storage: "
+ "140760200970240' in database 'tempdb' because the 'PRIMARY' filegroup is full due to "
+ 'lack of storage space or database files reaching the maximum allowed size.")',
+ "Could not allocate space for object 'dbo.sysobjvalues' in database 'tempdb' because the "
+ "'SECONDARY' filegroup is full due to lack of storage space or database files reaching "
+ "the maximum allowed size.",
+ ],
+ )
+ def test_tempdb_storage_exhausted_is_retryable(self, error_msg):
+ retryable = MSSQLSource().get_retryable_errors()
+ assert any(pattern in error_msg for pattern in retryable), error_msg
+
+ def test_tempdb_storage_exhausted_is_not_non_retryable(self):
+ error_msg = (
+ "Could not allocate space for object 'dbo.sysobjvalues' in database 'tempdb' because the "
+ "'PRIMARY' filegroup is full due to lack of storage space."
+ )
+ non_retryable = MSSQLSource().get_non_retryable_errors()
+ assert not any(pattern in error_msg for pattern in non_retryable.keys()), error_msg
+
+ def test_tempdb_storage_exhausted_has_a_friendly_exhausted_message(self):
+ exhausted = MSSQLSource().get_retry_exhausted_errors()
+ error_msg = (
+ "Could not allocate space for object 'dbo.SORT temporary run storage' in database "
+ "'tempdb' because the 'PRIMARY' filegroup is full due to lack of storage space."
+ )
+ matches = [message for pattern, message in exhausted.items() if pattern in error_msg]
+ assert matches, error_msg
+ assert "still enabled" in matches[0]
+
class TestMSSQLSourceValidateCredentials:
@pytest.fixture