Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-warehouse): report unreachable vendors instead of raising in credential probes (#120491)

Tom Owersd277cd41c76e
diff --git a/products/warehouse_sources/backend/presentation/views/external_data_schema.py b/products/warehouse_sources/backend/presentation/views/external_data_schema.py
index bdd59384f459..67299815fb77 100644
--- a/products/warehouse_sources/backend/presentation/views/external_data_schema.py
+++ b/products/warehouse_sources/backend/presentation/views/external_data_schema.py
@@ -2365,9 +2365,14 @@ class ExternalDataSchemaViewset(TeamAndOrgViewSetMixin, viewsets.ModelViewSet):
config = new_source.parse_config(source.job_inputs)
effective_api_version = new_source.resolve_api_version(instance.api_version or source.api_version)
- credentials_valid, credentials_error = new_source.validate_credentials(
- config, self.team_id, instance.name, api_version=effective_api_version
- )
+ try:
+ credentials_valid, credentials_error = new_source.validate_credentials(
+ config, self.team_id, instance.name, api_version=effective_api_version
+ )
+ except Exception as e:
+ credentials_valid, credentials_error = source_helpers._credentials_validation_failed(
+ new_source, self.team_id, e
+ )
if not credentials_valid:
return Response(
status=status.HTTP_400_BAD_REQUEST,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_guardduty/aws_guardduty.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_guardduty/aws_guardduty.py
index e274a8ba90b8..cb5166eec39e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_guardduty/aws_guardduty.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_guardduty/aws_guardduty.py
@@ -27,6 +27,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.htt
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http.transport import BoundedRetry
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import schema_for_resource
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import probe_status_message
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.awsguardduty import (
AwsGuarddutySourceConfig,
@@ -54,9 +55,10 @@ class AwsGuarddutyResumeConfig:
class AwsGuarddutyError(Exception):
- def __init__(self, code: str, message: str) -> None:
+ def __init__(self, code: str, message: str, status_code: int | None = None) -> None:
self.code = code
self.message = message
+ self.status_code = status_code
super().__init__(f"AWS GuardDuty request failed: {code} - {message}")
@@ -86,7 +88,7 @@ def error_for_response(response: requests.Response) -> AwsGuarddutyError:
if response.status_code == 400 and code in {"ThrottlingException", "TooManyRequestsException"}
else AwsGuarddutyError
)
- return error_class(code, message)
+ return error_class(code, message, response.status_code)
class AwsGuarddutyClient:
@@ -318,7 +320,7 @@ def validate_credentials(
except AwsGuarddutyError as error:
if schema_name is None and error.code in {"AccessDenied", "AccessDeniedException", "ForbiddenException"}:
return True, None
- message = permission_message(error, schema_name)
+ message = permission_message(error, schema_name) or probe_status_message("AWS GuardDuty", error.status_code)
if message:
return False, message
raise
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_guardduty/tests/test_aws_guardduty.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_guardduty/tests/test_aws_guardduty.py
index 83d5fcfe2541..95e52e24e07e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_guardduty/tests/test_aws_guardduty.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_guardduty/tests/test_aws_guardduty.py
@@ -414,10 +414,12 @@ def test_findings_validation_checks_hydration_permission(config: AwsGuarddutySou
assert network.requests_log[-1].path.endswith("/findings/get")
-def test_server_errors_propagate_from_validation(config: AwsGuarddutySourceConfig) -> None:
+def test_server_errors_report_an_unreachable_vendor(config: AwsGuarddutySourceConfig) -> None:
with scripted_network(always(ScriptedResponse(status=500, json={"__type": "InternalServerErrorException"}))):
- with pytest.raises(AwsGuarddutyError, match="InternalServerErrorException"):
- validate_credentials(config, VERSION)
+ assert validate_credentials(config, VERSION) == (
+ False,
+ "Couldn't reach AWS GuardDuty to check your credentials. Try again in a few minutes.",
+ )
@pytest.mark.parametrize(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_inspector/aws_inspector.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_inspector/aws_inspector.py
index ab5d831f0cbc..a3413851a90b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_inspector/aws_inspector.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_inspector/aws_inspector.py
@@ -23,6 +23,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.aws_inspec
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http.transport import BoundedRetry
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import probe_status_message
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.awsinspector import (
AwsInspectorSourceConfig,
@@ -60,10 +61,11 @@ class FanOutPosition:
class AwsInspectorError(Exception):
- def __init__(self, code: str, message: str) -> None:
+ def __init__(self, code: str, message: str, status_code: int | None = None) -> None:
super().__init__(f"AWS Inspector request failed: {code} - {message}")
self.code = code
self.message = message
+ self.status_code = status_code
def error_for_response(response: requests.Response) -> AwsInspectorError:
@@ -80,7 +82,7 @@ def error_for_response(response: requests.Response) -> AwsInspectorError:
phrase in message.lower() for phrase in ("not subscribed", "not enabled", "subscription is required")
):
code = "SubscriptionRequiredException"
- return AwsInspectorError(code, message)
+ return AwsInspectorError(code, message, response.status_code)
def error_message(error: AwsInspectorError) -> str | None:
@@ -443,7 +445,7 @@ def validate_credentials(
if schema_name is None:
return True, None
return False, f"Grant {', '.join(endpoint.iam_actions)} to read this table."
- message = error_message(error)
+ message = error_message(error) or probe_status_message("AWS Inspector", error.status_code)
if message:
return False, message
raise
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_inspector/tests/test_aws_inspector.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_inspector/tests/test_aws_inspector.py
index dd1b04e2d7e7..134231631bdb 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_inspector/tests/test_aws_inspector.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_inspector/tests/test_aws_inspector.py
@@ -12,7 +12,6 @@ from botocore.validate import validate_parameters
from products.warehouse_sources.backend.temporal.data_imports.sources.aws_inspector.aws_inspector import (
AwsInspectorClient,
- AwsInspectorError,
AwsInspectorResumeConfig,
aws_inspector_source,
error_for_response,
@@ -365,11 +364,15 @@ def test_credential_errors(
@pytest.mark.parametrize(
"status,code", [(429, "ThrottlingException"), (500, "InternalServerException"), (503, "HTTP 503")]
)
-def test_transient_errors_propagate(config: AwsInspectorSourceConfig, status: int, code: str) -> None:
+def test_transient_errors_report_an_unreachable_vendor(
+ config: AwsInspectorSourceConfig, status: int, code: str
+) -> None:
script = [ScriptedResponse(json={"__type": code}, status=status) for _ in range(4)]
with scripted_network(script):
- with pytest.raises(AwsInspectorError, match=code):
- validate_credentials(config)
+ assert validate_credentials(config) == (
+ False,
+ "Couldn't reach AWS Inspector to check your credentials. Try again in a few minutes.",
+ )
assert not any(
pattern in f"AWS Inspector request failed: {code}"
for pattern in AwsInspectorSource().get_non_retryable_errors()
@@ -499,8 +502,10 @@ def test_malformed_response(config: AwsInspectorSourceConfig, http: MagicMock, s
validate_credentials(config)
else:
http.post.return_value._content = b"not JSON"
- with pytest.raises(AwsInspectorError, match="HTTP 503"):
- validate_credentials(config)
+ assert validate_credentials(config) == (
+ False,
+ "Couldn't reach AWS Inspector to check your credentials. Try again in a few minutes.",
+ )
http.close.assert_called_once()
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_macie/aws_macie.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_macie/aws_macie.py
index 742de0c17b28..34c42da897ce 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_macie/aws_macie.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_macie/aws_macie.py
@@ -24,6 +24,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.aws_macie.
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http.transport import BoundedRetry
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import probe_status_message
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
if TYPE_CHECKING:
@@ -45,8 +46,9 @@ class AwsMacieResumeConfig:
class AwsMacieError(Exception):
- def __init__(self, code: str, message: str) -> None:
+ def __init__(self, code: str, message: str, status_code: int | None = None) -> None:
self.code = code
+ self.status_code = status_code
super().__init__(f"AWS Macie request failed: {code} - {message}")
@@ -113,7 +115,11 @@ class AwsMacieClient:
error = {}
raw_code = response.headers.get("x-amzn-ErrorType") or error.get("__type") or error.get("code")
code = str(raw_code or f"HTTP {response.status_code}").split(":")[0].split("#")[-1]
- raise AwsMacieError(code, str(error.get("message") or error.get("Message") or "Request failed.")[:500])
+ raise AwsMacieError(
+ code,
+ str(error.get("message") or error.get("Message") or "Request failed.")[:500],
+ response.status_code,
+ )
return response.json()
@@ -307,6 +313,9 @@ def validate_credentials(
for pattern, message in ERROR_MESSAGES.items():
if pattern.lower() in str(error).lower():
return False, message
+ status_message = probe_status_message("AWS Macie", error.status_code)
+ if status_message:
+ return False, status_message
raise
finally:
client.close()
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_macie/tests/test_aws_macie.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_macie/tests/test_aws_macie.py
index 5c794dd0c420..505dd7015162 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_macie/tests/test_aws_macie.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_macie/tests/test_aws_macie.py
@@ -424,12 +424,14 @@ def test_findings_permission_probe_reads_details(config: AwsMacieSourceConfig) -
@pytest.mark.parametrize("status,code", [(429, "ThrottlingException"), (500, "InternalServerException")])
-def test_transient_errors_propagate(config: AwsMacieSourceConfig, status: int, code: str) -> None:
+def test_transient_errors_report_an_unreachable_vendor(config: AwsMacieSourceConfig, status: int, code: str) -> None:
with scripted_network(
always(ScriptedResponse(json={"message": "Try again."}, status=status, headers={"x-amzn-ErrorType": code}))
) as network:
- with pytest.raises(AwsMacieError, match=code):
- validate_credentials(config)
+ assert validate_credentials(config) == (
+ False,
+ "Couldn't reach AWS Macie to check your credentials. Try again in a few minutes.",
+ )
assert all(request.path == "/findings" for request in network.requests_log)
assert not any(pattern.lower() in code.lower() for pattern in AwsMacieSource().get_non_retryable_errors())
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_sagemaker/aws_sagemaker.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_sagemaker/aws_sagemaker.py
index b7bbda2f8508..f1c37edfc187 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_sagemaker/aws_sagemaker.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_sagemaker/aws_sagemaker.py
@@ -23,6 +23,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.htt
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http.transport import BoundedRetry
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import schema_for_resource
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import probe_status_message
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.awssagemaker import (
AwsSagemakerSourceConfig,
@@ -72,8 +73,9 @@ class AwsSagemakerResumeConfig:
class AwsSagemakerError(Exception):
- def __init__(self, code: str) -> None:
+ def __init__(self, code: str, status_code: int | None = None) -> None:
self.code = code
+ self.status_code = status_code
super().__init__(f"AWS SageMaker request failed: {code}")
@@ -99,8 +101,8 @@ def error_for_response(response: requests.Response) -> AwsSagemakerError:
code = str(raw_code or f"HTTP {response.status_code}").split(":")[0].split("#")[-1]
# Only body-level throttles need another retry layer; the transport retries 429 and 5xx.
if response.status_code == 400 and code in THROTTLE_CODES:
- return AwsSagemakerThrottledError(code)
- return AwsSagemakerError(code)
+ return AwsSagemakerThrottledError(code, response.status_code)
+ return AwsSagemakerError(code, response.status_code)
class AwsSagemakerClient:
@@ -331,6 +333,9 @@ def validate_credentials(
for code, message in ERROR_MESSAGES.items():
if error.code.startswith(code):
return False, message
+ status_message = probe_status_message("AWS SageMaker", error.status_code)
+ if status_message:
+ return False, status_message
raise
except requests.RequestException:
return False, "Could not reach AWS SageMaker. Check the region and try again."
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_sagemaker/tests/test_aws_sagemaker.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_sagemaker/tests/test_aws_sagemaker.py
index 966ff19da170..0ad5cb84bb97 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_sagemaker/tests/test_aws_sagemaker.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_sagemaker/tests/test_aws_sagemaker.py
@@ -636,10 +636,14 @@ def test_unknown_table_and_version_are_rejected(config: AwsSagemakerSourceConfig
assert result.requests == []
-def test_transient_validation_error_propagates(config: AwsSagemakerSourceConfig, transport: MagicMock) -> None:
+def test_transient_validation_error_reports_an_unreachable_vendor(
+ config: AwsSagemakerSourceConfig, transport: MagicMock
+) -> None:
transport.post.return_value = response({"__type": "ServiceUnavailable"}, 503)
- with pytest.raises(AwsSagemakerError, match="ServiceUnavailable"):
- validate_credentials(config, "models", "2017-07-24")
+ assert validate_credentials(config, "models", "2017-07-24") == (
+ False,
+ "Couldn't reach AWS SageMaker to check your credentials. Try again in a few minutes.",
+ )
transport.close.assert_called_once()
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_step_functions/aws_step_functions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_step_functions/aws_step_functions.py
index 11967b79decf..3bd7ea731c6b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_step_functions/aws_step_functions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_step_functions/aws_step_functions.py
@@ -25,6 +25,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.aws_step_f
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http.transport import BoundedRetry
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import probe_status_message
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.awsstepfunctions import (
AwsStepFunctionsSourceConfig,
@@ -32,8 +33,9 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.generated_
class AwsStepFunctionsError(Exception):
- def __init__(self, code: str, message: str) -> None:
+ def __init__(self, code: str, message: str, status_code: int | None = None) -> None:
self.code = code
+ self.status_code = status_code
super().__init__(f"AWS Step Functions request failed: {code} - {message}")
@@ -131,9 +133,11 @@ class AwsStepFunctionsClient:
and code in {"ThrottlingException", "Throttling", "TooManyRequestsException", "KmsThrottlingException"}
else AwsStepFunctionsError
)
- raise error_class(code, message)
+ raise error_class(code, message, response.status_code)
if response.status_code != 200:
- raise AwsStepFunctionsError(f"HTTP {response.status_code}", "Unexpected AWS response status.")
+ raise AwsStepFunctionsError(
+ f"HTTP {response.status_code}", "Unexpected AWS response status.", response.status_code
+ )
result = response.json()
if not isinstance(result, dict):
raise ValueError("AWS Step Functions returned an invalid response.")
@@ -399,6 +403,9 @@ def validate_credentials(
return True, None
if error.code in ERROR_MESSAGES:
return False, ERROR_MESSAGES[error.code]
+ status_message = probe_status_message("AWS Step Functions", error.status_code)
+ if status_message:
+ return False, status_message
raise
finally:
client.close()
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_step_functions/tests/test_aws_step_functions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_step_functions/tests/test_aws_step_functions.py
index 56cd163c721e..30611782ddea 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_step_functions/tests/test_aws_step_functions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_step_functions/tests/test_aws_step_functions.py
@@ -358,12 +358,14 @@ def test_body_throttle_retries_with_a_new_signature(config: AwsStepFunctionsSour
@pytest.mark.parametrize("code", ["ServiceUnavailable", "UnknownError"])
-def test_validation_propagates_transient_and_unknown_errors(
+def test_validation_reports_an_unreachable_vendor_for_server_errors(
config: AwsStepFunctionsSourceConfig, session: MagicMock, code: str
) -> None:
session.post.return_value = response(500, {"__type": code})
- with pytest.raises(AwsStepFunctionsError):
- validate_credentials(config, None, API_VERSION)
+ assert validate_credentials(config, None, API_VERSION) == (
+ False,
+ "Couldn't reach AWS Step Functions to check your credentials. Try again in a few minutes.",
+ )
session.close.assert_called_once()
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_waf/aws_waf.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_waf/aws_waf.py
index d343f297ec9c..b0e61b513020 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_waf/aws_waf.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_waf/aws_waf.py
@@ -42,6 +42,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.aws_waf.se
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.datetime_utils import parse_datetime_value
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import probe_status_message
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.awswaf import AwsWafSourceConfig
@@ -55,14 +56,15 @@ class AwsWafResumeConfig:
class AwsWafError(Exception):
- def __init__(self, code: str, message: str) -> None:
+ def __init__(self, code: str, message: str, status_code: int | None = None) -> None:
super().__init__(f"AWS WAF request failed: {code} - {message}")
self.code = code
+ self.status_code = status_code
class AwsWafRetryableError(AwsWafError):
- def __init__(self, code: str, message: str, retry_after: float = 0) -> None:
- super().__init__(code, message)
+ def __init__(self, code: str, message: str, retry_after: float = 0, status_code: int | None = None) -> None:
+ super().__init__(code, message, status_code)
self.retry_after = retry_after
@@ -87,8 +89,8 @@ def error_for_response(response: requests.Response) -> AwsWafError:
retry_after = min(Retry().parse_retry_after(header), 60)
except InvalidHeader:
pass
- return AwsWafRetryableError(code, message, retry_after)
- return AwsWafError(code, message)
+ return AwsWafRetryableError(code, message, retry_after, response.status_code)
+ return AwsWafError(code, message, response.status_code)
def retry_wait(state: RetryCallState) -> float:
@@ -255,6 +257,9 @@ def validate_credentials(
return False, f"Grant {', '.join(others)} and {last} for this table."
if error.code in ERROR_MESSAGES:
return False, ERROR_MESSAGES[error.code]
+ status_message = probe_status_message("AWS WAF", error.status_code)
+ if status_message:
+ return False, status_message
raise
finally:
client.close()
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/axiom/tests/test_axiom.py b/products/warehouse_sources/backend/temporal/data_imports/sources/axiom/tests/test_axiom.py
index e0c74ced01be..0a037f6088dc 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/axiom/tests/test_axiom.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/axiom/tests/test_axiom.py
@@ -10,9 +10,6 @@ from requests_mock import Mocker
from products.warehouse_sources.backend.temporal.data_imports.sources.axiom.axiom import AxiomResumeConfig, axiom_source
from products.warehouse_sources.backend.temporal.data_imports.sources.axiom.source import AxiomSource
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.rest_client import (
- RESTClientRetryableError,
-)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
ScriptedResponse,
@@ -179,12 +176,22 @@ def test_auth_errors_match_validation_and_pipeline(status: int, body: dict[str,
@pytest.mark.parametrize(
- "status,exception", [(404, HTTPError), (429, RESTClientRetryableError), (503, RESTClientRetryableError)]
+ "status,expected",
+ [
+ (404, HTTPError),
+ (429, (False, "Couldn't reach Axiom to check your credentials. Try again in a few minutes.")),
+ (503, (False, "Couldn't reach Axiom to check your credentials. Try again in a few minutes.")),
+ ],
)
-def test_probe_preserves_non_auth_failures(status: int, exception: type[Exception]) -> None:
+def test_probe_preserves_non_auth_failures(status: int, expected: type[Exception] | tuple[bool, str]) -> None:
with scripted_network(always(ScriptedResponse(status=status, json={"code": status}))) as network:
- with pytest.raises(exception):
- AxiomSource().validate_credentials(AxiomSourceConfig(api_token="test-token"), 1)
+ source = AxiomSource()
+ config = AxiomSourceConfig(api_token="test-token")
+ if isinstance(expected, tuple):
+ assert source.validate_credentials(config, 1) == expected
+ else:
+ with pytest.raises(expected):
+ source.validate_credentials(config, 1)
assert {request.path for request in network.requests_log} == {"/v2/datasets"}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/azure_application_insights/tests/test_azure_application_insights_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/azure_application_insights/tests/test_azure_application_insights_source.py
index b984f7b8cc36..d96054062fee 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/azure_application_insights/tests/test_azure_application_insights_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/azure_application_insights/tests/test_azure_application_insights_source.py
@@ -9,9 +9,6 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.azure_appl
PERMISSION_ERROR,
AzureApplicationInsightsSource,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.auth import (
- OAuth2AuthRequestError,
-)
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.azureapplicationinsights import (
AzureApplicationInsightsSourceConfig,
)
@@ -67,9 +64,10 @@ def test_token_error_mapping(
if status in (400, 401):
assert source.validate_credentials(config, 1) == (False, AUTH_ERROR)
else:
- with pytest.raises(OAuth2AuthRequestError) as error:
- source.validate_credentials(config, 1)
- assert not error.value.is_permanent
+ assert source.validate_credentials(config, 1) == (
+ False,
+ "Couldn't reach Microsoft Azure (Application Insights) to check your credentials. Try again in a few minutes.",
+ )
assert http_mock.call_count == 1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/buildium/tests/test_buildium_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/buildium/tests/test_buildium_source.py
index 9bdfdb47f0c9..2680a3452b6e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/buildium/tests/test_buildium_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/buildium/tests/test_buildium_source.py
@@ -1,7 +1,5 @@
import pytest
-from requests.exceptions import HTTPError
-
from products.warehouse_sources.backend.temporal.data_imports.sources.buildium.settings import (
AUTH_ERROR,
PERMISSION_ERROR,
@@ -29,20 +27,21 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.generated_
)
@pytest.mark.parametrize(
("status", "expected"),
- [(200, (True, None)), (401, (False, AUTH_ERROR)), (403, (False, PERMISSION_ERROR)), (500, None)],
+ [
+ (200, (True, None)),
+ (401, (False, AUTH_ERROR)),
+ (403, (False, PERMISSION_ERROR)),
+ (500, (False, "Couldn't reach Buildium (RealPage) to check your credentials. Try again in a few minutes.")),
+ ],
)
def test_validate_credentials(
- schema_name: str | None, expected_path: str, status: int, expected: tuple[bool, str | None] | None
+ schema_name: str | None, expected_path: str, status: int, expected: tuple[bool, str | None]
) -> None:
config = BuildiumSourceConfig(client_id="example-client", client_secret="example-secret")
answer = ScriptedResponse(status=status)
script = always(answer) if status == 500 else [answer]
with scripted_network(script) as network:
- if expected is None:
- with pytest.raises(HTTPError):
- BuildiumSource().validate_credentials(config, 1, schema_name)
- else:
- assert BuildiumSource().validate_credentials(config, 1, schema_name) == expected
+ assert BuildiumSource().validate_credentials(config, 1, schema_name) == expected
assert network.requests_log
if status != 500:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/clarifai/tests/test_clarifai.py b/products/warehouse_sources/backend/temporal/data_imports/sources/clarifai/tests/test_clarifai.py
index c3d0778fc581..fba02f5aa618 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/clarifai/tests/test_clarifai.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/clarifai/tests/test_clarifai.py
@@ -277,11 +277,23 @@ class TestClarifaiCredentials:
assert network.requests_log[0].param("per_page") == "1"
assert network.requests_log[0].headers["authorization"] == "Key fake-clarifai-pat"
- @pytest.mark.parametrize("http_status", [400, 429, 500])
- def test_probe_does_not_disguise_other_errors(self, config: ClarifaiSourceConfig, http_status: int) -> None:
+ @pytest.mark.parametrize(
+ "http_status,expected",
+ [
+ (400, HTTPError),
+ (429, (False, "Couldn't reach Clarifai to check your credentials. Try again in a few minutes.")),
+ (500, (False, "Couldn't reach Clarifai to check your credentials. Try again in a few minutes.")),
+ ],
+ )
+ def test_probe_does_not_disguise_other_errors(
+ self, config: ClarifaiSourceConfig, http_status: int, expected: type[Exception] | tuple[bool, str]
+ ) -> None:
with scripted_network(always(ScriptedResponse(status=http_status, json={"status": {"code": 10020}}))):
- with pytest.raises((HTTPError, RESTClientRetryableError)):
- ClarifaiSource().validate_credentials(config, 42)
+ if isinstance(expected, tuple):
+ assert ClarifaiSource().validate_credentials(config, 42) == expected
+ else:
+ with pytest.raises(expected):
+ ClarifaiSource().validate_credentials(config, 42)
@pytest.mark.parametrize(
"host",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/base.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/base.py
index 10459a18844c..4b8050d13f13 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/base.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/base.py
@@ -1,7 +1,8 @@
import datetime
+import functools
import dataclasses
from abc import ABC, abstractmethod
-from collections.abc import Iterable, Mapping
+from collections.abc import Callable, Iterable, Mapping
from typing import TYPE_CHECKING, Any, Generic, Optional, TypeVar, Union, cast
import structlog
@@ -33,6 +34,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.can
from products.warehouse_sources.backend.temporal.data_imports.sources.common.config import Config
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import SourceSchema
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import (
+ is_unreachable_vendor_error,
+ unreachable_vendor_message,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import (
ResumableData,
SourceInputs,
@@ -116,6 +121,36 @@ def error_message_matches(error_msg: str, patterns: Iterable[str]) -> bool:
return any(pattern.lower() in error_msg_lower for pattern in patterns)
+CredentialProbe = TypeVar("CredentialProbe", bound=Callable[..., tuple[bool, str | None]])
+
+_GUARDED_PROBE = "_reports_unreachable_vendor"
+
+
+def _report_unreachable_vendor(probe: CredentialProbe) -> CredentialProbe:
+ """Wrap a `validate_credentials` override so that an unreachable vendor gives `(False, message)`.
+
+ The API calls the probe while a person creates or edits a source. A raised transport error there
+ shows them a message that blames their credentials. Every other error still raises."""
+
+ @functools.wraps(probe)
+ def guarded(self: "_BaseSource[Any]", *args: Any, **kwargs: Any) -> tuple[bool, str | None]:
+ try:
+ return probe(self, *args, **kwargs)
+ except Exception as error:
+ if not is_unreachable_vendor_error(error):
+ raise
+ # The class name only: a transport error message can repeat a URL that holds a credential.
+ logger.warning(
+ "validate_credentials could not reach the vendor",
+ source_type=str(self.source_type),
+ error_class=type(error).__name__,
+ )
+ return False, unreachable_vendor_message(self._vendor_label())
+
+ setattr(guarded, _GUARDED_PROBE, True)
+ return cast(CredentialProbe, guarded)
+
+
@dataclasses.dataclass(frozen=True)
class VersionDeprecation:
"""Deprecation metadata for a single supported version of a source's vendor API."""
@@ -413,11 +448,28 @@ class _BaseSource(ABC, Generic[ConfigType]):
def webhook_template(self) -> Optional["HogFunctionTemplateDC"]:
return None
+ def __init_subclass__(cls, **kwargs: Any) -> None:
+ super().__init_subclass__(**kwargs)
+ # Every override gets the guard, so a new source cannot forget it.
+ probe = cls.__dict__.get("validate_credentials")
+ if callable(probe) and not getattr(probe, _GUARDED_PROBE, False):
+ cls.validate_credentials = _report_unreachable_vendor(probe) # type: ignore[method-assign]
+
+ def _vendor_label(self) -> str:
+ # The label is for a message only, so a source config that cannot build must not hide the result.
+ try:
+ return self.get_source_config.label or str(self.source_type.value)
+ except Exception:
+ return "the vendor"
+
def validate_credentials(
self, config: ConfigType, team_id: int, schema_name: Optional[str] = None, api_version: str | None = None
) -> tuple[bool, str | None]:
"""Check whether the provided credentials are valid for this source. Returns an optional error message.
+ Every override runs inside a guard that turns an unreachable vendor (a dropped connection, a
+ timeout, a server error) into `(False, message)`, so an override can let those errors raise.
+
``api_version`` follows the `get_schemas` contract: the resolved pin of the source
instance being validated, or ``None`` (→ `default_version`) before a row exists."""
return True, None
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/source_helpers.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/source_helpers.py
index cf437b4ac437..4dc71861e880 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/source_helpers.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/source_helpers.py
@@ -10,6 +10,20 @@ from collections.abc import Callable, Mapping
import requests
from requests.auth import AuthBase
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.auth import (
+ OAuth2AuthRequestError,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.rest_client import (
+ RESTClientRetryableError,
+)
+
+_UNREACHABLE_ERRORS: tuple[type[BaseException], ...] = (
+ requests.ConnectionError,
+ requests.Timeout,
+ requests.exceptions.ChunkedEncodingError,
+ RESTClientRetryableError,
+)
+
def unreachable_vendor_message(vendor: str) -> str:
"""The `validate_credentials` message for a probe that got no usable answer from the vendor.
@@ -18,6 +32,39 @@ def unreachable_vendor_message(vendor: str) -> str:
return f"Couldn't reach {vendor} to check your credentials. Try again in a few minutes."
+def probe_status_message(vendor: str, status_code: int | None) -> str | None:
+ """A fallback message for an HTTP status that a probe has no own message for. `None` means that the
+ status says nothing about the credentials, so the probe must decide."""
+ if status_code == 401:
+ return f"{vendor} rejected the credentials. Check them and try again."
+ if status_code == 403:
+ return f"{vendor} denied access. Check that the credentials have the permissions this source needs."
+ if status_code is not None and (status_code == 429 or status_code >= 500):
+ return unreachable_vendor_message(vendor)
+ return None
+
+
+def is_unreachable_vendor_error(error: BaseException) -> bool:
+ """Whether a probe error means only that the vendor gave no usable answer: a dropped connection, a
+ timeout, a server error or a rate limit. Follows `raise ... from` causes, because many sources wrap
+ the transport error in their own. Also follows the implicit context of an error raised inside an
+ `except` block without `from`, unless `from None` suppressed it."""
+ seen: set[int] = set()
+ current: BaseException | None = error
+ while current is not None and id(current) not in seen:
+ seen.add(id(current))
+ if isinstance(current, _UNREACHABLE_ERRORS):
+ return True
+ if isinstance(current, requests.HTTPError) and current.response is not None:
+ status = current.response.status_code
+ if status == 429 or status >= 500:
+ return True
+ if isinstance(current, OAuth2AuthRequestError) and not current.is_permanent:
+ return True
+ current = current.__cause__ or (None if current.__suppress_context__ else current.__context__)
+ return False
+
+
def validate_via_probe(
session_factory: Callable[[], requests.Session],
url: str,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/tests/test_source_helpers.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/tests/test_source_helpers.py
new file mode 100644
index 000000000000..3c2410298ccc
--- /dev/null
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/tests/test_source_helpers.py
@@ -0,0 +1,91 @@
+from typing import cast
+
+import pytest
+
+import requests
+
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.auth import (
+ OAuth2AuthRequestError,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.rest_client import (
+ RESTClientRetryableError,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import (
+ is_unreachable_vendor_error,
+ probe_status_message,
+)
+
+
+def _http_error(status: int) -> requests.HTTPError:
+ response = requests.Response()
+ response.status_code = status
+ return requests.HTTPError(f"HTTP {status}", response=response)
+
+
+def _wrapped(cause: BaseException) -> ValueError:
+ try:
+ raise ValueError("vendor client failed") from cause
+ except ValueError as error:
+ return error
+
+
+def _implicitly_wrapped(context: BaseException) -> ValueError:
+ try:
+ try:
+ raise context
+ except type(context):
+ raise ValueError("vendor client failed") # noqa: B904 — the implicit context is the point
+ except ValueError as error:
+ return error
+
+
+def _suppressed(context: BaseException) -> ValueError:
+ try:
+ try:
+ raise context
+ except type(context):
+ raise ValueError("vendor client failed") from None
+ except ValueError as error:
+ return error
+
+
+ "error,expected",
+ [
+ (requests.ConnectionError("refused"), True),
+ (requests.Timeout("timed out"), True),
+ (requests.exceptions.ChunkedEncodingError("cut"), True),
+ (RESTClientRetryableError("HTTP 503"), True),
+ (_http_error(500), True),
+ (_http_error(429), True),
+ (OAuth2AuthRequestError("HTTP 502 from the token endpoint"), True),
+ (_wrapped(requests.ConnectionError("refused")), True),
+ (_implicitly_wrapped(requests.ConnectionError("refused")), True),
+ (_http_error(401), False),
+ (_http_error(403), False),
+ (_http_error(404), False),
+ (requests.HTTPError("no response", response=cast(requests.Response, None)), False),
+ (OAuth2AuthRequestError("invalid_client", is_permanent=True), False),
+ (_wrapped(_http_error(400)), False),
+ (_implicitly_wrapped(_http_error(400)), False),
+ (_suppressed(requests.ConnectionError("refused")), False),
+ (ValueError("bad config"), False),
+ ],
+)
+def test_is_unreachable_vendor_error(error: BaseException, expected: bool) -> None:
+ assert is_unreachable_vendor_error(error) is expected
+
+
+ "status,expected",
+ [
+ (401, "Acme rejected the credentials. Check them and try again."),
+ (403, "Acme denied access. Check that the credentials have the permissions this source needs."),
+ (429, "Couldn't reach Acme to check your credentials. Try again in a few minutes."),
+ (503, "Couldn't reach Acme to check your credentials. Try again in a few minutes."),
+ (400, None),
+ (None, None),
+ ],
+)
+def test_probe_status_message(status: int | None, expected: str | None) -> None:
+ assert probe_status_message("Acme", status) == expected
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/crisp/tests/test_crisp_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/crisp/tests/test_crisp_source.py
index d264ef0704ff..8ad2a6f0722f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/crisp/tests/test_crisp_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/crisp/tests/test_crisp_source.py
@@ -48,8 +48,10 @@ def test_credential_probe_does_not_wait_out_a_rate_limit_like_a_sync(config: Cri
with scripted_network(
always(ScriptedResponse(status=429, json={"error": True, "reason": "rate_limited"}))
) as network:
- with pytest.raises(Exception):
- CrispSource().validate_credentials(config, team_id=1)
+ assert CrispSource().validate_credentials(config, team_id=1) == (
+ False,
+ "Couldn't reach Crisp to check your credentials. Try again in a few minutes.",
+ )
assert len(network.requests_log) <= 5
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/criteo/tests/test_criteo_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/criteo/tests/test_criteo_source.py
index f283bc99f968..4a9ada449dc0 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/criteo/tests/test_criteo_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/criteo/tests/test_criteo_source.py
@@ -3,12 +3,6 @@ from typing import Any
import pytest
import time_machine
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.auth import (
- OAuth2AuthRequestError,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.rest_client import (
- RESTClientRetryableError,
-)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
ScriptedResponse,
SourceDriver,
@@ -92,11 +86,11 @@ def test_transient_probe_failures_do_not_mark_credentials_invalid(schema_name: s
ScriptedResponse(status=503, json={"errors": []}),
]
)
- with (
- scripted_network(script),
- pytest.raises(OAuth2AuthRequestError if schema_name is None else RESTClientRetryableError),
- ):
- CriteoSource().validate_credentials(config(), 1, schema_name)
+ with scripted_network(script):
+ assert CriteoSource().validate_credentials(config(), 1, schema_name) == (
+ False,
+ "Couldn't reach Criteo to check your credentials. Try again in a few minutes.",
+ )
@pytest.mark.parametrize(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/frontegg/tests/test_frontegg_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/frontegg/tests/test_frontegg_source.py
index fb570224d226..414c14c98026 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/frontegg/tests/test_frontegg_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/frontegg/tests/test_frontegg_source.py
@@ -2,8 +2,6 @@ from typing import Any, cast
import pytest
-from requests import HTTPError
-
from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
ScriptedResponse,
always,
@@ -26,7 +24,11 @@ class TestFronteggSource:
@pytest.mark.parametrize(
"status,expected",
- [(400, AUTH_ERROR), (429, "raise"), (500, "raise")],
+ [
+ (400, AUTH_ERROR),
+ (429, "Couldn't reach Frontegg to check your credentials. Try again in a few minutes."),
+ (500, "Couldn't reach Frontegg to check your credentials. Try again in a few minutes."),
+ ],
)
def test_validate_credentials(self, status: int, expected: str | None) -> None:
config = FronteggSourceConfig(client_id="example-client", api_key="fake-key", region="EU")
@@ -37,11 +39,7 @@ class TestFronteggSource:
else ScriptedResponse(status=status, json={})
)
with scripted_network(always(answer)) as network:
- if expected == "raise":
- with pytest.raises(HTTPError):
- source.validate_credentials(config, 1)
- else:
- assert source.validate_credentials(config, 1) == (status == 200, expected)
+ assert source.validate_credentials(config, 1) == (status == 200, expected)
assert {request.path for request in network.requests_log} == {"/auth/vendor"}
mappings = source.get_non_retryable_errors()
assert any(pattern in f"{status} Client Error" for pattern in mappings) == (status in (400, 401, 403))
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/growthbook/tests/test_growthbook_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/growthbook/tests/test_growthbook_source.py
index 5fd481e34421..6e302dea2961 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/growthbook/tests/test_growthbook_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/growthbook/tests/test_growthbook_source.py
@@ -5,9 +5,6 @@ from unittest.mock import MagicMock
import responses
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.rest_client import (
- RESTClientRetryableError,
-)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import UnknownResourceError
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.growthbook import (
@@ -54,14 +51,16 @@ def test_credential_probe_status_mapping(
@pytest.mark.parametrize("status", [429, 503])
-def test_transient_probe_failure_is_not_bad_credentials(
+def test_transient_probe_failure_reports_an_unreachable_vendor(
config: GrowthBookSourceConfig,
http: responses.RequestsMock,
status: int,
) -> None:
http.add(responses.GET, "https://api.growthbook.io/api/v2/features", status=status, json={"message": "Try again"})
- with pytest.raises(RESTClientRetryableError):
- GrowthBookSource().validate_credentials(config, 123)
+ assert GrowthBookSource().validate_credentials(config, 123) == (
+ False,
+ "Couldn't reach GrowthBook to check your credentials. Try again in a few minutes.",
+ )
@pytest.mark.parametrize(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/harness/tests/test_harness.py b/products/warehouse_sources/backend/temporal/data_imports/sources/harness/tests/test_harness.py
index f2317f2d6d0b..9e58bcd36536 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/harness/tests/test_harness.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/harness/tests/test_harness.py
@@ -10,9 +10,6 @@ from unittest.mock import MagicMock, patch
from requests import Response
from requests.exceptions import HTTPError
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.rest_client import (
- RESTClientRetryableError,
-)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
ScriptedResponse,
@@ -189,10 +186,12 @@ def test_sync_auth_errors_are_terminal(driver: SourceDriver, status: int) -> Non
@pytest.mark.parametrize("status", [429, 500, 503])
-def test_transient_probe_errors_propagate(config: HarnessSourceConfig, status: int) -> None:
+def test_transient_probe_errors_report_an_unreachable_vendor(config: HarnessSourceConfig, status: int) -> None:
with scripted_network(always(ScriptedResponse(status=status, json={}))) as network:
- with pytest.raises(RESTClientRetryableError):
- HarnessSource().validate_credentials(config, 1)
+ assert HarnessSource().validate_credentials(config, 1) == (
+ False,
+ "Couldn't reach Harness to check your credentials. Try again in a few minutes.",
+ )
assert len(network.requests_log) == 5
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/infisical/infisical.py b/products/warehouse_sources/backend/temporal/data_imports/sources/infisical/infisical.py
index 21da5015d66c..a6d69fe1097c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/infisical/infisical.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/infisical/infisical.py
@@ -17,6 +17,9 @@ from tenacity import RetryCallState, retry, retry_if_exception_type, stop_after_
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
from products.warehouse_sources.backend.temporal.data_imports.sources.common.mixins import _is_host_safe
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import (
+ unreachable_vendor_message,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.infisical.settings import (
INFISICAL_ENDPOINTS,
@@ -443,6 +446,8 @@ def validate_credentials(
return False, str(exc)
except InfisicalHostNotAllowedError:
return False, HOST_NOT_ALLOWED_ERROR
+ except InfisicalRetryableError:
+ return False, unreachable_vendor_message("Infisical")
except requests.RequestException as exc:
return False, str(exc)
@@ -473,6 +478,8 @@ def validate_credentials(
return False, str(exc)
except InfisicalHostNotAllowedError:
return False, HOST_NOT_ALLOWED_ERROR
+ except InfisicalRetryableError:
+ return False, unreachable_vendor_message("Infisical")
except requests.RequestException as exc:
return False, str(exc)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mercado_pago/tests/test_mercado_pago.py b/products/warehouse_sources/backend/temporal/data_imports/sources/mercado_pago/tests/test_mercado_pago.py
index 90162cb8907c..3a70ec15d779 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mercado_pago/tests/test_mercado_pago.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/mercado_pago/tests/test_mercado_pago.py
@@ -10,9 +10,6 @@ from unittest.mock import MagicMock
import responses
from requests.exceptions import HTTPError
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.rest_client import (
- RESTClientRetryableError,
-)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
ScriptedResponse,
@@ -186,16 +183,38 @@ def test_auth_errors_are_actionable(status: int) -> None:
assert len(network.requests_log) + len(result.requests) == 2
[email protected]("status", [200, 400, 429, 503])
-def test_credential_validation_uses_one_request(status: int) -> None:
+ "status,expected",
+ [
+ (200, (True, None)),
+ (400, HTTPError),
+ (
+ 429,
+ (
+ False,
+ "Couldn't reach Mercado Pago (Mercado Libre) to check your credentials. Try again in a few minutes.",
+ ),
+ ),
+ (
+ 503,
+ (
+ False,
+ "Couldn't reach Mercado Pago (Mercado Libre) to check your credentials. Try again in a few minutes.",
+ ),
+ ),
+ ],
+)
+def test_credential_validation_uses_one_request(
+ status: int, expected: type[Exception] | tuple[bool, str | None]
+) -> None:
body: dict[str, Any] = {"results": [], "paging": {"total": 0}} if status == 200 else {"error": "test_error"}
source = MercadoPagoSource()
config = MercadoPagoSourceConfig(access_token=TOKEN)
with scripted_network([ScriptedResponse(status=status, json=body)]) as network:
- if status == 200:
- assert source.validate_credentials(config, team_id=1) == (True, None)
+ if isinstance(expected, tuple):
+ assert source.validate_credentials(config, team_id=1) == expected
else:
- with pytest.raises(RESTClientRetryableError if status >= 429 else HTTPError):
+ with pytest.raises(expected):
source.validate_credentials(config, team_id=1)
assert len(network.requests_log) == 1
request = network.requests_log[0]
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/onehundredms/tests/test_onehundredms_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/onehundredms/tests/test_onehundredms_source.py
index 5a35ecbda365..958de52d7ca3 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/onehundredms/tests/test_onehundredms_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/onehundredms/tests/test_onehundredms_source.py
@@ -3,9 +3,6 @@ import pytest
from requests.exceptions import HTTPError
from requests_mock import Mocker
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.rest_client import (
- RESTClientRetryableError,
-)
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.onehundredms import (
OneHundredMsSourceConfig,
)
@@ -43,9 +40,11 @@ def test_credentials_probe_and_error_mapping(
@pytest.mark.parametrize("status", [429, 500])
-def test_transient_probe_errors_are_not_invalid_credentials(requests_mock: Mocker, status: int) -> None:
+def test_transient_probe_errors_report_an_unreachable_vendor(requests_mock: Mocker, status: int) -> None:
config = OneHundredMsSourceConfig(app_access_key="example-key", app_secret="example-secret-for-tests-only-12345")
requests_mock.get("https://api.100ms.live/v2/sessions", status_code=status, json={"code": status})
- with pytest.raises(RESTClientRetryableError):
- OneHundredMsSource().validate_credentials(config, 1)
+ assert OneHundredMsSource().validate_credentials(config, 1) == (
+ False,
+ "Couldn't reach 100ms to check your credentials. Try again in a few minutes.",
+ )
assert requests_mock.call_count == 1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/retell_ai/tests/test_retell_ai_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/retell_ai/tests/test_retell_ai_source.py
index f6b54258e96b..93a7245083dd 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/retell_ai/tests/test_retell_ai_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/retell_ai/tests/test_retell_ai_source.py
@@ -6,7 +6,6 @@ from unittest.mock import MagicMock
from requests.exceptions import HTTPError
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import RESTClientRetryableError
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.retellai import (
RetellAISourceConfig,
)
@@ -46,11 +45,26 @@ def test_malformed_api_keys_fail_before_http(key: str, http: MagicMock) -> None:
http.assert_not_called()
[email protected]("status", [400, 429, 500])
+ "status,expected",
+ [
+ (400, HTTPError),
+ (429, (False, "Couldn't reach Retell AI to check your credentials. Try again in a few minutes.")),
+ (500, (False, "Couldn't reach Retell AI to check your credentials. Try again in a few minutes.")),
+ ],
+)
def test_probe_errors_are_not_reported_as_invalid_credentials(
- status: int, http: MagicMock, monkeypatch: pytest.MonkeyPatch
+ status: int,
+ expected: type[Exception] | tuple[bool, str],
+ http: MagicMock,
+ monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr("time.sleep", lambda _: None)
http.return_value = response({}, status)
- with pytest.raises(HTTPError if status == 400 else RESTClientRetryableError):
- RetellAISource().validate_credentials(RetellAISourceConfig(api_key="fake-key"), 1)
+ source = RetellAISource()
+ config = RetellAISourceConfig(api_key="fake-key")
+ if isinstance(expected, tuple):
+ assert source.validate_credentials(config, 1) == expected
+ else:
+ with pytest.raises(expected):
+ source.validate_credentials(config, 1)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/sage_intacct/tests/test_sage_intacct.py b/products/warehouse_sources/backend/temporal/data_imports/sources/sage_intacct/tests/test_sage_intacct.py
index 9603b1c7f455..630fa619b680 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/sage_intacct/tests/test_sage_intacct.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/sage_intacct/tests/test_sage_intacct.py
@@ -5,9 +5,6 @@ import pytest
from requests.exceptions import HTTPError
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.auth import (
- OAuth2AuthRequestError,
-)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import UnknownResourceError
from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
ScriptedResponse,
@@ -159,13 +156,13 @@ def test_credential_probe_success() -> None:
@pytest.mark.parametrize("status", [400, 503])
def test_token_failure_distinguishes_permanent_from_transient(status: int) -> None:
with scripted_network([ScriptedResponse(status=status, json={"error": "invalid_client"})]):
- if status == 503:
- with pytest.raises(OAuth2AuthRequestError):
- SageIntacctSource().validate_credentials(config(), 1)
- else:
- valid, message = SageIntacctSource().validate_credentials(config(), 1)
- assert valid is False
- assert message == "Check your Sage Intacct client credentials and web services username."
+ valid, message = SageIntacctSource().validate_credentials(config(), 1)
+ assert valid is False
+ assert message == (
+ "Couldn't reach Sage Intacct to check your credentials. Try again in a few minutes."
+ if status == 503
+ else "Check your Sage Intacct client credentials and web services username."
+ )
@pytest.mark.parametrize(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/sanity/tests/test_sanity_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/sanity/tests/test_sanity_source.py
index 41e0f1781082..26da0022fa92 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/sanity/tests/test_sanity_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/sanity/tests/test_sanity_source.py
@@ -2,8 +2,6 @@ import json
import pytest
-from requests.exceptions import HTTPError
-
from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
ScriptedResponse,
scripted_network,
@@ -51,8 +49,10 @@ def test_credentials_map_auth_errors_and_check_the_selected_type(
def test_credentials_do_not_label_a_server_failure_as_invalid_credentials() -> None:
with scripted_network(always(ScriptedResponse(status=500, json={"error": "Internal Server Error"}))):
- with pytest.raises(HTTPError):
- SanitySource().validate_credentials(config(), team_id=1)
+ assert SanitySource().validate_credentials(config(), team_id=1) == (
+ False,
+ "Couldn't reach Sanity to check your credentials. Try again in a few minutes.",
+ )
@pytest.mark.parametrize(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/survicate/tests/test_survicate.py b/products/warehouse_sources/backend/temporal/data_imports/sources/survicate/tests/test_survicate.py
index 256fada01c4c..53a03b97c7cf 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/survicate/tests/test_survicate.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/survicate/tests/test_survicate.py
@@ -7,9 +7,6 @@ import pytest
from requests import HTTPError
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.rest_client import (
- RESTClientRetryableError,
-)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import UnknownResourceError
from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
DriveResult,
@@ -242,8 +239,10 @@ def test_credential_probe_and_error_mapping(status: int) -> None:
config = SurvicateSourceConfig(api_key="fake-api-key")
with scripted_network(always(response)) as network:
if status in (429, 500):
- with pytest.raises(RESTClientRetryableError):
- source.validate_credentials(config, 1)
+ assert source.validate_credentials(config, 1) == (
+ False,
+ "Couldn't reach Survicate to check your credentials. Try again in a few minutes.",
+ )
elif status == 404:
with pytest.raises(HTTPError):
source.validate_credentials(config, 1)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_credential_contract.py b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_credential_contract.py
index 684969311814..e766e98038e9 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_credential_contract.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_credential_contract.py
@@ -465,145 +465,7 @@ OK_NEEDS_VENDOR_BODY: set[str] = {
# Sources whose probe raises for these answers. Each entry is a defect to fix: the probe must return
# `(False, message)`. The value lists the answers that raise.
RAISES: dict[str, str] = {
- "AbnormalSecurity": "500 connection",
- "Acast": "500 connection",
- "Acculynx": "500 connection",
- "Ahrefs": "500 connection",
- "Alegra": "500 connection",
- "AmazonS3": "401 500 connection",
- "Appcues": "500 connection",
- "Arcade": "500 connection",
- "Asaas": "connection",
- "Astronomer": "500 connection",
- "Autumn": "connection",
- "AwsGuardduty": "401 500 connection",
- "AwsInspector": "500 connection",
- "AwsMacie": "401 403 500 connection",
- "AwsSagemaker": "401 403 500",
- "AwsStepFunctions": "401 403 500 connection",
- "AwsWaf": "401 403 500 connection",
- "Axiom": "500 connection",
- "AzureApplicationInsights": "500 connection",
- "Billomat": "connection",
- "Bluesky": "connection",
- "Braintrust": "500 connection",
- "BrowseAI": "500 connection",
- "Buildium": "500 connection",
- "BuyMeACoffee": "500 connection",
- "CastAi": "connection",
- "Catchpoint": "500 connection",
- "Chargebee": "connection",
- "Chargify": "connection",
- "Checkly": "500 connection",
- "CiscoMeraki": "500 connection",
- "Clarifai": "500 connection",
- "Clevertap": "500 connection",
- "Cliniko": "connection",
- "Clip": "500 connection",
- "Cloudability": "connection",
- "Cloudsmith": "connection",
- "Codemagic": "connection",
- "Crisp": "500 connection",
- "Descope": "connection",
- "Develocity": "500 connection",
- "Donorbox": "500 connection",
- "Embrace": "500 connection",
- "Fieldpulse": "500 connection",
- "Fintoc": "500 connection",
- "FirstPromoter": "connection",
- "Fourthwall": "connection",
- "Frontegg": "500 connection",
- "Gcore": "500 connection",
- "Givebutter": "500 connection",
- "Gleif": "connection",
- "GoLogin": "500 connection",
- "GrowthBook": "500 connection",
- "Harness": "500 connection",
- "HeyGen": "500 connection",
- "Hightouch": "connection",
- "Humanitec": "500 connection",
- "Imperva": "500 connection",
- "Infisical": "500",
- "KapaAI": "500 connection",
- "Kickscale": "connection",
- "Klarna": "500 connection",
- "Knock": "connection",
- "Kommo": "connection",
- "Komodor": "500 connection",
- "LexwareOffice": "500 connection",
- "Lightfield": "connection",
- "Lodgify": "500 connection",
- "Logrocket": "500 connection",
- "Maxio": "connection",
- "MercadoPago": "500 connection",
- "Mezmo": "500 connection",
- "MicrosoftDefenderCloudApps": "500 connection",
- "Mintlify": "500 connection",
- "Mono": "500 connection",
- "NoaaCdo": "500 connection",
- "OPUSWatch": "connection",
- "OneHundredMs": "500 connection",
- "Orb": "500 connection",
- "Paymongo": "500 connection",
- "Paystack": "connection",
- "PeecAI": "500 connection",
- "Pinecone": "500 connection",
- "Postscript": "connection",
- "Productive": "500 connection",
- "PromptWatch": "500 connection",
- "PromptingCompany": "500 connection",
- "ProofpointTap": "500 connection",
- "Qdrant": "500 connection",
- "Qonto": "500 connection",
- "Razorpay": "connection",
- "ReferralHero": "500 connection",
- "RetellAI": "500 connection",
- "RevolutMerchant": "500 connection",
- "SageIntacct": "500 connection",
- "Sanity": "500 connection",
- "Scalr": "500 connection",
- "Scrunch": "500 connection",
- "Semaphore": "500 connection",
- "Sequenzy": "connection",
- "Servicem8": "connection",
- "SevenShifts": "500 connection",
- "Sim": "500 connection",
- "Slash": "500 connection",
- "Speedcurve": "500 connection",
- "SpotIo": "connection",
- "SpotlerCRM": "connection",
- "Sprig": "500 connection",
- "Statsig": "500 connection",
- "Superwall": "500 connection",
- "Survicate": "500 connection",
- "Synthesia": "500 connection",
- "Systeme": "500 connection",
- "Teachable": "connection",
- "TeamupFitness": "500 connection",
- "Telli": "500 connection",
- "Telnyx": "connection",
- "TestDino": "500 connection",
- "Thousandeyes": "500 connection",
- "Ticketmaster": "500 connection",
- "TripleWhale": "500 connection",
- "Trustradius": "500 connection",
- "Turso": "500 connection",
- "Twitter": "connection",
- "TyntecSMS": "connection",
- "Umami": "500 connection",
- "UsTreasuryFiscalData": "500 connection",
- "Vimeo": "500 connection",
- "Vitally": "connection",
- "Wasabi": "connection",
- "WhoGho": "connection",
- "WorldBank": "connection",
- "Xsolla": "connection",
- "Ynab": "500 connection",
- "Yoco": "connection",
- "YoutubeData": "500 connection",
- "Zendesk": "connection",
- "ZendeskSunshine": "connection",
- "Zuora": "connection",
+ "AmazonS3": "401 500",
}
# Sources whose probe returns no message for these answers. Each entry is a defect to fix: the person
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/ticketmaster/tests/test_ticketmaster.py b/products/warehouse_sources/backend/temporal/data_imports/sources/ticketmaster/tests/test_ticketmaster.py
index 7f4f4161eb3b..0d2fdf58b7bc 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/ticketmaster/tests/test_ticketmaster.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/ticketmaster/tests/test_ticketmaster.py
@@ -2,8 +2,6 @@ from typing import Any
import pytest
-from requests.exceptions import HTTPError, RequestException
-
from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import UnknownResourceError
from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
DriveResult,
@@ -21,6 +19,8 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.ticketmast
validate_credentials,
)
+UNREACHABLE_MESSAGE = "Couldn't reach Ticketmaster to check your credentials. Try again in a few minutes."
+
def make_config(keyword: str = " Example act ") -> TicketmasterSourceConfig:
return TicketmasterSourceConfig(api_key="fake-ticketmaster-key", keyword=keyword)
@@ -102,20 +102,17 @@ def test_client_errors_hide_query_key(status: int) -> None:
assert len(result.requests) == 1
[email protected]("status", [429, 503])
-def test_credential_probe(status: int) -> None:
+ "status,expected_message",
+ [
+ (429, UNREACHABLE_MESSAGE),
+ (503, UNREACHABLE_MESSAGE),
+ (404, "Ticketmaster request failed (HTTP 404)."),
+ ],
+)
+def test_credential_probe(status: int, expected_message: str) -> None:
with scripted_network(always(ScriptedResponse(status=status, json={}))) as network:
- if status in (429, 503):
- with pytest.raises(HTTPError, match=f"HTTP {status}") as error:
- validate_credentials(make_config(), "venues", "v2")
- assert "fake-ticketmaster-key" not in str(error.value)
- else:
- valid, message = validate_credentials(make_config(), "venues", "v2")
- assert valid is (status == 200)
- if status == 200:
- assert message is None
- else:
- assert message in TicketmasterSource().get_non_retryable_errors()
+ assert validate_credentials(make_config(), "venues", "v2") == (False, expected_message)
assert {request.path for request in network.requests_log} == {"/discovery/v2/venues.json"}
for request in network.requests_log:
assert request.query == {
@@ -148,11 +145,6 @@ def test_unknown_endpoint_rejected_before_request() -> None:
@pytest.mark.parametrize("error", [ConnectionResetError("reset"), TimeoutError("timed out")])
-def test_credential_connection_error_hides_query_key(error: BaseException) -> None:
- with (
- scripted_network(always(error)),
- pytest.raises(RequestException, match="Ticketmaster connection failed") as raised,
- ):
- validate_credentials(make_config(), "events", "v2")
- assert "fake-ticketmaster-key" not in str(raised.value)
- assert raised.value.__suppress_context__
+def test_credential_connection_error_gives_a_message(error: BaseException) -> None:
+ with scripted_network(always(error)):
+ assert validate_credentials(make_config(), "events", "v2") == (False, UNREACHABLE_MESSAGE)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/ticketmaster/ticketmaster.py b/products/warehouse_sources/backend/temporal/data_imports/sources/ticketmaster/ticketmaster.py
index e42f02f6b87c..c6c5bbfbcb12 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/ticketmaster/ticketmaster.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/ticketmaster/ticketmaster.py
@@ -1,7 +1,7 @@
from typing import Any
from requests import Response
-from requests.exceptions import HTTPError, RequestException
+from requests.exceptions import RequestException
from posthog.dataclasses import frozen
@@ -20,6 +20,9 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import schema_for_resource
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import (
+ unreachable_vendor_message,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.ticketmaster import (
TicketmasterSourceConfig,
@@ -63,6 +66,9 @@ def validate_credentials(config: TicketmasterSourceConfig, endpoint: str, api_ve
if not config.keyword.strip():
return False, KEYWORD_ERROR
params: dict[str, str | int] = {"size": 1, "keyword": config.keyword.strip()}
+ # The API key travels in the query string, and a `requests` error repeats the full URL, so the
+ # probe returns a message instead of the error.
+ response: Response | None = None
try:
with make_tracked_session(redact_values=(config.api_key,)) as session:
response = session.get(
@@ -72,12 +78,14 @@ def validate_credentials(config: TicketmasterSourceConfig, endpoint: str, api_ve
timeout=30,
allow_redirects=False,
)
- except RequestException as error:
- raise RequestException(f"Ticketmaster connection failed ({type(error).__name__}). Try again.") from None
+ except RequestException:
+ pass
+ if response is None or response.status_code == 429 or response.status_code >= 500:
+ return False, unreachable_vendor_message("Ticketmaster")
if response.status_code in AUTH_ERRORS:
return False, AUTH_ERRORS[response.status_code]
if response.status_code != 200:
- raise HTTPError(f"Ticketmaster request failed (HTTP {response.status_code}). Try again.", response=response)
+ return False, f"Ticketmaster request failed (HTTP {response.status_code})."
return True, None
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/vimeo/tests/test_vimeo_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/vimeo/tests/test_vimeo_source.py
index 90a54b311244..799a22332b97 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/vimeo/tests/test_vimeo_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/vimeo/tests/test_vimeo_source.py
@@ -1,7 +1,5 @@
import pytest
-from requests import HTTPError
-
from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
ScriptedResponse,
always,
@@ -40,5 +38,8 @@ def test_credentials_probe(status: int, schema: str | None, path: str, expected:
def test_service_error_is_not_reported_as_invalid_credentials() -> None:
- with scripted_network(always(ScriptedResponse(status=503, json={}))), pytest.raises(HTTPError):
- VimeoSource().validate_credentials(VimeoSourceConfig(access_token="test-token"), 1)
+ with scripted_network(always(ScriptedResponse(status=503, json={}))):
+ assert VimeoSource().validate_credentials(VimeoSourceConfig(access_token="test-token"), 1) == (
+ False,
+ "Couldn't reach Vimeo to check your credentials. Try again in a few minutes.",
+ )
diff --git a/products/warehouse_sources/backend/tests/api/test_external_data_schema.py b/products/warehouse_sources/backend/tests/api/test_external_data_schema.py
index c4a84ae7921d..a2b3fdb4900b 100644
--- a/products/warehouse_sources/backend/tests/api/test_external_data_schema.py
+++ b/products/warehouse_sources/backend/tests/api/test_external_data_schema.py
@@ -43,6 +43,9 @@ from products.warehouse_sources.backend.facade.models import (
)
from products.warehouse_sources.backend.facade.types import ExternalDataSourceType
from products.warehouse_sources.backend.presentation.views.external_data_schema import schema_display_status
+from products.warehouse_sources.backend.presentation.views.external_data_source.helpers import (
+ INVALID_CREDENTIALS_FALLBACK_MESSAGE,
+)
from products.warehouse_sources.backend.temporal.data_imports.cdc.repair import _repair_lock_key
from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
VersionDeprecation,
@@ -166,6 +169,28 @@ class TestExternalDataSchema(APIBaseTest):
"primary_key_detection_supported": False,
}
+ def test_incremental_fields_handles_a_probe_that_raises(self):
+ source = ExternalDataSource.objects.create(
+ team=self.team,
+ source_type=ExternalDataSourceType.STRIPE,
+ job_inputs={"auth_method": {"selection": "api_key", "stripe_secret_key": "123"}},
+ )
+ schema = ExternalDataSchema.objects.create(
+ name="BalanceTransaction",
+ team=self.team,
+ source=source,
+ should_sync=True,
+ status=ExternalDataSchema.Status.COMPLETED,
+ sync_type=ExternalDataSchema.SyncType.FULL_REFRESH,
+ )
+ with mock.patch.object(StripeSource, "validate_credentials", side_effect=RuntimeError("probe exploded")):
+ response = self.client.post(
+ f"/api/environments/{self.team.pk}/external_data_schemas/{schema.id}/incremental_fields",
+ )
+
+ assert response.status_code == 400
+ assert response.json()["message"] == INVALID_CREDENTIALS_FALLBACK_MESSAGE
+
@parameterized.expand(
[
("non_retryable_source_error", Exception("Invalid API Key provided"), False),