Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-warehouse): keep api keys out of credential probe errors (#120454)

Tom Owers10e4218b671e
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/source_helpers.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/source_helpers.py
index 99e48bd4d90a..cf437b4ac437 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/source_helpers.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/source_helpers.py
@@ -11,6 +11,13 @@ import requests
from requests.auth import AuthBase
+def unreachable_vendor_message(vendor: str) -> str:
+ """The `validate_credentials` message for a probe that got no usable answer from the vendor.
+
+ It must not say that the credentials are invalid, because nothing proved that."""
+ return f"Couldn't reach {vendor} to check your credentials. Try again in a few minutes."
+
+
def validate_via_probe(
session_factory: Callable[[], requests.Session],
url: str,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/confluence/confluence.py b/products/warehouse_sources/backend/temporal/data_imports/sources/confluence/confluence.py
index be2ced1673e3..d8e8a670d925 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/confluence/confluence.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/confluence/confluence.py
@@ -19,7 +19,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.resource import Resource
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import ClientConfig
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import validate_via_probe
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import (
+ unreachable_vendor_message,
+ validate_via_probe,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.confluence.settings import (
CONFLUENCE_ENDPOINTS,
@@ -237,6 +240,6 @@ def validate_credentials(
return True, None
return False, "Your Confluence account does not have permission to access this resource."
if status is None:
- return False, None
+ return False, unreachable_vendor_message("Confluence")
return False, f"Confluence API returned status {status}."
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/confluence/tests/test_confluence.py b/products/warehouse_sources/backend/temporal/data_imports/sources/confluence/tests/test_confluence.py
index 52f0f934c68c..c33193fbe2be 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/confluence/tests/test_confluence.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/confluence/tests/test_confluence.py
@@ -91,7 +91,7 @@ class TestValidateCredentials:
with scripted_network(always(ConnectionError("boom"))) as network:
is_valid, message = validate_credentials("acme", "[email protected]", "token")
assert is_valid is False
- assert message is None
+ assert message == "Couldn't reach Confluence to check your credentials. Try again in a few minutes."
assert {request.path for request in network.requests_log} == {"/wiki/api/v2/spaces"}
@parameterized.expand(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openalex/openalex.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openalex/openalex.py
index 3378321fa91a..0260f5f647cc 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openalex/openalex.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openalex/openalex.py
@@ -2,6 +2,8 @@ import datetime
import dataclasses
from typing import Any, Optional
+import requests
+
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (
RESTAPIConfig,
@@ -12,6 +14,9 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import EndpointResource
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import (
+ unreachable_vendor_message,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.openalex.settings import (
OPENALEX_ENDPOINTS,
@@ -199,6 +204,15 @@ def openalex_source(
)
+def _probe(session: requests.Session, path: str, params: dict[str, str]) -> Optional[requests.Response]:
+ # The API key travels in the query string, and a `requests` error repeats the full URL in its
+ # message. Returning `None` keeps that message, and so the key, out of the caller's error report.
+ try:
+ return session.get(f"{OPENALEX_BASE_URL}{path}", params=params, timeout=REQUEST_TIMEOUT_SECONDS)
+ except requests.RequestException:
+ return None
+
+
def validate_credentials(api_key: str, filters: dict[str, Optional[str]]) -> tuple[bool, Optional[str]]:
"""Probe the API key, then each configured filter expression.
@@ -210,11 +224,9 @@ def validate_credentials(api_key: str, filters: dict[str, Optional[str]]) -> tup
probe_params: dict[str, str] = {"per_page": "1", "api_key": api_key}
- response = session.get(
- f"{OPENALEX_BASE_URL}/domains",
- params=probe_params,
- timeout=REQUEST_TIMEOUT_SECONDS,
- )
+ response = _probe(session, "/domains", probe_params)
+ if response is None:
+ return False, unreachable_vendor_message("OpenAlex")
if response.status_code in (401, 403):
return False, "Invalid OpenAlex API key"
if not response.ok:
@@ -224,11 +236,9 @@ def validate_credentials(api_key: str, filters: dict[str, Optional[str]]) -> tup
if not entity_filter or not entity_filter.strip():
continue
endpoint_config = OPENALEX_ENDPOINTS[endpoint]
- filter_response = session.get(
- f"{OPENALEX_BASE_URL}{endpoint_config.path}",
- params={**probe_params, "filter": entity_filter.strip()},
- timeout=REQUEST_TIMEOUT_SECONDS,
- )
+ filter_response = _probe(session, endpoint_config.path, {**probe_params, "filter": entity_filter.strip()})
+ if filter_response is None:
+ return False, unreachable_vendor_message("OpenAlex")
if filter_response.status_code == 400:
return False, f"The {endpoint} filter is not a valid OpenAlex filter expression"
if filter_response.status_code == 403:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_credential_contract.py b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_credential_contract.py
index 3157a98df786..684969311814 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_credential_contract.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_credential_contract.py
@@ -543,7 +543,6 @@ RAISES: dict[str, str] = {
"NoaaCdo": "500 connection",
"OPUSWatch": "connection",
"OneHundredMs": "500 connection",
- "Openalex": "connection",
"Orb": "500 connection",
"Paymongo": "500 connection",
"Paystack": "connection",
@@ -609,15 +608,10 @@ RAISES: dict[str, str] = {
# Sources whose probe returns no message for these answers. Each entry is a defect to fix: the person
# sees a generic message that does not say what is wrong.
-NO_MESSAGE: dict[str, str] = {
- "Confluence": "connection",
- "Xmatters": "500 connection",
-}
+NO_MESSAGE: dict[str, str] = {}
# Sources whose message or raised error contains a secret from the config. Each entry is a defect to fix.
-LEAKS_SECRET: set[str] = {
- "Openalex",
-}
+LEAKS_SECRET: set[str] = set()
@frozen
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/xmatters/xmatters.py b/products/warehouse_sources/backend/temporal/data_imports/sources/xmatters/xmatters.py
index 2f46057e5028..fde8b496ab80 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/xmatters/xmatters.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/xmatters/xmatters.py
@@ -21,7 +21,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.resource import Resource
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import ClientConfig
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import validate_via_probe
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import (
+ unreachable_vendor_message,
+ validate_via_probe,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.xmatters.settings import (
PAGE_SIZE,
@@ -168,7 +171,9 @@ def validate_credentials(
return False, status, "Invalid xMatters credentials"
if status == 403:
return False, status, "Your xMatters account does not have access to this resource"
- return False, status, None
+ if status is None or status >= 500:
+ return False, status, unreachable_vendor_message("xMatters")
+ return False, status, f"xMatters API returned status {status}"
def _fanout_source(