:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-warehouse): keep api keys out of credential probe errors (#120454)
Tom Owers10e4218b671e
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/source_helpers.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/source_helpers.pyindex 99e48bd4d90a..cf437b4ac437 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/source_helpers.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/source_helpers.py@@ -11,6 +11,13 @@ import requestsfrom requests.auth import AuthBase+def unreachable_vendor_message(vendor: str) -> str:+ """The `validate_credentials` message for a probe that got no usable answer from the vendor.++ It must not say that the credentials are invalid, because nothing proved that."""+ return f"Couldn't reach {vendor} to check your credentials. Try again in a few minutes."++def validate_via_probe(session_factory: Callable[[], requests.Session],url: str,diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/confluence/confluence.py b/products/warehouse_sources/backend/temporal/data_imports/sources/confluence/confluence.pyindex be2ced1673e3..d8e8a670d925 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/confluence/confluence.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/confluence/confluence.py@@ -19,7 +19,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.resfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.resource import Resourcefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import ClientConfigfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager-from products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import validate_via_probe+from products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import (+ unreachable_vendor_message,+ validate_via_probe,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponsefrom products.warehouse_sources.backend.temporal.data_imports.sources.confluence.settings import (CONFLUENCE_ENDPOINTS,@@ -237,6 +240,6 @@ def validate_credentials(return True, Nonereturn False, "Your Confluence account does not have permission to access this resource."if status is None:- return False, None+ return False, unreachable_vendor_message("Confluence")return False, f"Confluence API returned status {status}."diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/confluence/tests/test_confluence.py b/products/warehouse_sources/backend/temporal/data_imports/sources/confluence/tests/test_confluence.pyindex 52f0f934c68c..c33193fbe2be 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/confluence/tests/test_confluence.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/confluence/tests/test_confluence.py@@ -91,7 +91,7 @@ class TestValidateCredentials:with scripted_network(always(ConnectionError("boom"))) as network:is_valid, message = validate_credentials("acme", "[email protected]", "token")assert is_valid is False- assert message is None+ assert message == "Couldn't reach Confluence to check your credentials. Try again in a few minutes."assert {request.path for request in network.requests_log} == {"/wiki/api/v2/spaces"}@parameterized.expand(diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openalex/openalex.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openalex/openalex.pyindex 3378321fa91a..0260f5f647cc 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openalex/openalex.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openalex/openalex.py@@ -2,6 +2,8 @@ import datetimeimport dataclassesfrom typing import Any, Optional+import requests+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_sessionfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (RESTAPIConfig,@@ -12,6 +14,9 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res)from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import EndpointResourcefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager+from products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import (+ unreachable_vendor_message,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponsefrom products.warehouse_sources.backend.temporal.data_imports.sources.openalex.settings import (OPENALEX_ENDPOINTS,@@ -199,6 +204,15 @@ def openalex_source()+def _probe(session: requests.Session, path: str, params: dict[str, str]) -> Optional[requests.Response]:+ # The API key travels in the query string, and a `requests` error repeats the full URL in its+ # message. Returning `None` keeps that message, and so the key, out of the caller's error report.+ try:+ return session.get(f"{OPENALEX_BASE_URL}{path}", params=params, timeout=REQUEST_TIMEOUT_SECONDS)+ except requests.RequestException:+ return None++def validate_credentials(api_key: str, filters: dict[str, Optional[str]]) -> tuple[bool, Optional[str]]:"""Probe the API key, then each configured filter expression.@@ -210,11 +224,9 @@ def validate_credentials(api_key: str, filters: dict[str, Optional[str]]) -> tupprobe_params: dict[str, str] = {"per_page": "1", "api_key": api_key}- response = session.get(- f"{OPENALEX_BASE_URL}/domains",- params=probe_params,- timeout=REQUEST_TIMEOUT_SECONDS,- )+ response = _probe(session, "/domains", probe_params)+ if response is None:+ return False, unreachable_vendor_message("OpenAlex")if response.status_code in (401, 403):return False, "Invalid OpenAlex API key"if not response.ok:@@ -224,11 +236,9 @@ def validate_credentials(api_key: str, filters: dict[str, Optional[str]]) -> tupif not entity_filter or not entity_filter.strip():continueendpoint_config = OPENALEX_ENDPOINTS[endpoint]- filter_response = session.get(- f"{OPENALEX_BASE_URL}{endpoint_config.path}",- params={**probe_params, "filter": entity_filter.strip()},- timeout=REQUEST_TIMEOUT_SECONDS,- )+ filter_response = _probe(session, endpoint_config.path, {**probe_params, "filter": entity_filter.strip()})+ if filter_response is None:+ return False, unreachable_vendor_message("OpenAlex")if filter_response.status_code == 400:return False, f"The {endpoint} filter is not a valid OpenAlex filter expression"if filter_response.status_code == 403:diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_credential_contract.py b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_credential_contract.pyindex 3157a98df786..684969311814 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_credential_contract.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_credential_contract.py@@ -543,7 +543,6 @@ RAISES: dict[str, str] = {"NoaaCdo": "500 connection","OPUSWatch": "connection","OneHundredMs": "500 connection",- "Openalex": "connection","Orb": "500 connection","Paymongo": "500 connection","Paystack": "connection",@@ -609,15 +608,10 @@ RAISES: dict[str, str] = {# Sources whose probe returns no message for these answers. Each entry is a defect to fix: the person# sees a generic message that does not say what is wrong.-NO_MESSAGE: dict[str, str] = {- "Confluence": "connection",- "Xmatters": "500 connection",-}+NO_MESSAGE: dict[str, str] = {}# Sources whose message or raised error contains a secret from the config. Each entry is a defect to fix.-LEAKS_SECRET: set[str] = {- "Openalex",-}+LEAKS_SECRET: set[str] = set()@frozendiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/xmatters/xmatters.py b/products/warehouse_sources/backend/temporal/data_imports/sources/xmatters/xmatters.pyindex 2f46057e5028..fde8b496ab80 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/xmatters/xmatters.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/xmatters/xmatters.py@@ -21,7 +21,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.resfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.resource import Resourcefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import ClientConfigfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager-from products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import validate_via_probe+from products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import (+ unreachable_vendor_message,+ validate_via_probe,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponsefrom products.warehouse_sources.backend.temporal.data_imports.sources.xmatters.settings import (PAGE_SIZE,@@ -168,7 +171,9 @@ def validate_credentials(return False, status, "Invalid xMatters credentials"if status == 403:return False, status, "Your xMatters account does not have access to this resource"- return False, status, None+ if status is None or status >= 500:+ return False, status, unreachable_vendor_message("xMatters")+ return False, status, f"xMatters API returned status {status}"def _fanout_source(