:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(semgrep): explain why semgrep token validation failed (#126734)
Tom Owerscafc780d8399
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/semgrep/semgrep.py b/products/warehouse_sources/backend/temporal/data_imports/sources/semgrep/semgrep.pyindex 9f35ea167226..568c00ee3e00 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/semgrep/semgrep.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/semgrep/semgrep.py@@ -292,12 +292,11 @@ def semgrep_source()-def validate_credentials(api_token: str) -> bool:+def validate_credentials(api_token: str) -> tuple[bool, int | None]:# One cheap probe of the token itself: /deployments is the root resource every Web API token# can read, and it's the same call the sync fans out from.- ok, _status = validate_via_probe(+ return validate_via_probe(lambda: _client_session(api_token),f"{SEMGREP_BASE_URL}/deployments",headers={"Authorization": f"Bearer {api_token}", "Accept": "application/json"},)- return okdiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/semgrep/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/semgrep/source.pyindex d42155fb1131..105bc5644f0d 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/semgrep/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/semgrep/source.py@@ -30,6 +30,12 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.semgrep.se)from products.warehouse_sources.backend.types import ExternalDataSourceType+INVALID_TOKEN_MESSAGE = "Your Semgrep API token is invalid or has been revoked. Create a new token with the Web API scope under Settings → Tokens in Semgrep, then reconnect."+MISSING_SCOPE_MESSAGE = "Your Semgrep API token does not have permission to read this data. Grant the token the Web API scope (and make sure your plan includes API access), then reconnect."+UNREACHABLE_MESSAGE = (+ "PostHog couldn't reach Semgrep to check your API token. This is usually temporary, so wait a moment and try again."+)+@SourceRegistry.registerclass SemgrepSource(ResumableSource[SemgrepSourceConfig, SemgrepResumeConfig]):@@ -81,8 +87,8 @@ Create a token in Semgrep AppSec Platform under **Settings → Tokens** and gran# An invalid or revoked API token surfaces as a requests HTTPError when `_fetch_page`# calls `raise_for_status()`. Retrying can never satisfy a credential problem, so stop# the sync. Match the stable status text and base host, not the per-request path/query.- "401 Client Error: Unauthorized for url: https://semgrep.dev": "Your Semgrep API token is invalid or has been revoked. Create a new token with the Web API scope under Settings → Tokens in Semgrep, then reconnect.",- "403 Client Error: Forbidden for url: https://semgrep.dev": "Your Semgrep API token does not have permission to read this data. Grant the token the Web API scope (and make sure your plan includes API access), then reconnect.",+ "401 Client Error: Unauthorized for url: https://semgrep.dev": INVALID_TOKEN_MESSAGE,+ "403 Client Error: Forbidden for url: https://semgrep.dev": MISSING_SCOPE_MESSAGE,}def get_schemas(@@ -121,10 +127,14 @@ Create a token in Semgrep AppSec Platform under **Settings → Tokens** and gran) -> tuple[bool, str | None]:# /deployments is the cheapest probe and the root every synced endpoint fans out from, so# it confirms the token for any schema.- if validate_semgrep_credentials(config.api_token):+ ok, status = validate_semgrep_credentials(config.api_token)+ if ok:return True, None-- return False, "Invalid Semgrep API token"+ if status == 401:+ return False, INVALID_TOKEN_MESSAGE+ if status == 403:+ return False, MISSING_SCOPE_MESSAGE+ return False, UNREACHABLE_MESSAGEdef get_resumable_source_manager(self, inputs: SourceInputs) -> ResumableSourceManager[SemgrepResumeConfig]:return ResumableSourceManager[SemgrepResumeConfig](inputs, SemgrepResumeConfig)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/semgrep/tests/test_semgrep.py b/products/warehouse_sources/backend/temporal/data_imports/sources/semgrep/tests/test_semgrep.pyindex e37fc3293b78..f12cc5555878 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/semgrep/tests/test_semgrep.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/semgrep/tests/test_semgrep.py@@ -175,6 +175,6 @@ class TestSemgrepSourceResponse:class TestValidateCredentials:def test_ok(self) -> None:with scripted_network(always(ScriptedResponse(status=200, json={"deployments": []}))) as network:- assert validate_credentials("token") is True+ assert validate_credentials("token") == (True, 200)assert network.requests_log[0].headers["authorization"] == "Bearer token"diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/semgrep/tests/test_semgrep_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/semgrep/tests/test_semgrep_source.pyindex d3c057dad664..52f930ee71be 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/semgrep/tests/test_semgrep_source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/semgrep/tests/test_semgrep_source.py@@ -2,11 +2,18 @@ from typing import Anyfrom unittest.mock import MagicMock, patch+from parameterized import parameterized+from products.warehouse_sources.backend.temporal.data_imports.sources.semgrep.canonical_descriptions import (CANONICAL_DESCRIPTIONS,)from products.warehouse_sources.backend.temporal.data_imports.sources.semgrep.settings import ENDPOINTS-from products.warehouse_sources.backend.temporal.data_imports.sources.semgrep.source import SemgrepSource+from products.warehouse_sources.backend.temporal.data_imports.sources.semgrep.source import (+ INVALID_TOKEN_MESSAGE,+ MISSING_SCOPE_MESSAGE,+ UNREACHABLE_MESSAGE,+ SemgrepSource,+)def _config(api_token: str = "token") -> Any:@@ -19,21 +26,29 @@ class TestValidateCredentials:def test_success(self) -> None:with patch("products.warehouse_sources.backend.temporal.data_imports.sources.semgrep.source.validate_semgrep_credentials",- return_value=True,+ return_value=(True, 200),) as mocked:ok, error = SemgrepSource().validate_credentials(_config(), team_id=1)assert ok is Trueassert error is Nonemocked.assert_called_once_with("token")- def test_failure(self) -> None:+ @parameterized.expand(+ [+ ("unauthorized", 401, INVALID_TOKEN_MESSAGE),+ ("forbidden", 403, MISSING_SCOPE_MESSAGE),+ ("server_error", 503, UNREACHABLE_MESSAGE),+ ("unreachable", None, UNREACHABLE_MESSAGE),+ ]+ )+ def test_failure(self, _name: str, status: int | None, expected_error: str) -> None:with patch("products.warehouse_sources.backend.temporal.data_imports.sources.semgrep.source.validate_semgrep_credentials",- return_value=False,+ return_value=(False, status),):ok, error = SemgrepSource().validate_credentials(_config(), team_id=1, schema_name="sast_findings")assert ok is False- assert error is not None+ assert error == expected_errorclass TestCanonicalDescriptions: