Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(semgrep): add scans and dependency inventory tables (#130159)

Tom Owersf5213ab87326
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 97b7ac6025fa..d43530f66ff6 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -10785,10 +10785,10 @@ Today (5): `deployments`, `projects`, `sast_findings`, `sca_findings`, `secrets`
Diffed against: <https://semgrep.dev/api/v1/public_v1.openapi.yaml>
-- [ ] `POST /api/v1/deployments/{deploymentId}/dependencies (List dependencies)` — The full SCA dependency inventory that sca_findings point at — lets you join findings to the packages actually in use (high)
-- [ ] `POST /api/v1/deployments/{deploymentId}/scans/search (List scans)` — Scan run history per project — needed to trend findings over time and detect projects that stopped scanning (high)
-- [ ] `POST /api/v1/deployments/{deploymentId}/dependencies/repositories (List repositories with dependencies)` — Repository-level rollup that resolves which repos carry a given dependency (medium)
-- [ ] `POST /api/v1/deployments/{deploymentId}/dependencies/repositories/{repositoryId}/lockfiles` — Lockfile inventory per repo; useful for supply-chain coverage auditing (low)
+- [x] `POST /api/v1/deployments/{deploymentId}/dependencies (List dependencies)` — The full SCA dependency inventory that sca_findings point at — lets you join findings to the packages actually in use (high) → `dependencies` (full refresh, fan-out over `deployments`)
+- [x] `POST /api/v1/deployments/{deploymentId}/scans/search (List scans)` — Scan run history per project — needed to trend findings over time and detect projects that stopped scanning (high) → `scans` (full refresh, fan-out over `deployments`; Semgrep returns the last 30 days)
+- [x] `POST /api/v1/deployments/{deploymentId}/dependencies/repositories (List repositories with dependencies)` — Repository-level rollup that resolves which repos carry a given dependency (medium) → `dependency_repositories` (full refresh, fan-out over `deployments`)
+- [x] `POST /api/v1/deployments/{deploymentId}/dependencies/repositories/{repositoryId}/lockfiles` — Lockfile inventory per repo; useful for supply-chain coverage auditing (low) → `dependency_lockfiles` (full refresh, fan-out over `dependency_repositories`)
Note: The dependency and scan list endpoints are POST-with-body (paginated search), not GET, so they need a POST-capable resource config. Policies and SBOM export are config/ephemeral and deliberately excluded.
diff --git a/products/warehouse_sources/sources/semgrep/canonical_descriptions.py b/products/warehouse_sources/sources/semgrep/canonical_descriptions.py
index 11f90a6277ba..635969f967ac 100644
--- a/products/warehouse_sources/sources/semgrep/canonical_descriptions.py
+++ b/products/warehouse_sources/sources/semgrep/canonical_descriptions.py
@@ -113,4 +113,64 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
**_FAN_OUT_COLUMNS,
},
},
+ "scans": {
+ "description": "Semgrep scans run in the deployment, with their branch, commit, status, duration and finding counts. Semgrep returns the scans of the last 30 days.",
+ "docs_url": "https://semgrep.dev/api/v1/docs/#tag/ScansService",
+ "columns": {
+ "id": "ID of the scan.",
+ "repository_id": "Unique identifier for the repository of the scan.",
+ "branch": "The scanned branch.",
+ "commit": "The commit hash that was scanned.",
+ "started_at": "The timestamp when this scan started.",
+ "completed_at": "The timestamp when this scan completed (if it has completed).",
+ "status": "The current status of the scan, e.g. SCAN_STATUS_RUNNING, SCAN_STATUS_COMPLETED, SCAN_STATUS_PENDING or SCAN_STATUS_CANCELLED.",
+ "exit_code": "The exit code of the scan.",
+ "total_time": "Duration of the scan, in seconds.",
+ "is_full_scan": "Whether the scan was a full scan (true) or a diff scan (false).",
+ "is_partial_scan": "Whether the scan ran an explicit subset of rules rather than the full rule set.",
+ "enabled_products": "The products used when running the scan.",
+ "findings_counts": "Number of code, supply chain and secrets findings in the scan, and their total.",
+ **_FAN_OUT_COLUMNS,
+ },
+ },
+ "dependencies": {
+ "description": "Supply chain dependency inventory: each package version Semgrep found in a lockfile or manifest of the deployment's repositories. Dependencies of archived repositories are not returned.",
+ "docs_url": "https://semgrep.dev/api/v1/docs/#tag/SupplyChainService",
+ "columns": {
+ "dependency_key": "Hash of the repository, ecosystem, package, version and lockfile location that identifies this dependency (added by PostHog).",
+ "repositoryId": "ID of the repository the dependency is found in.",
+ "package": "The dependency's package name and version specifier.",
+ "ecosystem": "The ecosystem the dependency is in (e.g. npm, pypi, maven).",
+ "transitivity": "Whether the dependency is direct or transitive.",
+ "licenses": "Licenses the dependency is using.",
+ "definedAt": "Path and line number the dependency is declared in.",
+ "manifestDefinition": "Path to the manifest file that defines the subproject containing this dependency.",
+ "resolvedUrl": "The resolved URL of the dependency, such as a tarball, source repository or package cache directory.",
+ **_FAN_OUT_COLUMNS,
+ },
+ },
+ "dependency_repositories": {
+ "description": "Repositories in the deployment that have supply chain dependencies, with their dependency counts. Archived repositories are not returned.",
+ "docs_url": "https://semgrep.dev/api/v1/docs/#tag/SupplyChainService",
+ "columns": {
+ "id": "ID of the repository.",
+ "name": "Name of the repository.",
+ "numDependencies": "Total number of dependencies in the repository.",
+ "isScmArchived": "True if the repository is archived in its source code manager.",
+ "isSemgrepArchived": "True if the repository is archived in Semgrep.",
+ "hasDependencyPathScan": "True if the repository has been scanned with dependency path (graph) data.",
+ **_FAN_OUT_COLUMNS,
+ },
+ },
+ "dependency_lockfiles": {
+ "description": "Lockfiles in each repository with supply chain dependencies, with the number of dependencies each lockfile declares.",
+ "docs_url": "https://semgrep.dev/api/v1/docs/#tag/SupplyChainService",
+ "columns": {
+ "lockfilePath": "Path to the lockfile (e.g. foo/bar/package-lock.json).",
+ "numDependencies": "Total number of dependencies in the lockfile.",
+ "repository_id": "ID of the repository the lockfile belongs to (added by PostHog).",
+ "repository_name": "Name of the repository the lockfile belongs to (added by PostHog).",
+ **_FAN_OUT_COLUMNS,
+ },
+ },
}
diff --git a/products/warehouse_sources/sources/semgrep/docs/semgrep.mdx b/products/warehouse_sources/sources/semgrep/docs/semgrep.mdx
index 936dc60168f1..9178b02546e9 100644
--- a/products/warehouse_sources/sources/semgrep/docs/semgrep.mdx
+++ b/products/warehouse_sources/sources/semgrep/docs/semgrep.mdx
@@ -58,16 +58,23 @@ re-reads the resource and merges on the primary key. Findings are deduplicated a
<SourceTables />
-Rows in the `projects`, `sast_findings`, `sca_findings`, and `secrets` tables carry an extra
-`deployment_id` and `deployment_slug` column added by PostHog so you can join them back to the
-`deployments` table.
+Rows in every table except `deployments` carry an extra `deployment_id` and `deployment_slug`
+column added by PostHog so you can join them back to the `deployments` table. Rows in the
+`dependency_lockfiles` table also carry `repository_id` and `repository_name`, and rows in the
+`dependencies` table carry a `dependency_key` column that PostHog computes, because Semgrep gives
+dependencies no ID.
+
+The `scans` table holds the scans of the last 30 days, which is what Semgrep's API returns. The
+`scans`, `dependencies`, `dependency_repositories`, and `dependency_lockfiles` tables use API
+endpoints that Semgrep marks as experimental.
## Troubleshooting
- **"Invalid Semgrep API token"** when connecting: the token is wrong, has been revoked, or is missing
the **Web API** scope. Create a new token under **Settings → Tokens** and re-enter it.
- **Sync fails with a 403**: your plan may not include API access to a synced resource (for example,
- the `secrets` table requires the Semgrep Secrets product). Deselect the affected table or upgrade
+ the `secrets` table requires the Semgrep Secrets product, and the dependency tables require Semgrep
+ Supply Chain). Deselect the affected table or upgrade
your Semgrep plan.
<TroubleshootingLink />
diff --git a/products/warehouse_sources/sources/semgrep/semgrep.py b/products/warehouse_sources/sources/semgrep/semgrep.py
index 942072c8e2a4..efd644a5e1aa 100644
--- a/products/warehouse_sources/sources/semgrep/semgrep.py
+++ b/products/warehouse_sources/sources/semgrep/semgrep.py
@@ -1,4 +1,7 @@
+import json
+import hashlib
import dataclasses
+from collections.abc import Iterable
from typing import Any, Optional
import requests
@@ -11,6 +14,7 @@ from products.warehouse_sources.sources.sdk import (
ResumableSourceManager,
SinglePagePaginator,
SourceResponse,
+ build_chained_resource,
make_tracked_session,
rest_api_resource,
rest_api_resources,
@@ -82,31 +86,43 @@ class _PageNumberPaginator(BasePaginator):
class _CursorPaginator(BasePaginator):
"""Cursor pagination carrying the next cursor in the response body.
- Terminates on an empty page, a missing cursor, or a cursor unchanged from the request's — the
- same guard the hand-rolled loop used so a server that keeps echoing its final cursor can't spin
- forever.
+ Terminates on an empty page, a missing cursor, a false `has_more_key` flag, or a cursor
+ unchanged from the request's — the same guard the hand-rolled loop used so a server that keeps
+ echoing its final cursor can't spin forever.
"""
- def __init__(self, cursor_param: str = "cursor", cursor_key: str = "cursor") -> None:
+ def __init__(
+ self, cursor_param: str = "cursor", cursor_key: str = "cursor", has_more_key: Optional[str] = None
+ ) -> None:
super().__init__()
self.cursor_param = cursor_param
self.cursor_key = cursor_key
+ self.has_more_key = has_more_key
self._cursor: Optional[str] = None
- def init_request(self, request: requests.Request) -> None:
+ def _apply(self, request: requests.Request) -> None:
if self._cursor is not None:
if request.params is None:
request.params = {}
request.params[self.cursor_param] = self._cursor
+ def init_request(self, request: requests.Request) -> None:
+ self._apply(request)
+
def update_state(self, response: requests.Response, data: Optional[list[Any]] = None) -> None:
if not data:
self._has_next_page = False
return
try:
- next_cursor = response.json().get(self.cursor_key)
+ body = response.json()
except Exception:
- next_cursor = None
+ body = {}
+ if not isinstance(body, dict):
+ body = {}
+ next_cursor = body.get(self.cursor_key)
+ if self.has_more_key is not None and body.get(self.has_more_key) is False:
+ self._has_next_page = False
+ return
if not next_cursor or next_cursor == self._cursor:
self._has_next_page = False
return
@@ -114,10 +130,7 @@ class _CursorPaginator(BasePaginator):
self._has_next_page = True
def update_request(self, request: requests.Request) -> None:
- if self._cursor is not None:
- if request.params is None:
- request.params = {}
- request.params[self.cursor_param] = self._cursor
+ self._apply(request)
def get_resume_state(self) -> Optional[dict[str, Any]]:
return {"cursor": self._cursor} if self._has_next_page and self._cursor is not None else None
@@ -129,6 +142,23 @@ class _CursorPaginator(BasePaginator):
self._has_next_page = True
+class _BodyCursorPaginator(_CursorPaginator):
+ """Cursor pagination for Semgrep's POST search endpoints, which take the page size and cursor in
+ the JSON body. It writes the whole body, because the framework sends a fan-out child no body."""
+
+ def __init__(self, page_size_param: str, page_size: int, has_more_key: Optional[str] = None) -> None:
+ super().__init__(has_more_key=has_more_key)
+ self.page_size_param = page_size_param
+ self.page_size = page_size
+
+ def _apply(self, request: requests.Request) -> None:
+ body = dict(request.json or {})
+ body[self.page_size_param] = self.page_size
+ if self._cursor is not None:
+ body[self.cursor_param] = self._cursor
+ request.json = body
+
+
def _client_session(api_token: str) -> requests.Session:
# `redact_values` masks the bearer token in logged URLs and captured HTTP samples. `capture=False`
# keeps response bodies out of sample capture entirely: findings and secrets payloads carry
@@ -157,13 +187,49 @@ def _rename_deployment_fields(row: dict[str, Any]) -> dict[str, Any]:
return row
+def _add_dependency_key(row: dict[str, Any]) -> dict[str, Any]:
+ # A dependency is one package version declared at one lockfile location of one repository.
+ defined_at = row.get("definedAt") or {}
+ manifest = row.get("manifestDefinition") or {}
+ package = row.get("package") or {}
+ parts = [
+ row.get("repositoryId"),
+ row.get("ecosystem"),
+ package.get("name"),
+ package.get("versionSpecifier"),
+ defined_at.get("path"),
+ defined_at.get("startLine"),
+ manifest.get("path"),
+ ]
+ row["dependency_key"] = hashlib.sha256(json.dumps(parts, default=str).encode()).hexdigest()
+ return row
+
+
+_ROW_MAPS = {"dependencies": _add_dependency_key}
+
+
def _fanout_paginator(config: SemgrepEndpointConfig) -> BasePaginator:
assert config.page_size is not None
+ if config.pagination == "body_cursor":
+ assert config.page_size_param is not None
+ return _BodyCursorPaginator(config.page_size_param, config.page_size, has_more_key=config.has_more_key)
if config.pagination == "cursor":
return _CursorPaginator()
return _PageNumberPaginator(page_size=config.page_size)
+def _deployments_resource() -> EndpointResource:
+ return {
+ "name": "deployments",
+ "endpoint": {
+ "path": "/deployments",
+ "data_selector": "deployments",
+ "data_selector_required": True,
+ "paginator": SinglePagePaginator(),
+ },
+ }
+
+
def _child_resolve_param(
config: SemgrepEndpointConfig,
) -> tuple[str, str]: # nosemgrep: tuple-return-prefer-dataclass -- pre-existing, file moved
@@ -187,23 +253,17 @@ def _fanout_source(
child_params: dict[str, Any] = {
param_name: {"type": "resolve", "resource": "deployments", "field": resolve_field},
- ("limit" if config.pagination == "cursor" else "page_size"): config.page_size,
**config.params,
}
+ if config.pagination != "body_cursor":
+ child_params["limit" if config.pagination == "cursor" else "page_size"] = config.page_size
- parent_resource: EndpointResource = {
- "name": "deployments",
- "endpoint": {
- "path": "/deployments",
- "data_selector": "deployments",
- "data_selector_required": True,
- "paginator": SinglePagePaginator(),
- },
- }
+ parent_resource = _deployments_resource()
child_resource: EndpointResource = {
"name": endpoint,
"include_from_parent": ["id", "slug"],
"endpoint": {
+ "method": config.method,
"path": config.path,
"params": child_params,
"data_selector": config.data_key,
@@ -237,8 +297,60 @@ def _fanout_source(
resume_hook=save_checkpoint,
initial_paginator_state=initial_paginator_state,
)
- child = next(r for r in resources if r.name == endpoint)
- return child.add_map(_rename_deployment_fields)
+ child = next(r for r in resources if r.name == endpoint).add_map(_rename_deployment_fields)
+ row_map = _ROW_MAPS.get(endpoint)
+ return child.add_map(row_map) if row_map is not None else child
+
+
+def _repository_fanout_source(
+ api_token: str, endpoint: str, config: SemgrepEndpointConfig, team_id: int, job_id: str
+) -> Iterable[Any]:
+ # deployments -> repositories with dependencies -> per-repository child. A two-level chain
+ # carries no resume state, so an interrupted run starts the chain again.
+ repositories_config = SEMGREP_ENDPOINTS["dependency_repositories"]
+ repositories_resource: EndpointResource = {
+ "name": "dependency_repositories",
+ "include_from_parent": ["id", "slug"],
+ # Runs before the child reads these rows, so the child resolves the flat `deployment_id`.
+ "data_map": _rename_deployment_fields,
+ "endpoint": {
+ "method": repositories_config.method,
+ "path": repositories_config.path,
+ "params": {"deployment_id": {"type": "resolve", "resource": "deployments", "field": "id"}},
+ "data_selector": repositories_config.data_key,
+ "data_selector_required": True,
+ "paginator": _fanout_paginator(repositories_config),
+ },
+ }
+ child_resource: EndpointResource = {
+ "name": endpoint,
+ "include_from_parent": ["deployment_id", "deployment_slug", "id", "name"],
+ "endpoint": {
+ "method": config.method,
+ "path": config.path,
+ "params": {
+ "deployment_id": {"type": "resolve", "resource": "dependency_repositories", "field": "deployment_id"},
+ "repository_id": {"type": "resolve", "resource": "dependency_repositories", "field": "id"},
+ },
+ "data_selector": config.data_key,
+ "data_selector_required": True,
+ "paginator": _fanout_paginator(config),
+ },
+ }
+ return build_chained_resource(
+ resources=[_deployments_resource(), repositories_resource, child_resource],
+ child_name=endpoint,
+ parent_name="dependency_repositories",
+ parent_field_renames={
+ "deployment_id": "deployment_id",
+ "deployment_slug": "deployment_slug",
+ "id": "repository_id",
+ "name": "repository_name",
+ },
+ client_config=_client_config(api_token, SinglePagePaginator()),
+ team_id=team_id,
+ job_id=job_id,
+ )
def _single_source(api_token: str, endpoint: str, config: SemgrepEndpointConfig, team_id: int, job_id: str) -> Any:
@@ -269,8 +381,14 @@ def semgrep_source(
) -> SourceResponse:
config = SEMGREP_ENDPOINTS[endpoint]
+ resource: Any
+ supports_resume = True
if config.pagination == "none":
resource = _single_source(api_token, endpoint, config, team_id, job_id)
+ elif config.parent == "dependency_repositories":
+ resource = _repository_fanout_source(api_token, endpoint, config, team_id, job_id)
+ # The two-level chain saves no checkpoint, so a hand-off would restart it from the start.
+ supports_resume = False
else:
resource = _fanout_source(api_token, endpoint, config, team_id, job_id, resumable_source_manager)
@@ -284,6 +402,7 @@ def semgrep_source(
partition_format="month" if config.partition_key else None,
partition_keys=[config.partition_key] if config.partition_key else None,
sort_mode=config.sort_mode,
+ supports_resume=supports_resume,
)
diff --git a/products/warehouse_sources/sources/semgrep/settings.py b/products/warehouse_sources/sources/semgrep/settings.py
index da65a058b5f3..02d5017229cc 100644
--- a/products/warehouse_sources/sources/semgrep/settings.py
+++ b/products/warehouse_sources/sources/semgrep/settings.py
@@ -1,10 +1,10 @@
from dataclasses import dataclass, field
from typing import Literal
-PaginationStyle = Literal["none", "page", "cursor"]
+PaginationStyle = Literal["none", "page", "cursor", "body_cursor"]
-@dataclass
+@dataclass(frozen=True)
class SemgrepEndpointConfig:
# Path relative to the API base URL; may contain `{deployment_slug}` / `{deployment_id}`
# placeholders resolved per deployment during the fan-out.
@@ -12,9 +12,17 @@ class SemgrepEndpointConfig:
# Response key carrying the row list (Semgrep wraps every list payload in an object).
data_key: str
# `page` = zero-indexed `page`/`page_size` params; `cursor` = `cursor`/`limit` params with the
- # next cursor returned in the response body; `none` = single unpaginated request.
+ # next cursor returned in the response body; `body_cursor` = POST search endpoints that take the
+ # page size and cursor in the JSON body; `none` = single unpaginated request.
pagination: PaginationStyle = "page"
page_size: int | None = None
+ method: Literal["get", "post"] = "get"
+ # JSON body field carrying the page size for `body_cursor` endpoints.
+ page_size_param: str | None = None
+ # Response flag that marks the last page, when the endpoint returns one.
+ has_more_key: str | None = None
+ # `dependency_repositories` fans out a second level, from each repository with dependencies.
+ parent: Literal["deployments", "dependency_repositories"] = "deployments"
# Static query params sent on every request (e.g. `issue_type`, `dedup`).
params: dict[str, str] = field(default_factory=dict)
# Fan-out rows get `deployment_id`/`deployment_slug` injected, and the composite key guards
@@ -31,7 +39,8 @@ class SemgrepEndpointConfig:
# the Web API scope (Team/Enterprise tier). Tokens are scoped to a single deployment, so the
# per-deployment fan-out normally iterates exactly one deployment. No endpoint offers a server-side
# updated-since filter (findings' `since` filters on `relevant_since`, which does not advance when
-# a finding's status/triage changes), so every table syncs as full refresh (see source.py).
+# a finding's status/triage changes, and scans' `since` filters on creation time while a running
+# scan's status changes later), so every table syncs as full refresh (see source.py).
SEMGREP_ENDPOINTS: dict[str, SemgrepEndpointConfig] = {
"deployments": SemgrepEndpointConfig(
path="/deployments",
@@ -72,6 +81,47 @@ SEMGREP_ENDPOINTS: dict[str, SemgrepEndpointConfig] = {
partition_key="createdAt",
sort_mode="desc",
),
+ # The scans and dependency endpoints are POST searches marked experimental in Semgrep's spec.
+ "scans": SemgrepEndpointConfig(
+ path="/deployments/{deployment_id}/scans/search",
+ data_key="scans",
+ pagination="body_cursor",
+ method="post",
+ page_size=500,
+ page_size_param="limit",
+ partition_key="started_at",
+ ),
+ "dependencies": SemgrepEndpointConfig(
+ path="/deployments/{deployment_id}/dependencies",
+ data_key="dependencies",
+ pagination="body_cursor",
+ method="post",
+ page_size=1000,
+ page_size_param="pageSize",
+ has_more_key="hasMore",
+ # Dependency rows carry no id; `dependency_key` is a hash of the fields that identify one.
+ primary_keys=["deployment_id", "dependency_key"],
+ ),
+ "dependency_repositories": SemgrepEndpointConfig(
+ path="/deployments/{deployment_id}/dependencies/repositories",
+ data_key="repositorySummaries",
+ pagination="body_cursor",
+ method="post",
+ page_size=100,
+ page_size_param="pageSize",
+ has_more_key="hasMore",
+ ),
+ "dependency_lockfiles": SemgrepEndpointConfig(
+ path="/deployments/{deployment_id}/dependencies/repositories/{repository_id}/lockfiles",
+ data_key="lockfileSummaries",
+ pagination="body_cursor",
+ method="post",
+ page_size=100,
+ page_size_param="pageSize",
+ has_more_key="hasMore",
+ parent="dependency_repositories",
+ primary_keys=["deployment_id", "repository_id", "lockfilePath"],
+ ),
}
ENDPOINTS = tuple(SEMGREP_ENDPOINTS.keys())
diff --git a/products/warehouse_sources/sources/semgrep/source.py b/products/warehouse_sources/sources/semgrep/source.py
index f176c528a7e9..bc66ec187692 100644
--- a/products/warehouse_sources/sources/semgrep/source.py
+++ b/products/warehouse_sources/sources/semgrep/source.py
@@ -50,7 +50,7 @@ class SemgrepSource(ResumableSource[SemgrepSourceConfig, SemgrepResumeConfig]):
label="Semgrep",
releaseStatus=ReleaseStatus.GA,
keywords=["sast", "sca", "appsec"],
- caption="""Enter your Semgrep API token to sync your static analysis data — deployments, projects, code (SAST) and supply chain (SCA) findings, and secrets — into the PostHog Data warehouse.
+ caption="""Enter your Semgrep API token to sync your static analysis data into the PostHog Data warehouse. This covers deployments, projects, scans, code (SAST) and supply chain (SCA) findings, secrets, and the supply chain dependency inventory.
Create a token in Semgrep AppSec Platform under **Settings → Tokens** and grant it the **Web API** scope. The API requires a Semgrep Team or Enterprise plan.""",
iconPath="/static/services/semgrep.png",
diff --git a/products/warehouse_sources/sources/semgrep/tests/test_semgrep.py b/products/warehouse_sources/sources/semgrep/tests/test_semgrep.py
index 148b6ed09eb4..9aa34ac355ed 100644
--- a/products/warehouse_sources/sources/semgrep/tests/test_semgrep.py
+++ b/products/warehouse_sources/sources/semgrep/tests/test_semgrep.py
@@ -127,12 +127,155 @@ class TestCursorFanout:
assert secrets[0].param("cursor") == "abc"
+class TestBodyCursorFanout:
+ @parameterized.expand(
+ [
+ # scans returns no `hasMore`, so the missing cursor ends it.
+ ("scans", "scans", "limit", {}),
+ # The dependency endpoints keep returning a cursor on the last page; `hasMore` ends it.
+ ("dependencies", "dependencies", "pageSize", {"hasMore": False, "cursor": "999"}),
+ ("dependency_repositories", "repositorySummaries", "pageSize", {"hasMore": False, "cursor": "999"}),
+ ]
+ )
+ def test_posts_page_size_and_cursor_in_body(
+ self, endpoint: str, data_key: str, page_size_param: str, last_page_extra: dict[str, Any]
+ ) -> None:
+ result = _driver().run(
+ endpoint,
+ [
+ _json_response({"deployments": [DEPLOYMENT]}),
+ _json_response({data_key: [{"id": "1"}], "cursor": "c1", "hasMore": True}),
+ _json_response({data_key: [{"id": "2"}], **last_page_extra}),
+ ],
+ )
+
+ assert result.raised is None
+ assert len(result.rows) == 2
+ assert all(row["deployment_id"] == 123 and row["deployment_slug"] == "my-org" for row in result.rows)
+ children = result.requests[1:]
+ assert len(children) == 2
+ assert all(req.method == "POST" and req.path.startswith("/api/v1/deployments/123/") for req in children)
+ page_size = SEMGREP_ENDPOINTS[endpoint].page_size
+ assert [req.json() for req in children] == [
+ {page_size_param: page_size},
+ {page_size_param: page_size, "cursor": "c1"},
+ ]
+ assert any((state.fanout_state or {}).get("child_state") == {"cursor": "c1"} for state in result.saved_states)
+
+ def test_resumes_from_saved_body_cursor(self) -> None:
+ result = _driver().run(
+ "scans",
+ [_json_response({"deployments": [DEPLOYMENT]}), _json_response({"scans": [{"id": "2"}]})],
+ resume_state=SemgrepResumeConfig(
+ fanout_state={
+ "completed": [],
+ "current": "/deployments/123/scans/search",
+ "child_state": {"cursor": "abc"},
+ }
+ ),
+ )
+
+ assert [row["id"] for row in result.rows] == ["2"]
+ scans = _child_requests(result.requests, "/scans/search")
+ assert len(scans) == 1
+ assert scans[0].json() == {"limit": 500, "cursor": "abc"}
+
+
+def _dependency(path: str = "package-lock.json", line: str = "10", version: str = "4.17.21") -> dict[str, Any]:
+ return {
+ "repositoryId": "42",
+ "ecosystem": "npm",
+ "package": {"name": "lodash", "versionSpecifier": version},
+ "definedAt": {"path": path, "startLine": line},
+ "transitivity": "DIRECT",
+ }
+
+
+class TestDependencyKey:
+ @parameterized.expand(
+ [
+ ("same_dependency", _dependency(), _dependency(), True),
+ ("other_version", _dependency(), _dependency(version="4.17.20"), False),
+ ("other_lockfile", _dependency(), _dependency(path="web/package-lock.json"), False),
+ ("other_line", _dependency(), _dependency(line="11"), False),
+ # Missing nested objects must not raise, and still key the row.
+ ("bare_rows", {"ecosystem": "npm"}, {"ecosystem": "npm"}, True),
+ ("bare_vs_null_objects", {"ecosystem": "npm"}, {"ecosystem": "npm", "package": None}, True),
+ ]
+ )
+ def test_dependency_key(self, _name: str, first: dict[str, Any], second: dict[str, Any], same: bool) -> None:
+ result = _driver().run(
+ "dependencies",
+ [
+ _json_response({"deployments": [DEPLOYMENT]}),
+ _json_response({"dependencies": [first, second], "hasMore": False}),
+ ],
+ )
+
+ keys = [row["dependency_key"] for row in result.rows]
+ assert len(keys) == 2 and all(keys)
+ assert (keys[0] == keys[1]) is same
+
+
+class TestLockfilesChain:
+ @staticmethod
+ def _respond(request: RecordedRequest) -> ScriptedResponse:
+ if request.path.endswith("/deployments"):
+ return _json_response({"deployments": [DEPLOYMENT]})
+ if request.path.endswith("/dependencies/repositories"):
+ if request.json().get("cursor") is None:
+ return _json_response(
+ {"repositorySummaries": [{"id": 1, "name": "org/api"}], "cursor": "r1", "hasMore": True}
+ )
+ return _json_response({"repositorySummaries": [{"id": 2, "name": "org/web"}], "hasMore": False})
+ if request.path.endswith("/repositories/1/lockfiles"):
+ if request.json().get("cursor") is None:
+ return _json_response(
+ {"lockfileSummaries": [{"lockfilePath": "a/uv.lock"}], "cursor": "l1", "hasMore": True}
+ )
+ return _json_response({"lockfileSummaries": [{"lockfilePath": "b/uv.lock"}], "hasMore": False})
+ return _json_response({"lockfileSummaries": [{"lockfilePath": "package-lock.json"}], "hasMore": False})
+
+ def test_fans_out_over_every_repository_page_and_lockfile_page(self) -> None:
+ result = _driver().run("dependency_lockfiles", self._respond)
+
+ assert result.raised is None
+ assert [
+ (
+ row["deployment_id"],
+ row["deployment_slug"],
+ row["repository_id"],
+ row["repository_name"],
+ row["lockfilePath"],
+ )
+ for row in result.rows
+ ] == [
+ (123, "my-org", 1, "org/api", "a/uv.lock"),
+ (123, "my-org", 1, "org/api", "b/uv.lock"),
+ (123, "my-org", 2, "org/web", "package-lock.json"),
+ ]
+ # No prefixed parent columns leak into the table.
+ assert not any(key.startswith("_") for row in result.rows for key in row)
+ lockfile_requests = _child_requests(result.requests, "/lockfiles")
+ assert [(req.method, req.path, req.json()) for req in lockfile_requests] == [
+ ("POST", "/api/v1/deployments/123/dependencies/repositories/1/lockfiles", {"pageSize": 100}),
+ (
+ "POST",
+ "/api/v1/deployments/123/dependencies/repositories/1/lockfiles",
+ {"pageSize": 100, "cursor": "l1"},
+ ),
+ ("POST", "/api/v1/deployments/123/dependencies/repositories/2/lockfiles", {"pageSize": 100}),
+ ]
+
+
class TestSemgrepSourceResponse:
@staticmethod
def _empty_listing(request: RecordedRequest) -> ScriptedResponse:
if request.path.endswith("/deployments"):
return _json_response({"deployments": [DEPLOYMENT]})
- return _json_response({"findings": [], "projects": []})
+ return _json_response(
+ {"findings": [], "projects": [], "scans": [], "dependencies": [], "repositorySummaries": []}
+ )
@parameterized.expand([(name,) for name in ENDPOINTS])
def test_source_response_shape(self, name: str) -> None:
@@ -142,6 +285,8 @@ class TestSemgrepSourceResponse:
assert response.name == name
assert response.primary_keys == config.primary_keys
assert response.sort_mode == config.sort_mode
+ # Only the two-level lockfile chain saves no checkpoint.
+ assert response.supports_resume is (config.parent == "deployments")
if config.partition_key:
assert response.partition_keys == [config.partition_key]
assert response.partition_mode == "datetime"
@@ -149,19 +294,25 @@ class TestSemgrepSourceResponse:
else:
assert response.partition_keys is None
- def test_fan_out_endpoints_key_on_deployment_and_id(self) -> None:
+ def test_fan_out_endpoints_key_on_deployment(self) -> None:
# Guards against dropping the parent id from a fan-out child's key, which would seed
# duplicate rows if a token ever spans multiple deployments.
for name, config in SEMGREP_ENDPOINTS.items():
if name == "deployments":
assert config.primary_keys == ["id"]
else:
- assert config.primary_keys == ["deployment_id", "id"]
+ assert config.primary_keys[0] == "deployment_id"
+ assert len(config.primary_keys) > 1
def test_partition_keys_are_stable_creation_timestamps(self) -> None:
# Partitioning on a churning field (updated_at) rewrites partitions on every sync.
partitioned = {name: cfg.partition_key for name, cfg in SEMGREP_ENDPOINTS.items() if cfg.partition_key}
- assert partitioned == {"sast_findings": "created_at", "sca_findings": "created_at", "secrets": "createdAt"}
+ assert partitioned == {
+ "sast_findings": "created_at",
+ "sca_findings": "created_at",
+ "secrets": "createdAt",
+ "scans": "started_at",
+ }
class TestValidateCredentials: