:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add sites, scans and scan engines tables to rapid7 insightvm (#126977)
Tom Owers22d2793eb7a6
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 9c30402cda47..8ad54f6d461b 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -10034,9 +10034,9 @@ Note: Ramp publishes a full OpenAPI spec (2.1 MB) plus llms-api.txt; the docs HTDiffed against: <https://help.rapid7.com/insightvm/en-us/api/insightvm-api-v4.json>-- [ ] `sites (POST /vm/v4/integration/sites)` — Lookup table resolving the site IDs carried on every asset we already sync (high)-- [ ] `scan (GET /vm/v4/integration/scan)` — Scan history - when each site was assessed, which is needed to judge how stale asset and vulnerability data is (medium)-- [ ] `scan/engine` — Lookup resolving the engine that produced each scan (low)+- [x] `sites (POST /vm/v4/integration/sites)` — Lookup table resolving the site IDs carried on every asset we already sync (high)+- [x] `scan (GET /vm/v4/integration/scan)` — Scan history - when each site was assessed, which is needed to judge how stale asset and vulnerability data is (medium)+- [x] `scan/engine` — Lookup resolving the engine that produced each scan (low)Note: PostHog targets the InsightVM Cloud Integrations API (/vm/v4/integration on \*.api.insight.rapid7.com), whose spec exposes only 5 resources: assets, sites, vulnerabilities, scan and scan/engine. So 2 of 5 is not as thin as it looks. The much larger on-prem console API v3 (207 paths - solutions, tags, asset_groups, policies, exploits, vulnerability_exceptions, per-asset findings) is a different product surface at help.rapid7.com/insightvm/en-us/api/api-v3.json requiring a console host and Basic auth; adding it would be a new source variant, not extra endpoints on this one. Endpoints are static (products/warehouse_sources/.../sources/rapid7_insightvm/settings.py), no dynamic discovery.diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/rapid7_insightvm/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/rapid7_insightvm/canonical_descriptions.pyindex c9729586bd00..6bba5114f48e 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/rapid7_insightvm/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/rapid7_insightvm/canonical_descriptions.py@@ -46,4 +46,38 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"malware_kits": "Known malware kits that leverage the vulnerability.",},},+ "sites": {+ "description": "A site in InsightVM: a named group of assets that are scanned together.",+ "docs_url": "https://help.rapid7.com/insightvm/en-us/api/integrations.html",+ "columns": {+ "name": "Name of the site.",+ "type": "Type of the record. Always `SITE` for sites.",+ },+ },+ "scans": {+ "description": "A scan run in InsightVM, with the engine that ran it, its status, and when it started and finished.",+ "docs_url": "https://help.rapid7.com/insightvm/en-us/api/integrations.html",+ "columns": {+ "id": "Unique identifier for the scan.",+ "name": "Name of the scan.",+ "engine_id": "Identifier of the scan engine that ran the scan.",+ "status": "Status of the scan (e.g. Running, Success, Failed).",+ "started": "Timestamp when the scan started.",+ "finished": "Timestamp when the scan finished.",+ "details": "Additional details about the scan.",+ },+ },+ "scan_engines": {+ "description": "A scan engine registered with InsightVM, with its host, health status, and configuration profile.",+ "docs_url": "https://help.rapid7.com/insightvm/en-us/api/integrations.html",+ "columns": {+ "id": "Unique identifier for the scan engine.",+ "name": "Name of the scan engine.",+ "host_name": "Host name or address of the scan engine.",+ "status": "Health status of the scan engine (e.g. HEALTHY).",+ "registered": "Timestamp when the scan engine was registered.",+ "last_seen": "Timestamp when the scan engine last communicated with the platform.",+ "profile": "Configuration profile of the scan engine, including custom properties.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/rapid7_insightvm/rapid7_insightvm.py b/products/warehouse_sources/backend/temporal/data_imports/sources/rapid7_insightvm/rapid7_insightvm.pyindex fb780cd9989a..f88ca42cc612 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/rapid7_insightvm/rapid7_insightvm.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/rapid7_insightvm/rapid7_insightvm.py@@ -2,6 +2,7 @@ import dataclassesfrom typing import Any, Optionalfrom urllib.parse import urlencode+import structlogfrom requests import Responsefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session@@ -13,23 +14,31 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.resfind_values,)from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (+ BasePaginator,JSONResponseCursorPaginator,+ PageNumberPaginator,)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import Endpointfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManagerfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponsefrom products.warehouse_sources.backend.temporal.data_imports.sources.rapid7_insightvm.settings import (API_BASE_PATH,- MAX_PAGE_SIZE,+ PAGE_NUMBER_MAX_PAGE,RAPID7_INSIGHTVM_ENDPOINTS,REGION_HOSTS,+ Rapid7InsightvmEndpointConfig,)+logger = structlog.get_logger(__name__)+[email protected](frozen=True)class Rapid7InsightvmResumeConfig:# Opaque cursor token returned in the previous page's `metadata.cursor`. `None` starts the# endpoint from its first page.cursor: str | None = None+ # Next zero-based page for the page-number endpoints (scans, scan engines).+ page: int | None = Noneclass Rapid7InsightvmCursorPaginator(JSONResponseCursorPaginator):@@ -58,6 +67,25 @@ class Rapid7InsightvmCursorPaginator(JSONResponseCursorPaginator):self._has_next_page = True+class Rapid7InsightvmPageNumberPaginator(PageNumberPaginator):+ """Page-number pagination that warns when the collection outgrows the API's 10,000-result window."""++ def __init__(self) -> None:+ super().__init__(base_page=0, total_path="metadata.totalPages", maximum_page=PAGE_NUMBER_MAX_PAGE)++ def update_state(self, response: Response, data: Optional[list[Any]] = None) -> None:+ super().update_state(response, data)+ if self._has_next_page or self.page <= PAGE_NUMBER_MAX_PAGE:+ return+ total_pages = (response.json().get("metadata") or {}).get("totalPages")+ if isinstance(total_pages, int) and total_pages > PAGE_NUMBER_MAX_PAGE + 1:+ logger.warning(+ "Rapid7 InsightVM collection exceeds the 10,000-result page window, truncating",+ path=response.request.path_url if response.request else None,+ total_pages=total_pages,+ )++def _host(region: str) -> str:return REGION_HOSTS.get(region, REGION_HOSTS["us"])@@ -74,6 +102,25 @@ def _endpoint_url(region: str, path: str) -> str:return f"{_host(region)}{API_BASE_PATH}/{path}"+def _paginator(config: Rapid7InsightvmEndpointConfig) -> BasePaginator:+ if config.pagination == "page_number":+ return Rapid7InsightvmPageNumberPaginator()+ return Rapid7InsightvmCursorPaginator()+++def _endpoint(config: Rapid7InsightvmEndpointConfig) -> Endpoint:+ endpoint: Endpoint = {+ "method": config.method,+ "path": f"{API_BASE_PATH}/{config.path}",+ "params": {"size": config.page_size, **config.params},+ "data_selector": "data",+ }+ if config.method == "POST":+ # v4 search endpoints are POST operations; an empty JSON body returns all resources.+ endpoint["json"] = {}+ return endpoint++def rapid7_insightvm_source(api_key: str,region: str,@@ -92,7 +139,7 @@ def rapid7_insightvm_source(# logs and raised error messages; only the non-secret accept/content headers are set here."headers": {"Accept": "application/json", "Content-Type": "application/json"},"auth": {"type": "api_key", "api_key": api_key, "name": "X-Api-Key", "location": "header"},- "paginator": Rapid7InsightvmCursorPaginator(),+ "paginator": _paginator(config),# A 3xx from the credentialed endpoint would otherwise replay `X-Api-Key` to the redirect# target — reject redirects so the key stays pinned to the expected host."allow_redirects": False,@@ -101,14 +148,7 @@ def rapid7_insightvm_source("resources": [{"name": endpoint,- "endpoint": {- # v4 search endpoints are POST operations; an empty JSON body returns all resources.- "method": "POST",- "path": f"{API_BASE_PATH}/{config.path}",- "params": {"size": MAX_PAGE_SIZE},- "json": {},- "data_selector": "data",- },+ "endpoint": _endpoint(config),}],}@@ -118,12 +158,16 @@ def rapid7_insightvm_source(resume = resumable_source_manager.load_state()if resume is not None and resume.cursor is not None:initial_paginator_state = {"cursor": resume.cursor}+ elif resume is not None and resume.page is not None:+ initial_paginator_state = {"page": resume.page}def save_checkpoint(state: Optional[dict[str, Any]]) -> None:# Persist only when a next cursor remains; save AFTER a page is yielded so a crash re-yields# the last page (merge dedupes on the primary key) rather than skipping it.if state and state.get("cursor") is not None:resumable_source_manager.save_state(Rapid7InsightvmResumeConfig(cursor=str(state["cursor"])))+ elif state and state.get("page") is not None:+ resumable_source_manager.save_state(Rapid7InsightvmResumeConfig(page=int(state["page"])))resource = rest_api_resource(rest_config,diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/rapid7_insightvm/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/rapid7_insightvm/settings.pyindex ff77173428f9..a959231a4975 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/rapid7_insightvm/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/rapid7_insightvm/settings.py@@ -1,4 +1,5 @@from dataclasses import dataclass, field+from typing import Literalfrom products.warehouse_sources.backend.types import IncrementalField@@ -20,16 +21,27 @@ REGION_HOSTS: dict[str, str] = {API_BASE_PATH = "/vm/v4/integration"MAX_PAGE_SIZE = 1000+# The v4 docs cap a page at 500 rows. The GET collections (scans, scan engines) page with zero-based+# `page` + `size` and no cursor, which serves at most 10,000 results, so the walk stops at the last+# page inside that window instead of requesting one the API refuses.+DOCUMENTED_MAX_PAGE_SIZE = 500+PAGE_NUMBER_MAX_PAGE = 10_000 // DOCUMENTED_MAX_PAGE_SIZE - 1-@dataclass++@dataclass(frozen=True)class Rapid7InsightvmEndpointConfig:name: str# Path segment appended to `{host}{API_BASE_PATH}/` (e.g. "assets").path: strprimary_keys: list[str]+ # Search endpoints are POST + cursor; plain collection endpoints are GET + page number.+ method: Literal["GET", "POST"] = "POST"+ pagination: Literal["cursor", "page_number"] = "cursor"+ page_size: int = MAX_PAGE_SIZE+ params: dict[str, str] = field(default_factory=dict)# Incremental sync is only advertised where the API exposes a confirmed server-side timestamp# filter. The v4 search body accepts filter expressions but their timestamp-filter fidelity is- # unverified, so both endpoints ship full refresh only.+ # unverified and the GET collections have no time filter, so every endpoint ships full refresh.supports_incremental: bool = Falsesupports_append: bool = Falseincremental_fields: list[IncrementalField] = field(default_factory=list)@@ -53,6 +65,30 @@ RAPID7_INSIGHTVM_ENDPOINTS: dict[str, Rapid7InsightvmEndpointConfig] = {path="vulnerabilities",primary_keys=["id"],),+ # Sites have no id in the v4 API; a site name is unique and is how assets reference their site.+ "sites": Rapid7InsightvmEndpointConfig(+ name="sites",+ path="sites",+ primary_keys=["name"],+ page_size=DOCUMENTED_MAX_PAGE_SIZE,+ ),+ "scans": Rapid7InsightvmEndpointConfig(+ name="scans",+ path="scan",+ primary_keys=["id"],+ method="GET",+ pagination="page_number",+ page_size=DOCUMENTED_MAX_PAGE_SIZE,+ params={"includeDetails": "true"},+ ),+ "scan_engines": Rapid7InsightvmEndpointConfig(+ name="scan_engines",+ path="scan/engine",+ primary_keys=["id"],+ method="GET",+ pagination="page_number",+ page_size=DOCUMENTED_MAX_PAGE_SIZE,+ ),}ENDPOINTS = tuple(RAPID7_INSIGHTVM_ENDPOINTS.keys())diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/rapid7_insightvm/tests/test_rapid7_insightvm.py b/products/warehouse_sources/backend/temporal/data_imports/sources/rapid7_insightvm/tests/test_rapid7_insightvm.pyindex 74bd9538c213..65b0a517cee9 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/rapid7_insightvm/tests/test_rapid7_insightvm.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/rapid7_insightvm/tests/test_rapid7_insightvm.py@@ -4,6 +4,7 @@ from typing import Anyimport pytestfrom requests.exceptions import HTTPError+from structlog.testing import capture_logsfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (DriveResult,@@ -28,6 +29,10 @@ def _page(items: list[dict[str, Any]] | None, cursor: str | None) -> ScriptedResreturn ScriptedResponse(json={"data": items or [], "metadata": {"cursor": cursor} if cursor is not None else {}})+def _numbered_page(items: list[dict[str, Any]], number: int, total_pages: int) -> ScriptedResponse:+ return ScriptedResponse(json={"data": items, "metadata": {"number": number, "totalPages": total_pages}})++def _error(status: int, body: dict[str, Any] | None = None) -> ScriptedResponse:return ScriptedResponse(status=status, json=body or {})@@ -36,9 +41,14 @@ def _redirect(location: str = "https://evil.example.com/") -> ScriptedResponse:return ScriptedResponse(status=302, headers={"Location": location})-def _run(responses: Sequence[ScriptedResponse | BaseException], api_key: str = "key", **inputs: Any) -> DriveResult:+def _run(+ responses: Sequence[ScriptedResponse | BaseException],+ api_key: str = "key",+ schema: str = "assets",+ **inputs: Any,+) -> DriveResult:config = Rapid7InsightvmSourceConfig(api_key=api_key, region="us")- return SourceDriver(Rapid7InsightvmSource(), config).run("assets", responses, **inputs)+ return SourceDriver(Rapid7InsightvmSource(), config).run(schema, responses, **inputs)class TestPagination:@@ -68,6 +78,42 @@ class TestPagination:assert result.params("cursor") == [None]+class TestPageNumberPagination:+ @pytest.mark.parametrize(+ "schema, total_pages, expected_requests, truncated",+ [+ ("scans", 2, 2, False),+ ("scan_engines", 2, 2, False),+ ("scans", 20, 20, False),+ # The API serves at most 10,000 results through page/size, so the walk stops at page 19.+ ("scans", 25, 20, True),+ ],+ )+ def test_walks_pages_with_get_until_the_last_page(+ self, schema: str, total_pages: int, expected_requests: int, truncated: bool+ ) -> None:+ pages = [_numbered_page([{"id": f"{schema}-{n}"}], n, total_pages) for n in range(expected_requests)]+ with capture_logs() as logs:+ result = _run(pages, schema=schema)++ assert result.raised is None+ assert {request.method for request in result.requests} == {"GET"}+ assert result.params("page") == [str(n) for n in range(expected_requests)]+ assert set(result.params("size")) == {"500"}+ assert len(result.rows) == expected_requests+ assert [state.page for state in result.saved_states] == list(range(1, expected_requests))+ assert any("10,000-result page window" in log["event"] for log in logs) is truncated++ def test_resumes_from_saved_page(self) -> None:+ result = _run(+ [_numbered_page([{"id": "s"}], 3, 4)],+ schema="scans",+ resume_state=Rapid7InsightvmResumeConfig(page=3),+ )++ assert result.params("page") == ["3"]++class TestRetryAndErrorClassification:def test_client_error_raises_http_error(self) -> None:# A 403 is permanent — surface it as an HTTPError instead of retrying.