Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add sites, scans and scan engines tables to rapid7 insightvm (#126977)

Tom Owers22d2793eb7a6
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 9c30402cda47..8ad54f6d461b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -10034,9 +10034,9 @@ Note: Ramp publishes a full OpenAPI spec (2.1 MB) plus llms-api.txt; the docs HT
Diffed against: <https://help.rapid7.com/insightvm/en-us/api/insightvm-api-v4.json>
-- [ ] `sites (POST /vm/v4/integration/sites)` — Lookup table resolving the site IDs carried on every asset we already sync (high)
-- [ ] `scan (GET /vm/v4/integration/scan)` — Scan history - when each site was assessed, which is needed to judge how stale asset and vulnerability data is (medium)
-- [ ] `scan/engine` — Lookup resolving the engine that produced each scan (low)
+- [x] `sites (POST /vm/v4/integration/sites)` — Lookup table resolving the site IDs carried on every asset we already sync (high)
+- [x] `scan (GET /vm/v4/integration/scan)` — Scan history - when each site was assessed, which is needed to judge how stale asset and vulnerability data is (medium)
+- [x] `scan/engine` — Lookup resolving the engine that produced each scan (low)
Note: PostHog targets the InsightVM Cloud Integrations API (/vm/v4/integration on \*.api.insight.rapid7.com), whose spec exposes only 5 resources: assets, sites, vulnerabilities, scan and scan/engine. So 2 of 5 is not as thin as it looks. The much larger on-prem console API v3 (207 paths - solutions, tags, asset_groups, policies, exploits, vulnerability_exceptions, per-asset findings) is a different product surface at help.rapid7.com/insightvm/en-us/api/api-v3.json requiring a console host and Basic auth; adding it would be a new source variant, not extra endpoints on this one. Endpoints are static (products/warehouse_sources/.../sources/rapid7_insightvm/settings.py), no dynamic discovery.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/rapid7_insightvm/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/rapid7_insightvm/canonical_descriptions.py
index c9729586bd00..6bba5114f48e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/rapid7_insightvm/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/rapid7_insightvm/canonical_descriptions.py
@@ -46,4 +46,38 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"malware_kits": "Known malware kits that leverage the vulnerability.",
},
},
+ "sites": {
+ "description": "A site in InsightVM: a named group of assets that are scanned together.",
+ "docs_url": "https://help.rapid7.com/insightvm/en-us/api/integrations.html",
+ "columns": {
+ "name": "Name of the site.",
+ "type": "Type of the record. Always `SITE` for sites.",
+ },
+ },
+ "scans": {
+ "description": "A scan run in InsightVM, with the engine that ran it, its status, and when it started and finished.",
+ "docs_url": "https://help.rapid7.com/insightvm/en-us/api/integrations.html",
+ "columns": {
+ "id": "Unique identifier for the scan.",
+ "name": "Name of the scan.",
+ "engine_id": "Identifier of the scan engine that ran the scan.",
+ "status": "Status of the scan (e.g. Running, Success, Failed).",
+ "started": "Timestamp when the scan started.",
+ "finished": "Timestamp when the scan finished.",
+ "details": "Additional details about the scan.",
+ },
+ },
+ "scan_engines": {
+ "description": "A scan engine registered with InsightVM, with its host, health status, and configuration profile.",
+ "docs_url": "https://help.rapid7.com/insightvm/en-us/api/integrations.html",
+ "columns": {
+ "id": "Unique identifier for the scan engine.",
+ "name": "Name of the scan engine.",
+ "host_name": "Host name or address of the scan engine.",
+ "status": "Health status of the scan engine (e.g. HEALTHY).",
+ "registered": "Timestamp when the scan engine was registered.",
+ "last_seen": "Timestamp when the scan engine last communicated with the platform.",
+ "profile": "Configuration profile of the scan engine, including custom properties.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/rapid7_insightvm/rapid7_insightvm.py b/products/warehouse_sources/backend/temporal/data_imports/sources/rapid7_insightvm/rapid7_insightvm.py
index fb780cd9989a..f88ca42cc612 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/rapid7_insightvm/rapid7_insightvm.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/rapid7_insightvm/rapid7_insightvm.py
@@ -2,6 +2,7 @@ import dataclasses
from typing import Any, Optional
from urllib.parse import urlencode
+import structlog
from requests import Response
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
@@ -13,23 +14,31 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res
find_values,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
+ BasePaginator,
JSONResponseCursorPaginator,
+ PageNumberPaginator,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import Endpoint
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.rapid7_insightvm.settings import (
API_BASE_PATH,
- MAX_PAGE_SIZE,
+ PAGE_NUMBER_MAX_PAGE,
RAPID7_INSIGHTVM_ENDPOINTS,
REGION_HOSTS,
+ Rapid7InsightvmEndpointConfig,
)
+logger = structlog.get_logger(__name__)
+
[email protected](frozen=True)
class Rapid7InsightvmResumeConfig:
# Opaque cursor token returned in the previous page's `metadata.cursor`. `None` starts the
# endpoint from its first page.
cursor: str | None = None
+ # Next zero-based page for the page-number endpoints (scans, scan engines).
+ page: int | None = None
class Rapid7InsightvmCursorPaginator(JSONResponseCursorPaginator):
@@ -58,6 +67,25 @@ class Rapid7InsightvmCursorPaginator(JSONResponseCursorPaginator):
self._has_next_page = True
+class Rapid7InsightvmPageNumberPaginator(PageNumberPaginator):
+ """Page-number pagination that warns when the collection outgrows the API's 10,000-result window."""
+
+ def __init__(self) -> None:
+ super().__init__(base_page=0, total_path="metadata.totalPages", maximum_page=PAGE_NUMBER_MAX_PAGE)
+
+ def update_state(self, response: Response, data: Optional[list[Any]] = None) -> None:
+ super().update_state(response, data)
+ if self._has_next_page or self.page <= PAGE_NUMBER_MAX_PAGE:
+ return
+ total_pages = (response.json().get("metadata") or {}).get("totalPages")
+ if isinstance(total_pages, int) and total_pages > PAGE_NUMBER_MAX_PAGE + 1:
+ logger.warning(
+ "Rapid7 InsightVM collection exceeds the 10,000-result page window, truncating",
+ path=response.request.path_url if response.request else None,
+ total_pages=total_pages,
+ )
+
+
def _host(region: str) -> str:
return REGION_HOSTS.get(region, REGION_HOSTS["us"])
@@ -74,6 +102,25 @@ def _endpoint_url(region: str, path: str) -> str:
return f"{_host(region)}{API_BASE_PATH}/{path}"
+def _paginator(config: Rapid7InsightvmEndpointConfig) -> BasePaginator:
+ if config.pagination == "page_number":
+ return Rapid7InsightvmPageNumberPaginator()
+ return Rapid7InsightvmCursorPaginator()
+
+
+def _endpoint(config: Rapid7InsightvmEndpointConfig) -> Endpoint:
+ endpoint: Endpoint = {
+ "method": config.method,
+ "path": f"{API_BASE_PATH}/{config.path}",
+ "params": {"size": config.page_size, **config.params},
+ "data_selector": "data",
+ }
+ if config.method == "POST":
+ # v4 search endpoints are POST operations; an empty JSON body returns all resources.
+ endpoint["json"] = {}
+ return endpoint
+
+
def rapid7_insightvm_source(
api_key: str,
region: str,
@@ -92,7 +139,7 @@ def rapid7_insightvm_source(
# logs and raised error messages; only the non-secret accept/content headers are set here.
"headers": {"Accept": "application/json", "Content-Type": "application/json"},
"auth": {"type": "api_key", "api_key": api_key, "name": "X-Api-Key", "location": "header"},
- "paginator": Rapid7InsightvmCursorPaginator(),
+ "paginator": _paginator(config),
# A 3xx from the credentialed endpoint would otherwise replay `X-Api-Key` to the redirect
# target — reject redirects so the key stays pinned to the expected host.
"allow_redirects": False,
@@ -101,14 +148,7 @@ def rapid7_insightvm_source(
"resources": [
{
"name": endpoint,
- "endpoint": {
- # v4 search endpoints are POST operations; an empty JSON body returns all resources.
- "method": "POST",
- "path": f"{API_BASE_PATH}/{config.path}",
- "params": {"size": MAX_PAGE_SIZE},
- "json": {},
- "data_selector": "data",
- },
+ "endpoint": _endpoint(config),
}
],
}
@@ -118,12 +158,16 @@ def rapid7_insightvm_source(
resume = resumable_source_manager.load_state()
if resume is not None and resume.cursor is not None:
initial_paginator_state = {"cursor": resume.cursor}
+ elif resume is not None and resume.page is not None:
+ initial_paginator_state = {"page": resume.page}
def save_checkpoint(state: Optional[dict[str, Any]]) -> None:
# Persist only when a next cursor remains; save AFTER a page is yielded so a crash re-yields
# the last page (merge dedupes on the primary key) rather than skipping it.
if state and state.get("cursor") is not None:
resumable_source_manager.save_state(Rapid7InsightvmResumeConfig(cursor=str(state["cursor"])))
+ elif state and state.get("page") is not None:
+ resumable_source_manager.save_state(Rapid7InsightvmResumeConfig(page=int(state["page"])))
resource = rest_api_resource(
rest_config,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/rapid7_insightvm/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/rapid7_insightvm/settings.py
index ff77173428f9..a959231a4975 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/rapid7_insightvm/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/rapid7_insightvm/settings.py
@@ -1,4 +1,5 @@
from dataclasses import dataclass, field
+from typing import Literal
from products.warehouse_sources.backend.types import IncrementalField
@@ -20,16 +21,27 @@ REGION_HOSTS: dict[str, str] = {
API_BASE_PATH = "/vm/v4/integration"
MAX_PAGE_SIZE = 1000
+# The v4 docs cap a page at 500 rows. The GET collections (scans, scan engines) page with zero-based
+# `page` + `size` and no cursor, which serves at most 10,000 results, so the walk stops at the last
+# page inside that window instead of requesting one the API refuses.
+DOCUMENTED_MAX_PAGE_SIZE = 500
+PAGE_NUMBER_MAX_PAGE = 10_000 // DOCUMENTED_MAX_PAGE_SIZE - 1
-@dataclass
+
+@dataclass(frozen=True)
class Rapid7InsightvmEndpointConfig:
name: str
# Path segment appended to `{host}{API_BASE_PATH}/` (e.g. "assets").
path: str
primary_keys: list[str]
+ # Search endpoints are POST + cursor; plain collection endpoints are GET + page number.
+ method: Literal["GET", "POST"] = "POST"
+ pagination: Literal["cursor", "page_number"] = "cursor"
+ page_size: int = MAX_PAGE_SIZE
+ params: dict[str, str] = field(default_factory=dict)
# Incremental sync is only advertised where the API exposes a confirmed server-side timestamp
# filter. The v4 search body accepts filter expressions but their timestamp-filter fidelity is
- # unverified, so both endpoints ship full refresh only.
+ # unverified and the GET collections have no time filter, so every endpoint ships full refresh.
supports_incremental: bool = False
supports_append: bool = False
incremental_fields: list[IncrementalField] = field(default_factory=list)
@@ -53,6 +65,30 @@ RAPID7_INSIGHTVM_ENDPOINTS: dict[str, Rapid7InsightvmEndpointConfig] = {
path="vulnerabilities",
primary_keys=["id"],
),
+ # Sites have no id in the v4 API; a site name is unique and is how assets reference their site.
+ "sites": Rapid7InsightvmEndpointConfig(
+ name="sites",
+ path="sites",
+ primary_keys=["name"],
+ page_size=DOCUMENTED_MAX_PAGE_SIZE,
+ ),
+ "scans": Rapid7InsightvmEndpointConfig(
+ name="scans",
+ path="scan",
+ primary_keys=["id"],
+ method="GET",
+ pagination="page_number",
+ page_size=DOCUMENTED_MAX_PAGE_SIZE,
+ params={"includeDetails": "true"},
+ ),
+ "scan_engines": Rapid7InsightvmEndpointConfig(
+ name="scan_engines",
+ path="scan/engine",
+ primary_keys=["id"],
+ method="GET",
+ pagination="page_number",
+ page_size=DOCUMENTED_MAX_PAGE_SIZE,
+ ),
}
ENDPOINTS = tuple(RAPID7_INSIGHTVM_ENDPOINTS.keys())
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/rapid7_insightvm/tests/test_rapid7_insightvm.py b/products/warehouse_sources/backend/temporal/data_imports/sources/rapid7_insightvm/tests/test_rapid7_insightvm.py
index 74bd9538c213..65b0a517cee9 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/rapid7_insightvm/tests/test_rapid7_insightvm.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/rapid7_insightvm/tests/test_rapid7_insightvm.py
@@ -4,6 +4,7 @@ from typing import Any
import pytest
from requests.exceptions import HTTPError
+from structlog.testing import capture_logs
from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
DriveResult,
@@ -28,6 +29,10 @@ def _page(items: list[dict[str, Any]] | None, cursor: str | None) -> ScriptedRes
return ScriptedResponse(json={"data": items or [], "metadata": {"cursor": cursor} if cursor is not None else {}})
+def _numbered_page(items: list[dict[str, Any]], number: int, total_pages: int) -> ScriptedResponse:
+ return ScriptedResponse(json={"data": items, "metadata": {"number": number, "totalPages": total_pages}})
+
+
def _error(status: int, body: dict[str, Any] | None = None) -> ScriptedResponse:
return ScriptedResponse(status=status, json=body or {})
@@ -36,9 +41,14 @@ def _redirect(location: str = "https://evil.example.com/") -> ScriptedResponse:
return ScriptedResponse(status=302, headers={"Location": location})
-def _run(responses: Sequence[ScriptedResponse | BaseException], api_key: str = "key", **inputs: Any) -> DriveResult:
+def _run(
+ responses: Sequence[ScriptedResponse | BaseException],
+ api_key: str = "key",
+ schema: str = "assets",
+ **inputs: Any,
+) -> DriveResult:
config = Rapid7InsightvmSourceConfig(api_key=api_key, region="us")
- return SourceDriver(Rapid7InsightvmSource(), config).run("assets", responses, **inputs)
+ return SourceDriver(Rapid7InsightvmSource(), config).run(schema, responses, **inputs)
class TestPagination:
@@ -68,6 +78,42 @@ class TestPagination:
assert result.params("cursor") == [None]
+class TestPageNumberPagination:
+ @pytest.mark.parametrize(
+ "schema, total_pages, expected_requests, truncated",
+ [
+ ("scans", 2, 2, False),
+ ("scan_engines", 2, 2, False),
+ ("scans", 20, 20, False),
+ # The API serves at most 10,000 results through page/size, so the walk stops at page 19.
+ ("scans", 25, 20, True),
+ ],
+ )
+ def test_walks_pages_with_get_until_the_last_page(
+ self, schema: str, total_pages: int, expected_requests: int, truncated: bool
+ ) -> None:
+ pages = [_numbered_page([{"id": f"{schema}-{n}"}], n, total_pages) for n in range(expected_requests)]
+ with capture_logs() as logs:
+ result = _run(pages, schema=schema)
+
+ assert result.raised is None
+ assert {request.method for request in result.requests} == {"GET"}
+ assert result.params("page") == [str(n) for n in range(expected_requests)]
+ assert set(result.params("size")) == {"500"}
+ assert len(result.rows) == expected_requests
+ assert [state.page for state in result.saved_states] == list(range(1, expected_requests))
+ assert any("10,000-result page window" in log["event"] for log in logs) is truncated
+
+ def test_resumes_from_saved_page(self) -> None:
+ result = _run(
+ [_numbered_page([{"id": "s"}], 3, 4)],
+ schema="scans",
+ resume_state=Rapid7InsightvmResumeConfig(page=3),
+ )
+
+ assert result.params("page") == ["3"]
+
+
class TestRetryAndErrorClassification:
def test_client_error_raises_http_error(self) -> None:
# A 403 is permanent — surface it as an HTTPError instead of retrying.