Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add 4 missing tables to the TenableVulnerabilityManagement source (#120634)

Tom Owers49fa307f79c7
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
index 0b2218d753b8..6d0d633b95e6 100644
--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
@@ -1114,7 +1114,6 @@
1 products/warehouse_sources/backend/temporal/data_imports/sources/telnyx/telnyx.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/tempo/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/tempo/tempo.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/tenable_vulnerability_management/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/tenable_vulnerability_management/tenable_vulnerability_management.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/terraform_cloud/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/terraform_cloud/terraform_cloud.py
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 30488d25f18e..443730e000ba 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -12202,10 +12202,10 @@ Today (3): `assets`, `scans`, `vulnerabilities`
Diffed against: <https://developer.tenable.com/reference/exports-assets-request-export>
-- [ ] `plugins (GET /plugins/plugin)` — lookup resolving the plugin_id carried on every synced vulnerability finding to name, family, CVE, CVSS and VPR (high)
-- [ ] `scan history (GET /scans/{scan_id}/history)` — each individual run of a scan with status and timing; the scans table alone has no run-level history (high)
-- [ ] `compliance findings export (POST /compliance/export + status/chunks)` — audit/compliance results, the second half of Tenable's findings data alongside vulnerabilities (high)
-- [ ] `tag values (GET /tags/values)` — lookup table for the asset tags used to slice every asset and vulnerability report (high)
+- [x] `plugins (GET /plugins/plugin)` — lookup resolving the plugin_id carried on every synced vulnerability finding to name, family, CVE, CVSS and VPR (high)
+- [x] `scan history (GET /scans/{scan_id}/history)` — each individual run of a scan with status and timing; the scans table alone has no run-level history (high)
+- [x] `compliance findings export (POST /compliance/export + status/chunks)` — audit/compliance results, the second half of Tenable's findings data alongside vulnerabilities (high)
+- [x] `tag values (GET /tags/values)` — lookup table for the asset tags used to slice every asset and vulnerability report (high)
- [ ] `tag categories (GET /tags/categories)` — parent dimension for tag values, needed to group tags into category/value pairs (medium)
- [ ] `plugin families (GET /plugins/families)` — grouping dimension for plugins, standard breakdown axis in vulnerability reporting (medium)
- [ ] `networks (GET /networks)` — lookup resolving the network_id on assets and scanners (medium)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/tenable_vulnerability_management/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/tenable_vulnerability_management/canonical_descriptions.py
index 9c4c97e803d3..9dfb78e282cb 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/tenable_vulnerability_management/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/tenable_vulnerability_management/canonical_descriptions.py
@@ -75,4 +75,66 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"last_modification_date": "When the scan was last modified, as a Unix timestamp.",
},
},
+ "plugins": {
+ "description": "Tenable plugins, one row per plugin with the detection details used to resolve the plugin_id on vulnerability findings.",
+ "docs_url": "https://developer.tenable.com/reference/io-plugins-list",
+ "columns": {
+ "id": "ID of the plugin.",
+ "name": "Name of the plugin.",
+ "attributes": "Plugin details, such as description, solution, CVEs, CVSS scores, VPR, and publication dates.",
+ "plugin_modification_date": "When Tenable last updated the plugin.",
+ },
+ },
+ "scan_history": {
+ "description": "Scan history, one row per individual run of a scan.",
+ "docs_url": "https://developer.tenable.com/reference/scans-history",
+ "columns": {
+ "scan_id": "ID of the scan configuration that the run belongs to.",
+ "scan_schedule_uuid": "Schedule UUID of the scan configuration that the run belongs to.",
+ "scan_uuid": "UUID of the specific scan run.",
+ "id": "Unique ID of the scan run. Tenable does not return this ID for every run.",
+ "status": "Status of the scan run.",
+ "time_start": "When the scan run started, as a Unix timestamp.",
+ "time_end": "When the scan run stopped, as a Unix timestamp.",
+ "is_archived": "Whether the results of the scan run are archived.",
+ "visibility": "Visibility of the scan run results.",
+ "targets": "Whether the run used the default scan targets or custom targets.",
+ "reporting_mode": "Reporting mode of the scan run.",
+ },
+ },
+ "compliance_findings": {
+ "description": "Compliance findings from Tenable's compliance export, one row per audit check result on an asset (asset, check, and audit file).",
+ "docs_url": "https://developer.tenable.com/reference/io-exports-compliance-create",
+ "columns": {
+ "asset_uuid": "UUID of the asset on which the compliance check ran.",
+ "compliance_full_id": "Hash that identifies the content of the compliance check.",
+ "audit_file": "Name of the audit file that contains the compliance check.",
+ "check_id": "Unique identifier for the compliance finding.",
+ "check_name": "Name of the compliance check.",
+ "status": "Result of the compliance check: PASSED, FAILED, WARNING, SKIPPED, ERROR, or UNKNOWN.",
+ "state": "State of the finding: OPEN, REOPENED, or FIXED.",
+ "plugin_id": "ID of the compliance plugin.",
+ "first_seen": "When a compliance scan first assessed the asset with the check.",
+ "last_seen": "When a compliance scan most recently assessed the asset with the check.",
+ "last_observed": "When the finding was last observed on the asset, whether active or fixed.",
+ "last_fixed": "When the finding was last fixed on the asset.",
+ "indexed_at": "When the audit for the asset was indexed into Tenable.",
+ "asset": "Details about the asset, such as name, IP addresses, operating systems, and tags.",
+ },
+ },
+ "tag_values": {
+ "description": "Asset tag values, one row per value with the tag category it belongs to.",
+ "docs_url": "https://developer.tenable.com/reference/tags-list-tag-values",
+ "columns": {
+ "uuid": "UUID of the tag value.",
+ "value": "The tag value.",
+ "description": "Description of the tag value.",
+ "type": "Type of the tag value: static or dynamic.",
+ "category_uuid": "UUID of the tag category.",
+ "category_name": "Name of the tag category.",
+ "category_description": "Description of the tag category.",
+ "created_at": "When the tag value was created.",
+ "updated_at": "When the tag value was last updated.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/tenable_vulnerability_management/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/tenable_vulnerability_management/settings.py
index f281737d2c25..ebb770b25d40 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/tenable_vulnerability_management/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/tenable_vulnerability_management/settings.py
@@ -1,10 +1,10 @@
from dataclasses import dataclass, field
-from typing import Any, Optional
+from typing import Any, Literal, Optional
from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType
-@dataclass
+@dataclass(frozen=True)
class TenableEndpointConfig:
name: str
primary_keys: list[str]
@@ -20,8 +20,14 @@ class TenableEndpointConfig:
# Static body for the export request; the incremental filter is merged in per run.
export_payload: dict[str, Any] = field(default_factory=dict)
path: Optional[str] = None
- # Key holding the row list in a plain (non-export) endpoint's JSON response.
+ # Dotted path to the row list in a plain (non-export) endpoint's JSON response.
data_key: Optional[str] = None
+ # "page" sends 1-based `page` + `size`; "offset" sends `offset` + `limit`. None is a single request.
+ pagination: Optional[Literal["page", "offset"]] = None
+ page_size: int = 0
+ params: dict[str, Any] = field(default_factory=dict)
+ # Query param carrying the incremental watermark as a `YYYY-MM-DD` date on a plain endpoint.
+ since_date_param: Optional[str] = None
TENABLE_ENDPOINTS: dict[str, TenableEndpointConfig] = {
@@ -72,6 +78,68 @@ TENABLE_ENDPOINTS: dict[str, TenableEndpointConfig] = {
primary_keys=["id"],
incremental_fields=[],
),
+ # Plugin catalog, the lookup for the plugin_id on every vulnerability finding. `last_updated`
+ # ignores VPR changes, so an incremental run keeps the VPR of unchanged plugins as last synced.
+ "plugins": TenableEndpointConfig(
+ name="plugins",
+ path="/plugins/plugin",
+ data_key="data.plugin_details",
+ pagination="page",
+ page_size=1000,
+ since_date_param="last_updated",
+ primary_keys=["id"],
+ incremental_fields=[
+ {
+ "label": "plugin_modification_date",
+ "type": IncrementalFieldType.DateTime,
+ "field": "plugin_modification_date",
+ "field_type": IncrementalFieldType.DateTime,
+ },
+ ],
+ ),
+ # One row per run of each scan, fanned out over the scans list. The history endpoint has no
+ # time filter, so this table is full refresh only. Tenable omits `id` for some runs, so the
+ # per-run `scan_uuid` is the key.
+ "scan_history": TenableEndpointConfig(
+ name="scan_history",
+ path="/scans/{scan_id}/history",
+ data_key="history",
+ pagination="offset",
+ page_size=50,
+ params={"sort": "start_date:asc"},
+ primary_keys=["scan_id", "scan_uuid"],
+ ),
+ # Compliance (audit) findings. Tenable documents asset_uuid + compliance_full_id + audit_file
+ # as the composite key. No state filter is sent: without one, `since` returns open and reopened
+ # findings seen since the watermark plus findings fixed since it.
+ "compliance_findings": TenableEndpointConfig(
+ name="compliance_findings",
+ export_path="compliance",
+ since_filter="since",
+ export_payload={"num_findings": 5000},
+ primary_keys=["asset_uuid", "compliance_full_id", "audit_file"],
+ partition_key="first_seen",
+ incremental_fields=[
+ {
+ "label": "indexed_at",
+ "type": IncrementalFieldType.DateTime,
+ "field": "indexed_at",
+ "field_type": IncrementalFieldType.DateTime,
+ },
+ ],
+ ),
+ # Tag values with their category. The `updated_at` filter's timestamp unit is undocumented, so
+ # this small lookup table is full refresh only. A value is unique within its category, so the
+ # sort gives a stable order across pages.
+ "tag_values": TenableEndpointConfig(
+ name="tag_values",
+ path="/tags/values",
+ data_key="values",
+ pagination="offset",
+ page_size=5000,
+ params={"sort": "category_name:asc,value:asc"},
+ primary_keys=["uuid"],
+ ),
}
ENDPOINTS = tuple(TENABLE_ENDPOINTS.keys())
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/tenable_vulnerability_management/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/tenable_vulnerability_management/source.py
index 1768ac8d2cb9..c0923509ac33 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/tenable_vulnerability_management/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/tenable_vulnerability_management/source.py
@@ -142,6 +142,20 @@ Generate an API key pair (access key and secret key) from **Settings > My Accoun
)
if endpoint == "scans":
return "Scan configurations with their schedule and latest run status"
+ if endpoint == "plugins":
+ return (
+ "One row per Tenable plugin with its name, family, CVEs, CVSS and VPR. Incremental syncs pick "
+ "up plugins modified since the last run, but not plugins whose only change is a new VPR"
+ )
+ if endpoint == "scan_history":
+ return "One row per run of each scan, with its status and start and end times"
+ if endpoint == "compliance_findings":
+ return (
+ "One row per compliance check result (asset, check, audit file). Synced via Tenable's "
+ "asynchronous export API; incremental syncs pick up findings indexed since the last run"
+ )
+ if endpoint == "tag_values":
+ return "Asset tag values with their tag category"
return None
def _build_schema(endpoint: str) -> SourceSchema:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/tenable_vulnerability_management/tenable_vulnerability_management.py b/products/warehouse_sources/backend/temporal/data_imports/sources/tenable_vulnerability_management/tenable_vulnerability_management.py
index d74048e83dba..156ef520aef5 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/tenable_vulnerability_management/tenable_vulnerability_management.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/tenable_vulnerability_management/tenable_vulnerability_management.py
@@ -1,8 +1,9 @@
import time
import dataclasses
from collections.abc import Iterator
-from datetime import UTC, date, datetime
+from datetime import UTC, date, datetime, timedelta
from typing import Any, Optional
+from urllib.parse import urlencode
import requests
from structlog.types import FilteringBoundLogger
@@ -105,6 +106,15 @@ def _to_unix_timestamp(value: Any) -> int | None:
return None
+def _to_since_date(value: Any) -> str | None:
+ """Convert an incremental watermark to the `YYYY-MM-DD` date a plain endpoint filters on.
+ The filter is day-granular and exclusive, so it starts one day early to keep same-day updates."""
+ timestamp = _to_unix_timestamp(value)
+ if timestamp is None:
+ return None
+ return (datetime.fromtimestamp(timestamp, tz=UTC).date() - timedelta(days=1)).isoformat()
+
+
@retry(
retry=retry_if_exception_type(
(
@@ -224,9 +234,17 @@ def _normalize_vulnerability(row: dict[str, Any]) -> dict[str, Any]:
return row
+def _normalize_plugin(row: dict[str, Any]) -> dict[str, Any]:
+ # The incremental field must be a top-level column for the pipeline to track its watermark.
+ row["plugin_modification_date"] = (row.get("attributes") or {}).get("plugin_modification_date")
+ return row
+
+
def _normalize_rows(endpoint: str, rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
if endpoint == "vulnerabilities":
return [_normalize_vulnerability(row) for row in rows]
+ if endpoint == "plugins":
+ return [_normalize_plugin(row) for row in rows]
return rows
@@ -305,18 +323,82 @@ def _get_export_rows(
status = None
+def _extract_rows(data: Any, data_key: str | None) -> list[dict[str, Any]]:
+ if data_key:
+ for key in data_key.split("."):
+ data = data.get(key) if isinstance(data, dict) else None
+ return data if isinstance(data, list) else []
+
+
def _get_plain_rows(
session: requests.Session,
base_url: str,
+ endpoint: str,
config: TenableEndpointConfig,
headers: dict[str, str],
logger: FilteringBoundLogger,
+ path: str | None = None,
+ since_date: str | None = None,
) -> Iterator[list[dict[str, Any]]]:
- response = _request(session, "GET", f"{base_url}{config.path}", headers, logger)
- data = response.json()
- rows = data.get(config.data_key) if config.data_key else data
- if rows:
- yield rows
+ params = dict(config.params)
+ if since_date is not None and config.since_date_param:
+ params[config.since_date_param] = since_date
+ page = 0
+
+ while True:
+ if config.pagination == "page":
+ params.update({"page": page + 1, "size": config.page_size})
+ elif config.pagination == "offset":
+ params.update({"offset": page * config.page_size, "limit": config.page_size})
+
+ url = f"{base_url}{path or config.path}"
+ if params:
+ url = f"{url}?{urlencode(params)}"
+ rows = _extract_rows(_request(session, "GET", url, headers, logger).json(), config.data_key)
+ if rows:
+ yield _normalize_rows(endpoint, rows)
+
+ if config.pagination is None or len(rows) < config.page_size:
+ return
+ page += 1
+
+
+def _get_scan_history_rows(
+ session: requests.Session,
+ base_url: str,
+ endpoint: str,
+ config: TenableEndpointConfig,
+ headers: dict[str, str],
+ resumable_source_manager: ResumableSourceManager[TenableResumeConfig],
+ logger: FilteringBoundLogger,
+) -> Iterator[list[dict[str, Any]]]:
+ scans_config = TENABLE_ENDPOINTS["scans"]
+ history_path = config.path or ""
+ scans = _extract_rows(
+ _request(session, "GET", f"{base_url}{scans_config.path}", headers, logger).json(), scans_config.data_key
+ )
+
+ for scan in scans:
+ # Tenable recommends the schedule UUID over the numeric ID to address a scan.
+ scan_ref = scan.get("schedule_uuid") or scan.get("id")
+ if scan_ref is None:
+ continue
+ parent_fields = {"scan_id": scan.get("id"), "scan_schedule_uuid": scan.get("schedule_uuid")}
+ yielded = False
+ try:
+ for rows in _get_plain_rows(
+ session, base_url, endpoint, config, headers, logger, path=history_path.format(scan_id=scan_ref)
+ ):
+ yielded = True
+ yield [{**row, **parent_fields} for row in rows]
+ except requests.HTTPError as exc:
+ # A scan deleted after the scans list was read has no history to sync.
+ if exc.response is None or exc.response.status_code != 404:
+ raise
+ logger.debug(f"Tenable: scan {scan_ref} was not found, skipping its history")
+ if not yielded:
+ # The scan history run keeps no resume state, so no row is held back here.
+ resumable_source_manager.safe_point()
def get_rows(
@@ -334,8 +416,15 @@ def get_rows(
headers = _get_headers(access_key, secret_key)
session = _make_session(access_key, secret_key)
+ if endpoint == "scan_history":
+ yield from _get_scan_history_rows(
+ session, base_url, endpoint, config, headers, resumable_source_manager, logger
+ )
+ return
+
if config.export_path is None:
- yield from _get_plain_rows(session, base_url, config, headers, logger)
+ since_date = _to_since_date(db_incremental_field_last_value) if should_use_incremental_field else None
+ yield from _get_plain_rows(session, base_url, endpoint, config, headers, logger, since_date=since_date)
return
since = _to_unix_timestamp(db_incremental_field_last_value) if should_use_incremental_field else None
@@ -370,7 +459,10 @@ def tenable_source(
# Export chunks carry no ordering guarantee, so the incremental watermark (max value seen)
# is only persisted once the whole run succeeds — per-batch checkpointing would advance it
# past rows a crashed run still owes.
- sort_mode="desc" if config.export_path else "asc",
+ # Plain endpoints page by ID or name, not by the incremental field, so their order is unknown.
+ sort_mode="desc" if config.export_path else None,
+ # Only export reads checkpoint their position; a plain or fan-out read restarts from scratch.
+ supports_resume=bool(config.export_path),
partition_count=1 if config.partition_key else None,
partition_size=1 if config.partition_key else None,
partition_mode="datetime" if config.partition_key else None,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/tenable_vulnerability_management/tests/test_tenable_vulnerability_management.py b/products/warehouse_sources/backend/temporal/data_imports/sources/tenable_vulnerability_management/tests/test_tenable_vulnerability_management.py
index ee4604cd4a76..85528a14097a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/tenable_vulnerability_management/tests/test_tenable_vulnerability_management.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/tenable_vulnerability_management/tests/test_tenable_vulnerability_management.py
@@ -28,6 +28,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.tenable_vu
TenableResumeConfig,
_base_url,
_build_export_payload,
+ _to_since_date,
_to_unix_timestamp,
validate_credentials,
)
@@ -81,6 +82,20 @@ class TestToUnixTimestamp:
assert _to_unix_timestamp(value) == expected
+class TestToSinceDate:
+ @parameterized.expand(
+ [
+ ("midnight_starts_previous_day", datetime(2026, 7, 1, 0, 0, 0, tzinfo=UTC), "2026-06-30"),
+ ("late_in_day_starts_previous_day", "2026-07-01T23:59:59Z", "2026-06-30"),
+ ("month_boundary", date(2026, 3, 1), "2026-02-28"),
+ ("garbage_string", "not-a-date", None),
+ ("none", None, None),
+ ]
+ )
+ def test_conversion(self, _name: str, value: Any, expected: str | None) -> None:
+ assert _to_since_date(value) == expected
+
+
class TestBaseUrl:
@parameterized.expand(
[
@@ -99,9 +114,21 @@ class TestBaseUrl:
class TestBuildExportPayload:
- def test_assets_incremental_adds_updated_at(self) -> None:
- payload = _build_export_payload(TENABLE_ENDPOINTS["assets"], since=1782864000)
- assert payload == {"chunk_size": 1000, "filters": {"updated_at": 1782864000}}
+ @parameterized.expand(
+ [
+ ("assets_incremental", "assets", 1782864000, {"chunk_size": 1000, "filters": {"updated_at": 1782864000}}),
+ (
+ "compliance_incremental",
+ "compliance_findings",
+ 1782864000,
+ {"num_findings": 5000, "filters": {"since": 1782864000}},
+ ),
+ # A state filter would drop findings last seen before Tenable added states (state is NULL).
+ ("compliance_full_refresh", "compliance_findings", None, {"num_findings": 5000}),
+ ]
+ )
+ def test_payload(self, _name: str, endpoint: str, since: int | None, expected: dict[str, Any]) -> None:
+ assert _build_export_payload(TENABLE_ENDPOINTS[endpoint], since=since) == expected
class TestValidateCredentials:
@@ -233,6 +260,202 @@ class TestExportFlow:
assert result.final_state is None
+class TestCompliance:
+ def test_runs_the_compliance_export_flow(self) -> None:
+ row = {"asset_uuid": "a-1", "compliance_full_id": "c-1", "audit_file": "x.audit", "status": "FAILED"}
+ result = _run_export(
+ [
+ ScriptedResponse(json={"export_uuid": "u1"}),
+ ScriptedResponse(json={"status": "READY", "chunks_available": [1]}),
+ ScriptedResponse(json=[row]),
+ ScriptedResponse(json={"status": "FINISHED", "chunks_available": [1]}),
+ ],
+ endpoint="compliance_findings",
+ should_use_incremental_field=True,
+ db_incremental_field_last_value="2026-07-01T00:00:00Z",
+ )
+
+ assert result.raised is None
+ assert result.rows == [row]
+ assert result.paths == [
+ "/compliance/export",
+ "/compliance/export/u1/status",
+ "/compliance/export/u1/chunks/1",
+ "/compliance/export/u1/status",
+ ]
+ assert result.requests[0].json() == {"num_findings": 5000, "filters": {"since": 1782864000}}
+
+
+def _plugin(plugin_id: int) -> dict[str, Any]:
+ return {
+ "id": plugin_id,
+ "name": f"Plugin {plugin_id}",
+ "attributes": {"plugin_modification_date": "2026-06-15T00:00:00Z", "cve": ["CVE-2026-0001"]},
+ }
+
+
+def _plugins_page(plugin_ids: range) -> dict[str, Any]:
+ plugins = [_plugin(plugin_id) for plugin_id in plugin_ids]
+ return {"data": {"plugin_details": plugins}, "size": len(plugins), "total_count": 1500}
+
+
+def _tag_values_page(start: int, count: int) -> dict[str, Any]:
+ values = [{"uuid": f"t-{i}", "category_name": "location", "value": f"v{i}"} for i in range(start, start + count)]
+ return {"values": values, "pagination": {"total": 5002, "offset": start, "limit": 5000}}
+
+
+class TestPagedEndpoints:
+ @parameterized.expand(
+ [
+ (
+ "plugins_by_page",
+ "plugins",
+ [_plugins_page(range(1, 1001)), _plugins_page(range(1001, 1501))],
+ [{"page": ("1",), "size": ("1000",)}, {"page": ("2",), "size": ("1000",)}],
+ 1500,
+ ),
+ (
+ "tag_values_by_offset",
+ "tag_values",
+ [_tag_values_page(0, 5000), _tag_values_page(5000, 2)],
+ [
+ {"sort": ("category_name:asc,value:asc",), "offset": ("0",), "limit": ("5000",)},
+ {"sort": ("category_name:asc,value:asc",), "offset": ("5000",), "limit": ("5000",)},
+ ],
+ 5002,
+ ),
+ (
+ "full_page_then_empty_page",
+ "plugins",
+ [_plugins_page(range(1, 1001)), {"data": {"plugin_details": []}, "size": 0, "total_count": 1000}],
+ [{"page": ("1",), "size": ("1000",)}, {"page": ("2",), "size": ("1000",)}],
+ 1000,
+ ),
+ ]
+ )
+ def test_walks_pages_until_a_short_page(
+ self,
+ _name: str,
+ endpoint: str,
+ pages: list[dict[str, Any]],
+ expected_queries: list[dict[str, tuple[str, ...]]],
+ expected_rows: int,
+ ) -> None:
+ result = _run_export([ScriptedResponse(json=page) for page in pages], endpoint=endpoint)
+
+ assert result.raised is None
+ assert result.queries == expected_queries
+ assert len(result.rows) == expected_rows
+
+ @parameterized.expand(
+ [
+ ("incremental", True, "2026-07-01T10:00:00Z", "2026-06-30"),
+ ("full_refresh_ignores_stale_cursor", False, "2026-07-01T10:00:00Z", None),
+ ("first_incremental_run", True, None, None),
+ ]
+ )
+ def test_plugins_last_updated_filter(
+ self, _name: str, incremental: bool, last_value: str | None, expected: str | None
+ ) -> None:
+ result = _run_export(
+ [ScriptedResponse(json=_plugins_page(range(1, 3)))],
+ endpoint="plugins",
+ should_use_incremental_field=incremental,
+ db_incremental_field_last_value=last_value,
+ )
+
+ assert result.params("last_updated") == [expected]
+
+ def test_plugins_lift_modification_date_to_top_level(self) -> None:
+ result = _run_export([ScriptedResponse(json=_plugins_page(range(7, 8)))], endpoint="plugins")
+
+ assert result.rows[0]["plugin_modification_date"] == "2026-06-15T00:00:00Z"
+ assert result.rows[0]["attributes"]["cve"] == ["CVE-2026-0001"]
+
+ @parameterized.expand(
+ [
+ ("missing_data_key", {"size": 0, "total_count": 0}),
+ ("null_list", {"data": {"plugin_details": None}}),
+ ]
+ )
+ def test_empty_response_yields_nothing(self, _name: str, body: dict[str, Any]) -> None:
+ result = _run_export([ScriptedResponse(json=body)], endpoint="plugins")
+
+ assert result.raised is None
+ assert result.rows == []
+
+
+def _history_run(scan_uuid: str) -> dict[str, Any]:
+ return {"scan_uuid": scan_uuid, "status": "completed", "time_start": 1782864000, "time_end": 1782864600}
+
+
+class TestScanHistory:
+ def test_fans_out_over_scans_and_tags_rows_with_the_parent(self) -> None:
+ def respond(request: RecordedRequest) -> ScriptedResponse:
+ if request.path == "/scans":
+ return ScriptedResponse(
+ json={
+ "scans": [
+ {"id": 1, "schedule_uuid": "template-aaa"},
+ {"id": 2, "schedule_uuid": None},
+ {"id": 3, "schedule_uuid": "template-deleted"},
+ ]
+ }
+ )
+ if request.path == "/scans/template-aaa/history":
+ return ScriptedResponse(json={"history": [_history_run("run-1"), _history_run("run-2")]})
+ if request.path == "/scans/2/history":
+ return ScriptedResponse(json={"history": [], "pagination": {"total": 0}})
+ return ScriptedResponse(status=404, json={})
+
+ result = _run_export(respond, endpoint="scan_history")
+
+ assert result.raised is None
+ assert [(row["scan_id"], row["scan_schedule_uuid"], row["scan_uuid"]) for row in result.rows] == [
+ (1, "template-aaa", "run-1"),
+ (1, "template-aaa", "run-2"),
+ ]
+ assert result.paths == [
+ "/scans",
+ "/scans/template-aaa/history",
+ "/scans/2/history",
+ "/scans/template-deleted/history",
+ ]
+ assert result.queries[1] == {"sort": ("start_date:asc",), "offset": ("0",), "limit": ("50",)}
+
+ def test_pages_through_long_history(self) -> None:
+ result = _run_export(
+ [
+ ScriptedResponse(json={"scans": [{"id": 1, "schedule_uuid": "template-aaa"}]}),
+ ScriptedResponse(json={"history": [_history_run(f"run-{i}") for i in range(50)]}),
+ ScriptedResponse(json={"history": [_history_run("run-50")]}),
+ ],
+ endpoint="scan_history",
+ )
+
+ assert len(result.rows) == 51
+ assert result.params("offset") == [None, "0", "50"]
+
+ @parameterized.expand([("forbidden", 403), ("server_error_after_retries", 500)])
+ def test_non_404_history_errors_fail_the_run(self, _name: str, status: int) -> None:
+ def respond(request: RecordedRequest) -> ScriptedResponse:
+ if request.path == "/scans":
+ return ScriptedResponse(json={"scans": [{"id": 1, "schedule_uuid": "template-aaa"}]})
+ return ScriptedResponse(status=status, json={})
+
+ result = _run_export(respond, endpoint="scan_history")
+
+ assert result.raised is not None
+ assert result.rows == []
+
+ def test_account_without_scans_yields_nothing(self) -> None:
+ # Tenable returns `"scans": null` rather than an empty list when an account has no scans.
+ result = _run_export([ScriptedResponse(json={"folders": [], "scans": None})], endpoint="scan_history")
+
+ assert result.raised is None
+ assert result.paths == ["/scans"]
+
+
class TestScans:
def test_yields_scan_rows(self) -> None:
result = _run_export(
@@ -248,7 +471,7 @@ def _empty_account(request: RecordedRequest) -> ScriptedResponse:
return ScriptedResponse(json={"export_uuid": "u1"})
if request.path.endswith("/status"):
return ScriptedResponse(json={"status": "FINISHED", "chunks_available": []})
- return ScriptedResponse(json={"scans": []})
+ return ScriptedResponse(json={"scans": [], "values": [], "data": {"plugin_details": []}})
class TestTenableSource:
@@ -261,11 +484,20 @@ class TestTenableSource:
["first_found"],
),
("assets", ["id"], "desc", ["created_at"]),
- ("scans", ["id"], "asc", None),
+ ("scans", ["id"], None, None),
+ ("plugins", ["id"], None, None),
+ ("scan_history", ["scan_id", "scan_uuid"], None, None),
+ (
+ "compliance_findings",
+ ["asset_uuid", "compliance_full_id", "audit_file"],
+ "desc",
+ ["first_seen"],
+ ),
+ ("tag_values", ["uuid"], None, None),
]
)
def test_source_response_shape(
- self, endpoint: str, primary_keys: list[str], sort_mode: str, partition_keys: list[str] | None
+ self, endpoint: str, primary_keys: list[str], sort_mode: str | None, partition_keys: list[str] | None
) -> None:
result = _run_export(_empty_account, endpoint=endpoint)
@@ -278,3 +510,4 @@ class TestTenableSource:
assert response.sort_mode == sort_mode
assert response.partition_keys == partition_keys
assert response.partition_mode == ("datetime" if partition_keys else None)
+ assert response.supports_resume == (sort_mode == "desc")