:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add 4 missing tables to the TenableVulnerabilityManagement source (#120634)
Tom Owers49fa307f79c7
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txtindex 0b2218d753b8..6d0d633b95e6 100644--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt@@ -1114,7 +1114,6 @@1 products/warehouse_sources/backend/temporal/data_imports/sources/telnyx/telnyx.py1 products/warehouse_sources/backend/temporal/data_imports/sources/tempo/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/tempo/tempo.py-1 products/warehouse_sources/backend/temporal/data_imports/sources/tenable_vulnerability_management/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/tenable_vulnerability_management/tenable_vulnerability_management.py1 products/warehouse_sources/backend/temporal/data_imports/sources/terraform_cloud/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/terraform_cloud/terraform_cloud.pydiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 30488d25f18e..443730e000ba 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -12202,10 +12202,10 @@ Today (3): `assets`, `scans`, `vulnerabilities`Diffed against: <https://developer.tenable.com/reference/exports-assets-request-export>-- [ ] `plugins (GET /plugins/plugin)` — lookup resolving the plugin_id carried on every synced vulnerability finding to name, family, CVE, CVSS and VPR (high)-- [ ] `scan history (GET /scans/{scan_id}/history)` — each individual run of a scan with status and timing; the scans table alone has no run-level history (high)-- [ ] `compliance findings export (POST /compliance/export + status/chunks)` — audit/compliance results, the second half of Tenable's findings data alongside vulnerabilities (high)-- [ ] `tag values (GET /tags/values)` — lookup table for the asset tags used to slice every asset and vulnerability report (high)+- [x] `plugins (GET /plugins/plugin)` — lookup resolving the plugin_id carried on every synced vulnerability finding to name, family, CVE, CVSS and VPR (high)+- [x] `scan history (GET /scans/{scan_id}/history)` — each individual run of a scan with status and timing; the scans table alone has no run-level history (high)+- [x] `compliance findings export (POST /compliance/export + status/chunks)` — audit/compliance results, the second half of Tenable's findings data alongside vulnerabilities (high)+- [x] `tag values (GET /tags/values)` — lookup table for the asset tags used to slice every asset and vulnerability report (high)- [ ] `tag categories (GET /tags/categories)` — parent dimension for tag values, needed to group tags into category/value pairs (medium)- [ ] `plugin families (GET /plugins/families)` — grouping dimension for plugins, standard breakdown axis in vulnerability reporting (medium)- [ ] `networks (GET /networks)` — lookup resolving the network_id on assets and scanners (medium)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/tenable_vulnerability_management/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/tenable_vulnerability_management/canonical_descriptions.pyindex 9c4c97e803d3..9dfb78e282cb 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/tenable_vulnerability_management/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/tenable_vulnerability_management/canonical_descriptions.py@@ -75,4 +75,66 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"last_modification_date": "When the scan was last modified, as a Unix timestamp.",},},+ "plugins": {+ "description": "Tenable plugins, one row per plugin with the detection details used to resolve the plugin_id on vulnerability findings.",+ "docs_url": "https://developer.tenable.com/reference/io-plugins-list",+ "columns": {+ "id": "ID of the plugin.",+ "name": "Name of the plugin.",+ "attributes": "Plugin details, such as description, solution, CVEs, CVSS scores, VPR, and publication dates.",+ "plugin_modification_date": "When Tenable last updated the plugin.",+ },+ },+ "scan_history": {+ "description": "Scan history, one row per individual run of a scan.",+ "docs_url": "https://developer.tenable.com/reference/scans-history",+ "columns": {+ "scan_id": "ID of the scan configuration that the run belongs to.",+ "scan_schedule_uuid": "Schedule UUID of the scan configuration that the run belongs to.",+ "scan_uuid": "UUID of the specific scan run.",+ "id": "Unique ID of the scan run. Tenable does not return this ID for every run.",+ "status": "Status of the scan run.",+ "time_start": "When the scan run started, as a Unix timestamp.",+ "time_end": "When the scan run stopped, as a Unix timestamp.",+ "is_archived": "Whether the results of the scan run are archived.",+ "visibility": "Visibility of the scan run results.",+ "targets": "Whether the run used the default scan targets or custom targets.",+ "reporting_mode": "Reporting mode of the scan run.",+ },+ },+ "compliance_findings": {+ "description": "Compliance findings from Tenable's compliance export, one row per audit check result on an asset (asset, check, and audit file).",+ "docs_url": "https://developer.tenable.com/reference/io-exports-compliance-create",+ "columns": {+ "asset_uuid": "UUID of the asset on which the compliance check ran.",+ "compliance_full_id": "Hash that identifies the content of the compliance check.",+ "audit_file": "Name of the audit file that contains the compliance check.",+ "check_id": "Unique identifier for the compliance finding.",+ "check_name": "Name of the compliance check.",+ "status": "Result of the compliance check: PASSED, FAILED, WARNING, SKIPPED, ERROR, or UNKNOWN.",+ "state": "State of the finding: OPEN, REOPENED, or FIXED.",+ "plugin_id": "ID of the compliance plugin.",+ "first_seen": "When a compliance scan first assessed the asset with the check.",+ "last_seen": "When a compliance scan most recently assessed the asset with the check.",+ "last_observed": "When the finding was last observed on the asset, whether active or fixed.",+ "last_fixed": "When the finding was last fixed on the asset.",+ "indexed_at": "When the audit for the asset was indexed into Tenable.",+ "asset": "Details about the asset, such as name, IP addresses, operating systems, and tags.",+ },+ },+ "tag_values": {+ "description": "Asset tag values, one row per value with the tag category it belongs to.",+ "docs_url": "https://developer.tenable.com/reference/tags-list-tag-values",+ "columns": {+ "uuid": "UUID of the tag value.",+ "value": "The tag value.",+ "description": "Description of the tag value.",+ "type": "Type of the tag value: static or dynamic.",+ "category_uuid": "UUID of the tag category.",+ "category_name": "Name of the tag category.",+ "category_description": "Description of the tag category.",+ "created_at": "When the tag value was created.",+ "updated_at": "When the tag value was last updated.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/tenable_vulnerability_management/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/tenable_vulnerability_management/settings.pyindex f281737d2c25..ebb770b25d40 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/tenable_vulnerability_management/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/tenable_vulnerability_management/settings.py@@ -1,10 +1,10 @@from dataclasses import dataclass, field-from typing import Any, Optional+from typing import Any, Literal, Optionalfrom products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType-@dataclass+@dataclass(frozen=True)class TenableEndpointConfig:name: strprimary_keys: list[str]@@ -20,8 +20,14 @@ class TenableEndpointConfig:# Static body for the export request; the incremental filter is merged in per run.export_payload: dict[str, Any] = field(default_factory=dict)path: Optional[str] = None- # Key holding the row list in a plain (non-export) endpoint's JSON response.+ # Dotted path to the row list in a plain (non-export) endpoint's JSON response.data_key: Optional[str] = None+ # "page" sends 1-based `page` + `size`; "offset" sends `offset` + `limit`. None is a single request.+ pagination: Optional[Literal["page", "offset"]] = None+ page_size: int = 0+ params: dict[str, Any] = field(default_factory=dict)+ # Query param carrying the incremental watermark as a `YYYY-MM-DD` date on a plain endpoint.+ since_date_param: Optional[str] = NoneTENABLE_ENDPOINTS: dict[str, TenableEndpointConfig] = {@@ -72,6 +78,68 @@ TENABLE_ENDPOINTS: dict[str, TenableEndpointConfig] = {primary_keys=["id"],incremental_fields=[],),+ # Plugin catalog, the lookup for the plugin_id on every vulnerability finding. `last_updated`+ # ignores VPR changes, so an incremental run keeps the VPR of unchanged plugins as last synced.+ "plugins": TenableEndpointConfig(+ name="plugins",+ path="/plugins/plugin",+ data_key="data.plugin_details",+ pagination="page",+ page_size=1000,+ since_date_param="last_updated",+ primary_keys=["id"],+ incremental_fields=[+ {+ "label": "plugin_modification_date",+ "type": IncrementalFieldType.DateTime,+ "field": "plugin_modification_date",+ "field_type": IncrementalFieldType.DateTime,+ },+ ],+ ),+ # One row per run of each scan, fanned out over the scans list. The history endpoint has no+ # time filter, so this table is full refresh only. Tenable omits `id` for some runs, so the+ # per-run `scan_uuid` is the key.+ "scan_history": TenableEndpointConfig(+ name="scan_history",+ path="/scans/{scan_id}/history",+ data_key="history",+ pagination="offset",+ page_size=50,+ params={"sort": "start_date:asc"},+ primary_keys=["scan_id", "scan_uuid"],+ ),+ # Compliance (audit) findings. Tenable documents asset_uuid + compliance_full_id + audit_file+ # as the composite key. No state filter is sent: without one, `since` returns open and reopened+ # findings seen since the watermark plus findings fixed since it.+ "compliance_findings": TenableEndpointConfig(+ name="compliance_findings",+ export_path="compliance",+ since_filter="since",+ export_payload={"num_findings": 5000},+ primary_keys=["asset_uuid", "compliance_full_id", "audit_file"],+ partition_key="first_seen",+ incremental_fields=[+ {+ "label": "indexed_at",+ "type": IncrementalFieldType.DateTime,+ "field": "indexed_at",+ "field_type": IncrementalFieldType.DateTime,+ },+ ],+ ),+ # Tag values with their category. The `updated_at` filter's timestamp unit is undocumented, so+ # this small lookup table is full refresh only. A value is unique within its category, so the+ # sort gives a stable order across pages.+ "tag_values": TenableEndpointConfig(+ name="tag_values",+ path="/tags/values",+ data_key="values",+ pagination="offset",+ page_size=5000,+ params={"sort": "category_name:asc,value:asc"},+ primary_keys=["uuid"],+ ),}ENDPOINTS = tuple(TENABLE_ENDPOINTS.keys())diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/tenable_vulnerability_management/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/tenable_vulnerability_management/source.pyindex 1768ac8d2cb9..c0923509ac33 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/tenable_vulnerability_management/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/tenable_vulnerability_management/source.py@@ -142,6 +142,20 @@ Generate an API key pair (access key and secret key) from **Settings > My Accoun)if endpoint == "scans":return "Scan configurations with their schedule and latest run status"+ if endpoint == "plugins":+ return (+ "One row per Tenable plugin with its name, family, CVEs, CVSS and VPR. Incremental syncs pick "+ "up plugins modified since the last run, but not plugins whose only change is a new VPR"+ )+ if endpoint == "scan_history":+ return "One row per run of each scan, with its status and start and end times"+ if endpoint == "compliance_findings":+ return (+ "One row per compliance check result (asset, check, audit file). Synced via Tenable's "+ "asynchronous export API; incremental syncs pick up findings indexed since the last run"+ )+ if endpoint == "tag_values":+ return "Asset tag values with their tag category"return Nonedef _build_schema(endpoint: str) -> SourceSchema:diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/tenable_vulnerability_management/tenable_vulnerability_management.py b/products/warehouse_sources/backend/temporal/data_imports/sources/tenable_vulnerability_management/tenable_vulnerability_management.pyindex d74048e83dba..156ef520aef5 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/tenable_vulnerability_management/tenable_vulnerability_management.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/tenable_vulnerability_management/tenable_vulnerability_management.py@@ -1,8 +1,9 @@import timeimport dataclassesfrom collections.abc import Iterator-from datetime import UTC, date, datetime+from datetime import UTC, date, datetime, timedeltafrom typing import Any, Optional+from urllib.parse import urlencodeimport requestsfrom structlog.types import FilteringBoundLogger@@ -105,6 +106,15 @@ def _to_unix_timestamp(value: Any) -> int | None:return None+def _to_since_date(value: Any) -> str | None:+ """Convert an incremental watermark to the `YYYY-MM-DD` date a plain endpoint filters on.+ The filter is day-granular and exclusive, so it starts one day early to keep same-day updates."""+ timestamp = _to_unix_timestamp(value)+ if timestamp is None:+ return None+ return (datetime.fromtimestamp(timestamp, tz=UTC).date() - timedelta(days=1)).isoformat()++@retry(retry=retry_if_exception_type((@@ -224,9 +234,17 @@ def _normalize_vulnerability(row: dict[str, Any]) -> dict[str, Any]:return row+def _normalize_plugin(row: dict[str, Any]) -> dict[str, Any]:+ # The incremental field must be a top-level column for the pipeline to track its watermark.+ row["plugin_modification_date"] = (row.get("attributes") or {}).get("plugin_modification_date")+ return row++def _normalize_rows(endpoint: str, rows: list[dict[str, Any]]) -> list[dict[str, Any]]:if endpoint == "vulnerabilities":return [_normalize_vulnerability(row) for row in rows]+ if endpoint == "plugins":+ return [_normalize_plugin(row) for row in rows]return rows@@ -305,18 +323,82 @@ def _get_export_rows(status = None+def _extract_rows(data: Any, data_key: str | None) -> list[dict[str, Any]]:+ if data_key:+ for key in data_key.split("."):+ data = data.get(key) if isinstance(data, dict) else None+ return data if isinstance(data, list) else []++def _get_plain_rows(session: requests.Session,base_url: str,+ endpoint: str,config: TenableEndpointConfig,headers: dict[str, str],logger: FilteringBoundLogger,+ path: str | None = None,+ since_date: str | None = None,) -> Iterator[list[dict[str, Any]]]:- response = _request(session, "GET", f"{base_url}{config.path}", headers, logger)- data = response.json()- rows = data.get(config.data_key) if config.data_key else data- if rows:- yield rows+ params = dict(config.params)+ if since_date is not None and config.since_date_param:+ params[config.since_date_param] = since_date+ page = 0++ while True:+ if config.pagination == "page":+ params.update({"page": page + 1, "size": config.page_size})+ elif config.pagination == "offset":+ params.update({"offset": page * config.page_size, "limit": config.page_size})++ url = f"{base_url}{path or config.path}"+ if params:+ url = f"{url}?{urlencode(params)}"+ rows = _extract_rows(_request(session, "GET", url, headers, logger).json(), config.data_key)+ if rows:+ yield _normalize_rows(endpoint, rows)++ if config.pagination is None or len(rows) < config.page_size:+ return+ page += 1+++def _get_scan_history_rows(+ session: requests.Session,+ base_url: str,+ endpoint: str,+ config: TenableEndpointConfig,+ headers: dict[str, str],+ resumable_source_manager: ResumableSourceManager[TenableResumeConfig],+ logger: FilteringBoundLogger,+) -> Iterator[list[dict[str, Any]]]:+ scans_config = TENABLE_ENDPOINTS["scans"]+ history_path = config.path or ""+ scans = _extract_rows(+ _request(session, "GET", f"{base_url}{scans_config.path}", headers, logger).json(), scans_config.data_key+ )++ for scan in scans:+ # Tenable recommends the schedule UUID over the numeric ID to address a scan.+ scan_ref = scan.get("schedule_uuid") or scan.get("id")+ if scan_ref is None:+ continue+ parent_fields = {"scan_id": scan.get("id"), "scan_schedule_uuid": scan.get("schedule_uuid")}+ yielded = False+ try:+ for rows in _get_plain_rows(+ session, base_url, endpoint, config, headers, logger, path=history_path.format(scan_id=scan_ref)+ ):+ yielded = True+ yield [{**row, **parent_fields} for row in rows]+ except requests.HTTPError as exc:+ # A scan deleted after the scans list was read has no history to sync.+ if exc.response is None or exc.response.status_code != 404:+ raise+ logger.debug(f"Tenable: scan {scan_ref} was not found, skipping its history")+ if not yielded:+ # The scan history run keeps no resume state, so no row is held back here.+ resumable_source_manager.safe_point()def get_rows(@@ -334,8 +416,15 @@ def get_rows(headers = _get_headers(access_key, secret_key)session = _make_session(access_key, secret_key)+ if endpoint == "scan_history":+ yield from _get_scan_history_rows(+ session, base_url, endpoint, config, headers, resumable_source_manager, logger+ )+ return+if config.export_path is None:- yield from _get_plain_rows(session, base_url, config, headers, logger)+ since_date = _to_since_date(db_incremental_field_last_value) if should_use_incremental_field else None+ yield from _get_plain_rows(session, base_url, endpoint, config, headers, logger, since_date=since_date)returnsince = _to_unix_timestamp(db_incremental_field_last_value) if should_use_incremental_field else None@@ -370,7 +459,10 @@ def tenable_source(# Export chunks carry no ordering guarantee, so the incremental watermark (max value seen)# is only persisted once the whole run succeeds — per-batch checkpointing would advance it# past rows a crashed run still owes.- sort_mode="desc" if config.export_path else "asc",+ # Plain endpoints page by ID or name, not by the incremental field, so their order is unknown.+ sort_mode="desc" if config.export_path else None,+ # Only export reads checkpoint their position; a plain or fan-out read restarts from scratch.+ supports_resume=bool(config.export_path),partition_count=1 if config.partition_key else None,partition_size=1 if config.partition_key else None,partition_mode="datetime" if config.partition_key else None,diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/tenable_vulnerability_management/tests/test_tenable_vulnerability_management.py b/products/warehouse_sources/backend/temporal/data_imports/sources/tenable_vulnerability_management/tests/test_tenable_vulnerability_management.pyindex ee4604cd4a76..85528a14097a 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/tenable_vulnerability_management/tests/test_tenable_vulnerability_management.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/tenable_vulnerability_management/tests/test_tenable_vulnerability_management.py@@ -28,6 +28,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.tenable_vuTenableResumeConfig,_base_url,_build_export_payload,+ _to_since_date,_to_unix_timestamp,validate_credentials,)@@ -81,6 +82,20 @@ class TestToUnixTimestamp:assert _to_unix_timestamp(value) == expected+class TestToSinceDate:+ @parameterized.expand(+ [+ ("midnight_starts_previous_day", datetime(2026, 7, 1, 0, 0, 0, tzinfo=UTC), "2026-06-30"),+ ("late_in_day_starts_previous_day", "2026-07-01T23:59:59Z", "2026-06-30"),+ ("month_boundary", date(2026, 3, 1), "2026-02-28"),+ ("garbage_string", "not-a-date", None),+ ("none", None, None),+ ]+ )+ def test_conversion(self, _name: str, value: Any, expected: str | None) -> None:+ assert _to_since_date(value) == expected++class TestBaseUrl:@parameterized.expand([@@ -99,9 +114,21 @@ class TestBaseUrl:class TestBuildExportPayload:- def test_assets_incremental_adds_updated_at(self) -> None:- payload = _build_export_payload(TENABLE_ENDPOINTS["assets"], since=1782864000)- assert payload == {"chunk_size": 1000, "filters": {"updated_at": 1782864000}}+ @parameterized.expand(+ [+ ("assets_incremental", "assets", 1782864000, {"chunk_size": 1000, "filters": {"updated_at": 1782864000}}),+ (+ "compliance_incremental",+ "compliance_findings",+ 1782864000,+ {"num_findings": 5000, "filters": {"since": 1782864000}},+ ),+ # A state filter would drop findings last seen before Tenable added states (state is NULL).+ ("compliance_full_refresh", "compliance_findings", None, {"num_findings": 5000}),+ ]+ )+ def test_payload(self, _name: str, endpoint: str, since: int | None, expected: dict[str, Any]) -> None:+ assert _build_export_payload(TENABLE_ENDPOINTS[endpoint], since=since) == expectedclass TestValidateCredentials:@@ -233,6 +260,202 @@ class TestExportFlow:assert result.final_state is None+class TestCompliance:+ def test_runs_the_compliance_export_flow(self) -> None:+ row = {"asset_uuid": "a-1", "compliance_full_id": "c-1", "audit_file": "x.audit", "status": "FAILED"}+ result = _run_export(+ [+ ScriptedResponse(json={"export_uuid": "u1"}),+ ScriptedResponse(json={"status": "READY", "chunks_available": [1]}),+ ScriptedResponse(json=[row]),+ ScriptedResponse(json={"status": "FINISHED", "chunks_available": [1]}),+ ],+ endpoint="compliance_findings",+ should_use_incremental_field=True,+ db_incremental_field_last_value="2026-07-01T00:00:00Z",+ )++ assert result.raised is None+ assert result.rows == [row]+ assert result.paths == [+ "/compliance/export",+ "/compliance/export/u1/status",+ "/compliance/export/u1/chunks/1",+ "/compliance/export/u1/status",+ ]+ assert result.requests[0].json() == {"num_findings": 5000, "filters": {"since": 1782864000}}+++def _plugin(plugin_id: int) -> dict[str, Any]:+ return {+ "id": plugin_id,+ "name": f"Plugin {plugin_id}",+ "attributes": {"plugin_modification_date": "2026-06-15T00:00:00Z", "cve": ["CVE-2026-0001"]},+ }+++def _plugins_page(plugin_ids: range) -> dict[str, Any]:+ plugins = [_plugin(plugin_id) for plugin_id in plugin_ids]+ return {"data": {"plugin_details": plugins}, "size": len(plugins), "total_count": 1500}+++def _tag_values_page(start: int, count: int) -> dict[str, Any]:+ values = [{"uuid": f"t-{i}", "category_name": "location", "value": f"v{i}"} for i in range(start, start + count)]+ return {"values": values, "pagination": {"total": 5002, "offset": start, "limit": 5000}}+++class TestPagedEndpoints:+ @parameterized.expand(+ [+ (+ "plugins_by_page",+ "plugins",+ [_plugins_page(range(1, 1001)), _plugins_page(range(1001, 1501))],+ [{"page": ("1",), "size": ("1000",)}, {"page": ("2",), "size": ("1000",)}],+ 1500,+ ),+ (+ "tag_values_by_offset",+ "tag_values",+ [_tag_values_page(0, 5000), _tag_values_page(5000, 2)],+ [+ {"sort": ("category_name:asc,value:asc",), "offset": ("0",), "limit": ("5000",)},+ {"sort": ("category_name:asc,value:asc",), "offset": ("5000",), "limit": ("5000",)},+ ],+ 5002,+ ),+ (+ "full_page_then_empty_page",+ "plugins",+ [_plugins_page(range(1, 1001)), {"data": {"plugin_details": []}, "size": 0, "total_count": 1000}],+ [{"page": ("1",), "size": ("1000",)}, {"page": ("2",), "size": ("1000",)}],+ 1000,+ ),+ ]+ )+ def test_walks_pages_until_a_short_page(+ self,+ _name: str,+ endpoint: str,+ pages: list[dict[str, Any]],+ expected_queries: list[dict[str, tuple[str, ...]]],+ expected_rows: int,+ ) -> None:+ result = _run_export([ScriptedResponse(json=page) for page in pages], endpoint=endpoint)++ assert result.raised is None+ assert result.queries == expected_queries+ assert len(result.rows) == expected_rows++ @parameterized.expand(+ [+ ("incremental", True, "2026-07-01T10:00:00Z", "2026-06-30"),+ ("full_refresh_ignores_stale_cursor", False, "2026-07-01T10:00:00Z", None),+ ("first_incremental_run", True, None, None),+ ]+ )+ def test_plugins_last_updated_filter(+ self, _name: str, incremental: bool, last_value: str | None, expected: str | None+ ) -> None:+ result = _run_export(+ [ScriptedResponse(json=_plugins_page(range(1, 3)))],+ endpoint="plugins",+ should_use_incremental_field=incremental,+ db_incremental_field_last_value=last_value,+ )++ assert result.params("last_updated") == [expected]++ def test_plugins_lift_modification_date_to_top_level(self) -> None:+ result = _run_export([ScriptedResponse(json=_plugins_page(range(7, 8)))], endpoint="plugins")++ assert result.rows[0]["plugin_modification_date"] == "2026-06-15T00:00:00Z"+ assert result.rows[0]["attributes"]["cve"] == ["CVE-2026-0001"]++ @parameterized.expand(+ [+ ("missing_data_key", {"size": 0, "total_count": 0}),+ ("null_list", {"data": {"plugin_details": None}}),+ ]+ )+ def test_empty_response_yields_nothing(self, _name: str, body: dict[str, Any]) -> None:+ result = _run_export([ScriptedResponse(json=body)], endpoint="plugins")++ assert result.raised is None+ assert result.rows == []+++def _history_run(scan_uuid: str) -> dict[str, Any]:+ return {"scan_uuid": scan_uuid, "status": "completed", "time_start": 1782864000, "time_end": 1782864600}+++class TestScanHistory:+ def test_fans_out_over_scans_and_tags_rows_with_the_parent(self) -> None:+ def respond(request: RecordedRequest) -> ScriptedResponse:+ if request.path == "/scans":+ return ScriptedResponse(+ json={+ "scans": [+ {"id": 1, "schedule_uuid": "template-aaa"},+ {"id": 2, "schedule_uuid": None},+ {"id": 3, "schedule_uuid": "template-deleted"},+ ]+ }+ )+ if request.path == "/scans/template-aaa/history":+ return ScriptedResponse(json={"history": [_history_run("run-1"), _history_run("run-2")]})+ if request.path == "/scans/2/history":+ return ScriptedResponse(json={"history": [], "pagination": {"total": 0}})+ return ScriptedResponse(status=404, json={})++ result = _run_export(respond, endpoint="scan_history")++ assert result.raised is None+ assert [(row["scan_id"], row["scan_schedule_uuid"], row["scan_uuid"]) for row in result.rows] == [+ (1, "template-aaa", "run-1"),+ (1, "template-aaa", "run-2"),+ ]+ assert result.paths == [+ "/scans",+ "/scans/template-aaa/history",+ "/scans/2/history",+ "/scans/template-deleted/history",+ ]+ assert result.queries[1] == {"sort": ("start_date:asc",), "offset": ("0",), "limit": ("50",)}++ def test_pages_through_long_history(self) -> None:+ result = _run_export(+ [+ ScriptedResponse(json={"scans": [{"id": 1, "schedule_uuid": "template-aaa"}]}),+ ScriptedResponse(json={"history": [_history_run(f"run-{i}") for i in range(50)]}),+ ScriptedResponse(json={"history": [_history_run("run-50")]}),+ ],+ endpoint="scan_history",+ )++ assert len(result.rows) == 51+ assert result.params("offset") == [None, "0", "50"]++ @parameterized.expand([("forbidden", 403), ("server_error_after_retries", 500)])+ def test_non_404_history_errors_fail_the_run(self, _name: str, status: int) -> None:+ def respond(request: RecordedRequest) -> ScriptedResponse:+ if request.path == "/scans":+ return ScriptedResponse(json={"scans": [{"id": 1, "schedule_uuid": "template-aaa"}]})+ return ScriptedResponse(status=status, json={})++ result = _run_export(respond, endpoint="scan_history")++ assert result.raised is not None+ assert result.rows == []++ def test_account_without_scans_yields_nothing(self) -> None:+ # Tenable returns `"scans": null` rather than an empty list when an account has no scans.+ result = _run_export([ScriptedResponse(json={"folders": [], "scans": None})], endpoint="scan_history")++ assert result.raised is None+ assert result.paths == ["/scans"]++class TestScans:def test_yields_scan_rows(self) -> None:result = _run_export(@@ -248,7 +471,7 @@ def _empty_account(request: RecordedRequest) -> ScriptedResponse:return ScriptedResponse(json={"export_uuid": "u1"})if request.path.endswith("/status"):return ScriptedResponse(json={"status": "FINISHED", "chunks_available": []})- return ScriptedResponse(json={"scans": []})+ return ScriptedResponse(json={"scans": [], "values": [], "data": {"plugin_details": []}})class TestTenableSource:@@ -261,11 +484,20 @@ class TestTenableSource:["first_found"],),("assets", ["id"], "desc", ["created_at"]),- ("scans", ["id"], "asc", None),+ ("scans", ["id"], None, None),+ ("plugins", ["id"], None, None),+ ("scan_history", ["scan_id", "scan_uuid"], None, None),+ (+ "compliance_findings",+ ["asset_uuid", "compliance_full_id", "audit_file"],+ "desc",+ ["first_seen"],+ ),+ ("tag_values", ["uuid"], None, None),])def test_source_response_shape(- self, endpoint: str, primary_keys: list[str], sort_mode: str, partition_keys: list[str] | None+ self, endpoint: str, primary_keys: list[str], sort_mode: str | None, partition_keys: list[str] | None) -> None:result = _run_export(_empty_account, endpoint=endpoint)@@ -278,3 +510,4 @@ class TestTenableSource:assert response.sort_mode == sort_modeassert response.partition_keys == partition_keysassert response.partition_mode == ("datetime" if partition_keys else None)+ assert response.supports_resume == (sort_mode == "desc")