:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add 4 missing tables to the SonatypeNexus source (#120007)
Tom Owers919579a7bd7a
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 7e2e72f89bfb..d68db45db6a4 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -11436,10 +11436,10 @@ Today (4): `assets`, `components`, `repositories`, `tasks`Diffed against: <https://help.sonatype.com/en/api-reference.html>-- [ ] `security/users` — user inventory with source realm and assigned roles; the only way to attribute repository access (medium)-- [ ] `security/roles` — lookup resolving the role IDs referenced by users (medium)-- [ ] `security/privileges` — lookup resolving the privilege IDs referenced by roles (medium)-- [ ] `blobstores (list + {name}/quota-status)` — storage footprint and quota headroom per blob store - the capacity metric behind component/asset growth (medium)+- [x] `security/users` — user inventory with source realm and assigned roles; the only way to attribute repository access (medium)+- [x] `security/roles` — lookup resolving the role IDs referenced by users (medium)+- [x] `security/privileges` — lookup resolving the privilege IDs referenced by roles (medium)+- [x] `blobstores (list + {name}/quota-status)` — storage footprint and quota headroom per blob store - the capacity metric behind component/asset growth (medium)Note: Source targets Nexus Repository Manager 3 only (/service/rest/v1), not IQ Server / Lifecycle. The Nexus Repository API surface is genuinely small and mostly configuration (capabilities, cleanup-policies, email, http-configuration, licensing, lifecycle, log-management, nodes, read-only, script, status, support) - components/assets/repositories/tasks already cover the bulk of queryable data, so coverage is close to proportionate.diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/sonatype_nexus/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/sonatype_nexus/canonical_descriptions.pyindex 5d67768d3845..e90794b1f7b0 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/sonatype_nexus/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/sonatype_nexus/canonical_descriptions.py@@ -61,4 +61,64 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"lastRun": "When the task last ran.",},},+ "users": {+ "description": "A user account known to the Nexus instance, with its user source (realm) and assigned roles. Local users from the default source are listed in full. Users from external sources such as LDAP or SAML come from a Nexus search that returns at most 100 users.",+ "docs_url": "https://help.sonatype.com/en/security-management-api.html",+ "columns": {+ "userId": "The user id used to log in. Unique within its user source.",+ "firstName": "The first name of the user.",+ "lastName": "The last name of the user.",+ "emailAddress": "The email address of the user.",+ "source": "The user source (realm) the user comes from, e.g. default for local users, LDAP, or SAML.",+ "status": "The user's status: active, locked, disabled, or changepassword.",+ "readOnly": "Whether Nexus can change the user's properties. When true, only role assignments can change.",+ "roles": "Ids of the roles assigned to the user within Nexus.",+ "externalRoles": "Roles assigned to the user in an external source, such as an LDAP group.",+ },+ },+ "roles": {+ "description": "A role that groups privileges and other roles, and that users are assigned to.",+ "docs_url": "https://help.sonatype.com/en/security-management-api.html",+ "columns": {+ "id": "The id of the role. Users reference roles by this id.",+ "source": "The user source the role comes from, e.g. default for roles defined in Nexus.",+ "name": "The name of the role.",+ "description": "The description of the role.",+ "readOnly": "Whether the role is built in and can't be changed.",+ "privileges": "Names of the privileges assigned to the role.",+ "roles": "Ids of other roles contained in this role.",+ },+ },+ "privileges": {+ "description": "A privilege that grants actions on a part of the Nexus instance, such as a repository or an administrative domain. Roles reference privileges by name.",+ "docs_url": "https://help.sonatype.com/en/security-management-api.html",+ "columns": {+ "name": "The unique name of the privilege.",+ "type": "The privilege type: application, wildcard, repository-admin, repository-view, repository-content-selector, or script.",+ "description": "The description of the privilege.",+ "readOnly": "Whether the privilege is built in and can't be changed.",+ "actions": "The actions the privilege grants, e.g. READ, BROWSE, EDIT, ADD, DELETE, RUN, or ALL.",+ "domain": "For application privileges, the domain the privilege grants access to.",+ "pattern": "For wildcard privileges, the colon-separated permission string.",+ "format": "For repository privileges, the repository format the privilege applies to.",+ "repository": "For repository privileges, the repository name the privilege applies to.",+ "contentSelector": "For content selector privileges, the name of the content selector.",+ "scriptName": "For script privileges, the name of the script.",+ },+ },+ "blob_stores": {+ "description": "A blob store that holds the binary content of repositories, with its storage usage and soft quota status.",+ "docs_url": "https://help.sonatype.com/en/blob-store-api.html",+ "columns": {+ "name": "The unique name of the blob store.",+ "type": "The blob store type, such as File or S3.",+ "unavailable": "Whether the blob store is unavailable.",+ "blobCount": "The number of blobs in the blob store.",+ "totalSizeInBytes": "The total size of the blobs in the blob store, in bytes.",+ "availableSpaceInBytes": "The space left in the blob store, in bytes.",+ "softQuota": "The soft quota settings: the quota type and the limit in MB. Null when no quota is set.",+ "quotaViolation": "Whether the blob store violates its soft quota. Null when no quota is set or the blob store is unavailable.",+ "quotaMessage": "The quota status message from Nexus. Null when no quota is set or the blob store is unavailable.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/sonatype_nexus/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/sonatype_nexus/settings.pyindex 8f73b5cb6384..05e6d21ff5ea 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/sonatype_nexus/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/sonatype_nexus/settings.py@@ -46,6 +46,36 @@ SONATYPE_NEXUS_ENDPOINTS: dict[str, SonatypeNexusEndpointConfig] = {name="tasks",path="/tasks",),+ # A user id is only unique within its user source (realm), e.g. `default` vs `LDAP`.+ "users": SonatypeNexusEndpointConfig(+ name="users",+ path="/security/users",+ primary_keys=["source", "userId"],+ paginated=False,+ ),+ "roles": SonatypeNexusEndpointConfig(+ name="roles",+ path="/security/roles",+ primary_keys=["source", "id"],+ paginated=False,+ ),+ "privileges": SonatypeNexusEndpointConfig(+ name="privileges",+ path="/security/privileges",+ primary_keys=["name"],+ paginated=False,+ ),+ "blob_stores": SonatypeNexusEndpointConfig(+ name="blob_stores",+ path="/blobstores",+ primary_keys=["name"],+ paginated=False,+ ),}+# Nexus caps a user search that names no source, or an external source, at 100 users.+# Only the local `default` source is listed in full.+NEXUS_DEFAULT_USER_SOURCE = "default"+NEXUS_USER_SEARCH_LIMIT = 100+ENDPOINTS = tuple(SONATYPE_NEXUS_ENDPOINTS.keys())diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/sonatype_nexus/sonatype_nexus.py b/products/warehouse_sources/backend/temporal/data_imports/sources/sonatype_nexus/sonatype_nexus.pyindex 7a4c62f91c4c..16e1a02cb309 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/sonatype_nexus/sonatype_nexus.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/sonatype_nexus/sonatype_nexus.py@@ -3,7 +3,7 @@ import timeimport dataclassesfrom collections.abc import Iteratorfrom typing import Any, Optional-from urllib.parse import urlencode, urlparse+from urllib.parse import quote, urlencode, urlparseimport requestsfrom structlog.types import FilteringBoundLogger@@ -16,6 +16,8 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.resfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponsefrom products.warehouse_sources.backend.temporal.data_imports.sources.sonatype_nexus.settings import (NEXUS_API_PATH,+ NEXUS_DEFAULT_USER_SOURCE,+ NEXUS_USER_SEARCH_LIMIT,SONATYPE_NEXUS_ENDPOINTS,)@@ -318,6 +320,52 @@ def _get_repository_fanout_rows()+def _get_user_rows(+ session: requests.Session, endpoint_url: str, logger: FilteringBoundLogger+) -> Iterator[list[dict[str, Any]]]:+ local_users = _fetch_page(session, _build_url(endpoint_url, {"source": NEXUS_DEFAULT_USER_SOURCE}), logger)["items"]+ if local_users:+ yield local_users++ # The cross-source search is the only way to list external-realm users (LDAP, SAML, Crowd), but it+ # repeats local users, so keep only the external rows to not duplicate the full local list.+ all_users = _fetch_page(session, endpoint_url, logger)["items"]+ if len(all_users) >= NEXUS_USER_SEARCH_LIMIT:+ logger.warning(+ f"Sonatype Nexus: the cross-source user search hit the {NEXUS_USER_SEARCH_LIMIT}-user cap, "+ "so some external-realm users may be missing"+ )+ external_users = [user for user in all_users if user.get("source") != NEXUS_DEFAULT_USER_SOURCE]+ if external_users:+ yield external_users+++def _get_blob_store_rows(+ session: requests.Session, endpoint_url: str, logger: FilteringBoundLogger+) -> Iterator[list[dict[str, Any]]]:+ blob_stores = _fetch_page(session, endpoint_url, logger)["items"]+ for blob_store in blob_stores:+ quota_status: dict[str, Any] = {}+ # Without a soft quota the status is a fixed "no quota" reply, and an unavailable blob store+ # can't report usage, so only ask for the status when it can carry information.+ if blob_store.get("softQuota") and not blob_store.get("unavailable"):+ status_url = f"{endpoint_url}/{quote(blob_store['name'], safe='')}/quota-status"+ try:+ quota_status = _fetch_page(session, status_url, logger)+ except requests.HTTPError as exc:+ if exc.response is not None and exc.response.status_code == 404:+ logger.warning(f"Sonatype Nexus: blob store {blob_store['name']} not found, skipping")+ continue+ raise+ yield [+ {+ **blob_store,+ "quotaViolation": quota_status.get("isViolation"),+ "quotaMessage": quota_status.get("message"),+ }+ ]++def get_rows(host: str,username: str,@@ -335,6 +383,14 @@ def get_rows(yield from _get_repository_fanout_rows(session, base_url, endpoint_url, logger, resumable_source_manager)return+ if endpoint == "users":+ yield from _get_user_rows(session, endpoint_url, logger)+ return++ if endpoint == "blob_stores":+ yield from _get_blob_store_rows(session, endpoint_url, logger)+ return+if not config.paginated:data = _fetch_page(session, endpoint_url, logger)items = data["items"]diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/sonatype_nexus/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/sonatype_nexus/source.pyindex c8fa23e0b534..b531b9ea0ee8 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/sonatype_nexus/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/sonatype_nexus/source.py@@ -61,7 +61,7 @@ class SonatypeNexusSource(Enter your instance URL (e.g. `https://nexus.example.com`) and the username and password of a user with read access to the repositories you want to sync. A [user token](https://help.sonatype.com/en/user-tokens.html) also works: use the token name code as the username and the pass code as the password.-The tasks table additionally requires the `nx-tasks-read` privilege; deselect it if your user doesn't have it.""",+Some tables need an extra privilege: `nx-tasks-read` for tasks, `nx-users-read` for users, `nx-roles-read` for roles, `nx-privileges-read` for privileges, and `nx-blobstores-read` for blob stores. Deselect a table if your user doesn't have its privilege.""",iconPath="/static/services/sonatype_nexus.png",docsUrl="https://posthog.com/docs/cdp/sources/sonatype-nexus",releaseStatus=ReleaseStatus.ALPHA,@@ -106,7 +106,7 @@ The tasks table additionally requires the `nx-tasks-read` privilege; deselect itdef get_non_retryable_errors(self) -> dict[str, str | None]:return {"401 Client Error: Unauthorized for url": "Your Nexus credentials are invalid or have been revoked. Check the username and password (or user token) and reconnect.",- "403 Client Error: Forbidden for url": "Your Nexus user does not have access to this resource. The tasks table requires the nx-tasks-read privilege, and repository content requires browse/read access — deselect the affected tables or use a user with the required privileges.",+ "403 Client Error: Forbidden for url": "Your Nexus user does not have access to this resource. Repository content needs browse or read access, and the tasks, users, roles, privileges, and blob stores tables each need their read privilege (for example nx-tasks-read). Deselect the affected tables or use a user with the required privileges.",}def get_schemas(diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/sonatype_nexus/tests/test_sonatype_nexus.py b/products/warehouse_sources/backend/temporal/data_imports/sources/sonatype_nexus/tests/test_sonatype_nexus.pyindex da43eeb3062f..666afe77aebd 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/sonatype_nexus/tests/test_sonatype_nexus.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/sonatype_nexus/tests/test_sonatype_nexus.py@@ -307,6 +307,83 @@ class TestGetRowsRepositoryFanout:assert isinstance(result.raised, SonatypeNexusPaginationError)+class TestGetRowsUsers:+ @pytest.mark.parametrize(+ "all_users, expected_external",+ [+ ([{"userId": "admin", "source": "default"}, {"userId": "jdoe", "source": "LDAP"}], [("LDAP", "jdoe")]),+ # A user id is unique only within its source, so a SAML user named like a local user is kept.+ ([{"userId": "admin", "source": "default"}, {"userId": "admin", "source": "SAML"}], [("SAML", "admin")]),+ ([{"userId": "admin", "source": "default"}], []),+ ([], []),+ ],+ )+ def test_local_users_in_full_then_only_external_users_from_cross_source_search(self, all_users, expected_external):+ local_users = [{"userId": "admin", "source": "default"}, {"userId": "ci-bot", "source": "default"}]++ result = _run("users", [_page(local_users), _page(all_users)])++ assert result.raised is None+ assert result.params("source") == ["default", None]+ assert [(row["source"], row["userId"]) for row in result.rows] == [+ ("default", "admin"),+ ("default", "ci-bot"),+ *expected_external,+ ]+++_QUOTA = {"type": "spaceUsedQuota", "limit": 1024}+++class TestGetRowsBlobStores:+ @pytest.mark.parametrize(+ "blob_store, script, expected_quota",+ [+ (+ {"name": "default", "softQuota": _QUOTA},+ [+ _page(+ {"isViolation": True, "message": "Blob store default is over quota", "blobStoreName": "default"}+ )+ ],+ (True, "Blob store default is over quota"),+ ),+ # Without a soft quota, or for an unavailable blob store, no quota status is requested.+ ({"name": "default", "softQuota": None}, [], (None, None)),+ ({"name": "default"}, [], (None, None)),+ ({"name": "default", "softQuota": _QUOTA, "unavailable": True}, [], (None, None)),+ ],+ )+ def test_quota_status_merged_into_blob_store_row(self, blob_store, script, expected_quota):+ result = _run("blob_stores", [_page([blob_store]), *script])++ assert result.raised is None+ assert len(result.requests) == 1 + len(script)+ assert [(row["name"], row["quotaViolation"], row["quotaMessage"]) for row in result.rows] == [+ ("default", *expected_quota)+ ]++ def test_blob_store_deleted_before_quota_status_is_skipped_and_name_is_encoded(self):+ result = _run(+ "blob_stores",+ [+ _page([{"name": "s3 store/a", "softQuota": _QUOTA}, {"name": "default", "softQuota": _QUOTA}]),+ _page({}, status_code=404),+ _page({"isViolation": False, "message": "ok", "blobStoreName": "default"}),+ ],+ )++ assert result.raised is None+ assert result.urls[1] == "https://n.example.com/service/rest/v1/blobstores/s3%20store%2Fa/quota-status"+ assert [row["name"] for row in result.rows] == ["default"]++ def test_quota_status_forbidden_fails_the_sync(self):+ result = _run("blob_stores", [_page([{"name": "default", "softQuota": _QUOTA}]), _page({}, status_code=403)])++ assert isinstance(result.raised, requests.HTTPError)+ assert result.rows == []++class TestSonatypeNexusSource:@pytest.mark.parametrize("endpoint", list(ENDPOINTS))def test_source_response_primary_keys_and_no_partitioning(self, endpoint):