Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add 4 missing tables to the Gitguardian source (#125044)

Tom Owers70c6c27109a1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index c42b89b3ae18..28af0ca9cbb2 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -5285,13 +5285,13 @@ Diffed against: <https://api.gitguardian.com/v1/openapi.json>
- [x] `/v1/incidents/secrets/{incident_id}/activity-logs` — incident state-transition history — remediation MTTR cannot be computed from current status alone (high)
- [x] `/v1/honeytokens_events` — honeytoken trigger events across the org; honeytokens are synced but the events they exist to capture are not (high)
- [x] `/v1/teams/{team_id}/team_memberships` — join table between the members and teams we already sync (high)
-- [ ] `/v1/incidents/secrets/{incident_id}/notes` — analyst remediation notes attached to each incident (medium)
+- [x] `/v1/incidents/secrets/{incident_id}/notes` — analyst remediation notes attached to each incident (medium)
- [x] `/v1/custom_tags` — lookup resolving the custom tag IDs applied to incidents (medium)
- [x] `/v1/audit_logs` — org-wide audit trail of who changed what in GitGuardian (medium)
- [x] `/v1/public-incidents/secrets` — publicly leaked secrets — a separate perimeter from internal incidents, with its own severity picture (medium)
-- [ ] `/v1/teams/{team_id}/sources` — join table mapping the sources we sync to owning teams (medium)
-- [ ] `/v1/incidents/secrets/{incident_id}/leaks` — where each secret surfaced publicly, the exposure dimension of an incident (medium)
-- [ ] `/v1/public-perimeter/developers` — developers associated with public leaks, for per-developer exposure reporting (low)
+- [x] `/v1/teams/{team_id}/sources` — join table mapping the sources we sync to owning teams (medium)
+- [x] `/v1/incidents/secrets/{incident_id}/leaks` — where each secret surfaced publicly, the exposure dimension of an incident (medium)
+- [x] `/v1/public-perimeter/developers` — developers associated with public leaks, for per-developer exposure reporting (low)
- [ ] `/v1/honeytokens/{honeytoken_id}/activity-logs` — per-honeytoken activity timeline (low)
Note: Fetched the live 5 MB OpenAPI document from the vendor's own /v1/openapi.json.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/gitguardian/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/gitguardian/canonical_descriptions.py
index a6fb347284d7..06a832367b1d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/gitguardian/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/gitguardian/canonical_descriptions.py
@@ -214,4 +214,57 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"risk_score": "Risk score of the incident, from 0 (low) to 100 (critical). Null when not yet computed.",
},
},
+ "secret_incident_notes": {
+ "description": "A note left on a secret incident by a workspace member or through the API, such as a remediation comment.",
+ "docs_url": "https://api.gitguardian.com/docs#tag/Secret-Incident-Notes",
+ "columns": {
+ "id": "Identifier of the note.",
+ "incident_id": "Identifier of the secret incident the note belongs to.",
+ "member_id": "Identifier of the member who created the note. Null if an API key created it or the member was deleted.",
+ "api_token_id": "Identifier of the API key that created the note. Null if the note was not created through the API.",
+ "created_at": "Date and time the note was created.",
+ "updated_at": "Date and time the content of the note was last updated. Null if it was never modified.",
+ "comment": "Content of the note.",
+ },
+ },
+ "secret_incident_leaks": {
+ "description": "A public location where the secret of a secret incident was found leaked, such as a GitHub repository, issue, or gist.",
+ "docs_url": "https://api.gitguardian.com/docs#tag/Secret-Incidents",
+ "columns": {
+ "incident_id": "Identifier of the secret incident the leak belongs to. Added by PostHog from the parent incident.",
+ "source": "Type of public source the secret leaked to (github, github_issue, github_gist, unknown).",
+ "name": "Name of the public source (e.g. the repository name).",
+ "url": "URL of the public location where the secret leaked.",
+ },
+ },
+ "team_sources": {
+ "description": "A monitored source that belongs to a team's perimeter. Joins the sources table to the teams table.",
+ "docs_url": "https://api.gitguardian.com/docs#tag/Team-Sources",
+ "columns": {
+ "team_id": "Identifier of the team whose perimeter includes the source. Added by PostHog from the parent team.",
+ "id": "Identifier of the source.",
+ "url": "URL of the source on its provider.",
+ "type": "Type of the source (e.g. github, gitlab, bitbucket).",
+ "full_name": "Full name of the source.",
+ "health": "Health of the source (safe, unknown, at_risk).",
+ "open_incidents_count": "Number of open secret incidents with at least one occurrence on the source.",
+ "closed_incidents_count": "Number of closed secret incidents with at least one occurrence on the source.",
+ "visibility": "Visibility of the source on its provider (public, private, internal).",
+ },
+ },
+ "public_developers": {
+ "description": "A developer in the public perimeter: a person GitGuardian links to your organization so it can monitor their public activity for leaked secrets.",
+ "docs_url": "https://api.gitguardian.com/docs#tag/Developer",
+ "columns": {
+ "id": "Identifier of the developer.",
+ "type": "Type of the developer identity (github_user, git_user, container_image_user).",
+ "emails": "Email addresses linked to the developer.",
+ "first_linked_at": "Date and time the developer was first linked to the organization.",
+ "is_active": "Whether the developer is actively monitored.",
+ "github_id": "GitHub user id of the developer.",
+ "github_login": "GitHub login of the developer.",
+ "name": "Name of the developer.",
+ "reasons": "Reasons the developer is linked to the organization (e.g. github_organization, git_email, manual).",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/gitguardian/gitguardian.py b/products/warehouse_sources/backend/temporal/data_imports/sources/gitguardian/gitguardian.py
index 49a50ca53d78..037a858ae12e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/gitguardian/gitguardian.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/gitguardian/gitguardian.py
@@ -167,8 +167,20 @@ def check_endpoint_access(api_key: str, base_url: str, endpoint: str) -> str | N
"""
config = GITGUARDIAN_ENDPOINTS[endpoint]
# Fan-out children share their parent's scope, and their path needs a real parent id.
- probe_path = GITGUARDIAN_ENDPOINTS[config.parent_endpoint].path if config.parent_endpoint else config.path
- url = _build_url(base_url, probe_path, {"per_page": 1})
+ probe = GITGUARDIAN_ENDPOINTS[config.parent_endpoint] if config.parent_endpoint else config
+ probes = [(probe, config.required_scope)]
+ probes += [
+ (GITGUARDIAN_ENDPOINTS[name], GITGUARDIAN_ENDPOINTS[name].required_scope) for name in config.also_requires
+ ]
+ for probe_config, scope in probes:
+ reason = _probe_endpoint(api_key, base_url, probe_config, scope)
+ if reason:
+ return reason
+ return None
+
+
+def _probe_endpoint(api_key: str, base_url: str, probe: GitGuardianEndpointConfig, scope: str) -> str | None:
+ url = _build_url(base_url, probe.path, {"per_page": 1} if probe.page_size else {})
try:
# capture=False keeps GitGuardian security metadata (repo names, file paths, secret hashes)
# out of the HTTP sample bucket — the name-based scrubbers can't recognise these fields.
@@ -183,11 +195,11 @@ def check_endpoint_access(api_key: str, base_url: str, endpoint: str) -> str | N
detail = _error_detail(response)
if detail:
return detail
- return f"Your API token is missing the `{config.required_scope}` scope required for this table."
+ return f"Your API token is missing the `{scope}` scope required for this table."
def _list_params(config: GitGuardianEndpointConfig) -> dict[str, Any]:
- params: dict[str, Any] = {"per_page": config.page_size}
+ params: dict[str, Any] = {"per_page": config.page_size} if config.page_size else {}
if config.ordering:
params["ordering"] = config.ordering
return params
@@ -245,7 +257,10 @@ def _get_fan_out_rows(
session, _build_url(base_url, path, _list_params(config)), headers, base_url, config.name, logger
):
if page.rows:
- yield page.rows
+ if config.parent_id_field:
+ yield [{**row, config.parent_id_field: parent_id} for row in page.rows]
+ else:
+ yield page.rows
except requests.HTTPError as e:
# The parent can be deleted between listing it and fetching its children.
if e.response is not None and e.response.status_code == 404:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/gitguardian/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/gitguardian/settings.py
index a382f3549acc..92f2442cffcf 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/gitguardian/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/gitguardian/settings.py
@@ -21,7 +21,8 @@ class GitGuardianEndpointConfig:
# "desc" for incremental endpoints with no `ordering` param: the pipeline then promotes the
# watermark only when the sync completes, so it never depends on the API's row order.
sort_mode: Literal["asc", "desc"] = "asc"
- page_size: int = 100 # GitGuardian caps per_page at 100
+ # GitGuardian caps per_page at 100. None for endpoints that take no paging params.
+ page_size: Optional[int] = 100
primary_keys: list[str] = field(default_factory=lambda: ["id"])
should_sync_default: bool = True
# Scope named in permission errors when the probe can't extract the API's own detail message.
@@ -44,6 +45,11 @@ class GitGuardianEndpointConfig:
# parent's `id` into the `{parent_id}` placeholder. The probe for scope checks hits the parent
# instead, since the child path can't be requested without a real id.
parent_endpoint: Optional[str] = None
+ # Column that receives the parent's `id` on each child row, for children whose rows don't
+ # carry it themselves. Needed to keep the primary key unique across parents.
+ parent_id_field: Optional[str] = None
+ # Other endpoints whose scopes this table also needs; the permission probe checks each one.
+ also_requires: tuple[str, ...] = ()
statuses: tuple[str, ...] = ()
@@ -155,6 +161,59 @@ GITGUARDIAN_ENDPOINTS: dict[str, GitGuardianEndpointConfig] = {
should_sync_default=False,
incremental_fields=[],
),
+ # Incident notes: analyst comments left on each secret incident, fanned out per incident.
+ # Notes can be edited (`updated_at`) and there is no server-side time filter, so full
+ # refresh. Off by default because it costs at least one request per incident.
+ "secret_incident_notes": GitGuardianEndpointConfig(
+ name="secret_incident_notes",
+ path="/v1/incidents/secrets/{parent_id}/notes",
+ parent_endpoint="secret_incidents",
+ partition_key="created_at",
+ ordering="created_at",
+ primary_keys=["incident_id", "id"],
+ required_scope="incidents:read",
+ should_sync_default=False,
+ incremental_fields=[],
+ ),
+ # Incident leaks: the public places (GitHub repos, issues, gists) where each incident's secret
+ # surfaced. The endpoint is unpaginated and its rows carry no id or incident id, so the
+ # incident id is stamped on and the leak URL completes the key. Off by default because it
+ # costs one request per incident.
+ "secret_incident_leaks": GitGuardianEndpointConfig(
+ name="secret_incident_leaks",
+ path="/v1/incidents/secrets/{parent_id}/leaks",
+ parent_endpoint="secret_incidents",
+ parent_id_field="incident_id",
+ page_size=None,
+ primary_keys=["incident_id", "url"],
+ required_scope="incidents:read",
+ should_sync_default=False,
+ incremental_fields=[],
+ ),
+ # Team sources: the source <-> team join table, fanned out per team. Rows are full source
+ # objects with no team id, so the team id is stamped on to keep the key unique across teams.
+ # Also needs `sources:read`, so the probe checks `/v1/sources` too.
+ "team_sources": GitGuardianEndpointConfig(
+ name="team_sources",
+ path="/v1/teams/{parent_id}/sources",
+ parent_endpoint="teams",
+ parent_id_field="team_id",
+ also_requires=("sources",),
+ primary_keys=["team_id", "id"],
+ required_scope="teams:read",
+ incremental_fields=[],
+ ),
+ # Public perimeter developers: developers GitGuardian links to the organization for public
+ # monitoring. The endpoint takes no paging params and has no time filter, so full refresh.
+ # Off by default because it needs the Public Monitoring product.
+ "public_developers": GitGuardianEndpointConfig(
+ name="public_developers",
+ path="/v1/public-perimeter/developers",
+ page_size=None,
+ required_scope="public-perimeter:read",
+ should_sync_default=False,
+ incremental_fields=[],
+ ),
# Secret detectors: lookup for the detector stamped on incidents and occurrences. Keyed by
# `name` (the API has no numeric id); open/resolved counts mutate, so full refresh.
"secret_detectors": GitGuardianEndpointConfig(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/gitguardian/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/gitguardian/source.py
index b04086e3279b..9d9da380fc2d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/gitguardian/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/gitguardian/source.py
@@ -60,9 +60,9 @@ class GitguardianSource(ResumableSource[GitguardianSourceConfig, GitGuardianResu
category=DataWarehouseSourceCategory.ENGINEERING___MONITORING,
label="GitGuardian",
releaseStatus=ReleaseStatus.BETA,
- caption="""Enter a GitGuardian API token to sync your secret incidents, public secret incidents, occurrences, incident activity logs, secret detectors, custom tags, monitored sources, honeytokens, honeytoken events, members, teams, team memberships, and audit logs into the PostHog Data warehouse.
+ caption="""Enter a GitGuardian API token to sync your secret incidents, public secret incidents, occurrences, incident activity logs, incident notes, incident leaks, secret detectors, custom tags, monitored sources, honeytokens, honeytoken events, members, teams, team memberships, team sources, public perimeter developers, and audit logs into the PostHog Data warehouse.
-Create a service account token (recommended for unattended syncs) or a personal access token in your [GitGuardian API settings](https://dashboard.gitguardian.com/api). Grant the read scopes for the tables you want to sync: `incidents:read`, `custom_tags:read`, `sources:read`, `honeytokens:read`, `members:read`, `teams:read`, and `audit_logs:read`. Reading incidents requires a token with at least the Manager access level.
+Create a service account token (recommended for unattended syncs) or a personal access token in your [GitGuardian API settings](https://dashboard.gitguardian.com/api). Grant the read scopes for the tables you want to sync: `incidents:read`, `custom_tags:read`, `sources:read`, `honeytokens:read`, `members:read`, `teams:read`, `audit_logs:read`, and `public-perimeter:read`. Reading incidents requires a token with at least the Manager access level.
Workspaces on the EU instance should set the API URL to `https://api.eu1.gitguardian.com`; self-hosted instances should enter their own API URL.
""",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/gitguardian/tests/test_gitguardian.py b/products/warehouse_sources/backend/temporal/data_imports/sources/gitguardian/tests/test_gitguardian.py
index d11134c70318..9af585f7eae5 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/gitguardian/tests/test_gitguardian.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/gitguardian/tests/test_gitguardian.py
@@ -172,6 +172,60 @@ class TestFanOut:
]
assert result.saved_states == []
+ @parameterized.expand(
+ [
+ (
+ "notes_carry_incident_id",
+ "secret_incident_notes",
+ [
+ f"{BASE_URL}/v1/incidents/secrets?per_page=100&ordering=date",
+ f"{BASE_URL}/v1/incidents/secrets/7/notes?per_page=100&ordering=created_at",
+ ],
+ [{"id": 3, "incident_id": 7, "comment": "Rotated the key"}],
+ [{"id": 3, "incident_id": 7, "comment": "Rotated the key"}],
+ ),
+ (
+ "leaks_are_unpaginated_and_get_incident_id",
+ "secret_incident_leaks",
+ [
+ f"{BASE_URL}/v1/incidents/secrets?per_page=100&ordering=date",
+ f"{BASE_URL}/v1/incidents/secrets/7/leaks",
+ ],
+ [{"source": "github_gist", "name": "fake-gist", "url": "https://gist.example.com/abc"}],
+ [
+ {
+ "source": "github_gist",
+ "name": "fake-gist",
+ "url": "https://gist.example.com/abc",
+ "incident_id": 7,
+ }
+ ],
+ ),
+ (
+ "team_sources_get_team_id",
+ "team_sources",
+ [f"{BASE_URL}/v1/teams?per_page=100", f"{BASE_URL}/v1/teams/7/sources?per_page=100"],
+ [{"id": 42, "full_name": "example-org/example-repo"}],
+ [{"id": 42, "full_name": "example-org/example-repo", "team_id": 7}],
+ ),
+ ]
+ )
+ def test_child_rows_are_keyed_to_their_parent(
+ self,
+ _name: str,
+ endpoint: str,
+ expected_urls: list[str],
+ child_rows: list[dict[str, Any]],
+ expected_rows: list[dict[str, Any]],
+ ) -> None:
+ result = _driver().run(endpoint, [_page([{"id": 7}]), _page(child_rows)])
+ assert result.raised is None
+ assert result.urls == expected_urls
+ assert result.rows == expected_rows
+ assert result.response is not None
+ primary_keys = result.response.primary_keys or []
+ assert primary_keys and all(key in row for row in result.rows for key in primary_keys)
+
def test_parent_deleted_mid_sync_is_skipped(self) -> None:
result = _driver().run(
"team_memberships",
@@ -276,6 +330,7 @@ class TestCheckEndpointAccess:
("honeytokens", "honeytokens:read"),
("custom_tags", "custom_tags:read"),
("audit_logs", "audit_logs:read"),
+ ("public_developers", "public-perimeter:read"),
]
)
def test_denial_without_detail_names_the_required_scope(self, endpoint: str, scope: str) -> None:
@@ -314,6 +369,22 @@ class TestCheckEndpointAccess:
check_endpoint_access("gg_sat_x", BASE_URL, endpoint)
assert [request.path for request in network.requests_log] == [parent_path]
+ def test_team_sources_probes_teams_and_sources(self) -> None:
+ with scripted_network([ScriptedResponse(status=200), ScriptedResponse(status=200)]) as network:
+ reason = check_endpoint_access("gg_sat_x", BASE_URL, "team_sources")
+ assert reason is None
+ assert [request.path for request in network.requests_log] == ["/v1/teams", "/v1/sources"]
+
+ def test_team_sources_without_sources_scope_is_blocked(self) -> None:
+ with scripted_network([ScriptedResponse(status=200), ScriptedResponse(status=403, body=b"no json")]):
+ reason = check_endpoint_access("gg_sat_x", BASE_URL, "team_sources")
+ assert reason is not None and "sources:read" in reason
+
+ def test_unpaginated_endpoint_is_probed_without_paging_params(self) -> None:
+ with scripted_network([ScriptedResponse(status=200)]) as network:
+ check_endpoint_access("gg_sat_x", BASE_URL, "public_developers")
+ assert [request.url for request in network.requests_log] == [f"{BASE_URL}/v1/public-perimeter/developers"]
+
class TestFetchPageRetries:
@parameterized.expand(