:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(warehouse): add gologin tags, folders, deleted profiles and cloud usage tables (#121116)
Tom Owers2dbe9312784b
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 90c14178675d..955475096af6 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -5430,16 +5430,16 @@ Note: The swagger-ui URL recorded in the source (https://customapi.goldcast.io/s## GoLogin — gaps-Today (3): `profiles`, `proxy_devices`, `workspaces`+Today (7): `cloud_usage_daily`, `deleted_profiles`, `folders`, `profiles`, `proxy_devices`, `tags`, `workspaces`Diffed against: <https://api.gologin.com/docs-json>-- [ ] `Folders (GET /folders)` — explain folder references on synced profiles and group profile activity by folder (high)-- [ ] `Tags (GET /tags/all)` — resolve profile tag references for consistent analysis of browser profile groups (high)-- [ ] `Workspace cloud usage statistics (GET /cloud-usage/stats)` — measure daily cloud launches, runtime minutes, and consumption against workspace limits (high)+- [x] `Folders (GET /folders)` — explain folder references on synced profiles and group profile activity by folder (high)+- [x] `Tags (GET /tags/all)` — resolve profile tag references for consistent analysis of browser profile groups (high)+- [x] `Workspace cloud usage statistics (GET /cloud-usage/stats)` — measure daily cloud launches, runtime minutes, and consumption against workspace limits (high)- [ ] `Profile cloud usage statistics (GET /cloud-usage/profile/{browserId}/stats)` — attribute cloud runtime to individual profiles and identify profiles with high resource consumption (medium)- [ ] `Workspace members (GET /workspaces/{wid}, members collection)` — join workspace members and roles to profile activity for collaboration and access analysis (medium)-- [ ] `Deleted profiles (GET /deleted-profiles/v2)` — track profile deletion and retain lifecycle coverage beyond the current active profile inventory (medium)+- [x] `Deleted profiles (GET /deleted-profiles/v2)` — track profile deletion and retain lifecycle coverage beyond the current active profile inventory (medium)- [ ] `GoLogin proxy traffic (GET /users-proxies/geolocation/traffic)` — monitor remaining proxy traffic allowances alongside cloud usage and profile activity (medium)Note: Parsed all GET operations and response schemas in the official OpenAPI document. The profiles endpoint already covers all workspaces; workspace list responses omit the members available from workspace detail reads. Cloud usage needs workspace or profile fan-out, and workspace statistics cover at most 30 days per request. Excluded cookies, fingerprints, credentials, proxy configuration, and workspace settings.diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/gologin/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/gologin/canonical_descriptions.pyindex e376ee707e3d..05043963bb95 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/gologin/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/gologin/canonical_descriptions.py@@ -42,4 +42,50 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"updatedAt": "Time when the device was last updated.",},},+ "tags": {+ "description": "Tags and custom statuses that can be assigned to profiles, one row per workspace and tag.",+ "docs_url": "https://gologin.com/docs/api-reference/tags/get-all-tags",+ "columns": {+ "id": "Unique identifier of the tag.",+ "workspace_id": "Identifier of the workspace that contains the tag.",+ "title": "Name of the tag.",+ "color": "Display color of the tag.",+ "field": "Whether the entry is a tag (`tags`) or a custom status (`custom-status`).",+ },+ },+ "folders": {+ "description": "Profile folders in each workspace, with the current user's folder permissions.",+ "docs_url": "https://gologin.com/docs/api-reference/workspace/get-workspace-info",+ "columns": {+ "id": "Unique identifier of the folder.",+ "workspace_id": "Identifier of the workspace that contains the folder.",+ "name": "Name of the folder. Profiles reference folders by this name.",+ "permissions": "Actions the current user can take in the folder.",+ "order": "Position of the folder in the folder list.",+ "associatedProfiles": "Identifiers of the profiles in the folder.",+ "isAllProfiles": "Whether the folder is the workspace's all-profiles folder.",+ },+ },+ "deleted_profiles": {+ "description": "Profiles deleted from a workspace. GoLogin keeps a deleted profile for 120 hours.",+ "docs_url": "https://gologin.com/docs/api-reference/profile/get-list-of-deleted-profiles",+ "columns": {+ "profileId": "Identifier of the deleted profile.",+ "workspace_id": "Identifier of the workspace the profile was deleted from.",+ "name": "Name of the deleted profile.",+ "deletedAt": "Time when the profile was deleted.",+ "deletedByUserEmail": "Email of the user who deleted the profile.",+ },+ },+ "cloud_usage_daily": {+ "description": "Daily cloud browser usage per workspace.",+ "docs_url": "https://gologin.com/docs/api-reference/cloud-usage/get-detailed-cloud-usage-statistics-for-a-workspace",+ "columns": {+ "date": "Day the usage applies to.",+ "workspace_id": "Identifier of the workspace.",+ "launches": "Number of cloud profile launches on the day.",+ "hours": "Cloud browser hours used on the day.",+ "maxConcurrentInstances": "Highest number of cloud profiles running at the same time on the day.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/gologin/gologin.py b/products/warehouse_sources/backend/temporal/data_imports/sources/gologin/gologin.pyindex 2997eb5e5ed1..548af6885a95 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/gologin/gologin.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/gologin/gologin.py@@ -1,26 +1,41 @@-from typing import Any+from datetime import UTC, date, datetime+from typing import Any, Optional+from requests import Request, Responsefrom requests.exceptions import HTTPErrorfrom posthog.dataclasses import frozen+from products.warehouse_sources.backend.temporal.data_imports.sources.common.datetime_utils import parse_datetime_valuefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (RESTAPIConfig,rest_api_resource,)from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.auth import BearerTokenAuth+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (+ build_chained_resource,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (+ BasePaginator,SinglePagePaginator,)from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.rest_client import RESTClient+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import (+ ClientConfig,+ Endpoint,+ EndpointResource,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManagerfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponsefrom products.warehouse_sources.backend.temporal.data_imports.sources.gologin.settings import (AUTH_ERROR,BASE_URL,+ CLOUD_USAGE_MAX_DAYS,ENDPOINTS,PERMISSION_ERROR,+ PRIMARY_KEYS,REQUEST_TIMEOUT,+ WORKSPACE_ENDPOINTS,)@@ -29,6 +44,40 @@ class GoLoginResumeConfig:page: int+class DeletedProfilesPaginator(BasePaginator):+ """Advances `offset` by the rows received until GoLogin reports no more deleted profiles."""++ def __init__(self) -> None:+ super().__init__()+ self.offset = 0++ def init_request(self, request: Request) -> None:+ request.params = {**(request.params or {}), "offset": self.offset}++ def update_state(self, response: Response, data: Optional[list[Any]] = None) -> None:+ received = len(data or [])+ self.offset += received+ self._has_next_page = received > 0 and response.json().get("isMoreDeletedProfilesAvailable") is True++ def update_request(self, request: Request) -> None:+ request.params = {**(request.params or {}), "offset": self.offset}+++def cloud_usage_days(should_use_incremental_field: bool, db_incremental_field_last_value: Any, today: date) -> int:+ if should_use_incremental_field:+ last_value = parse_datetime_value(db_incremental_field_last_value)+ if last_value is not None:+ # Re-read the watermark day, because it was still in progress when it last synced.+ return min(max((today - last_value.date()).days + 1, 1), CLOUD_USAGE_MAX_DAYS)+ return CLOUD_USAGE_MAX_DAYS+++def _normalize_tag(row: dict[str, Any]) -> dict[str, Any]:+ if "id" not in row and "_id" in row:+ row["id"] = row.pop("_id")+ return row++def validate_credentials(api_key: str) -> tuple[bool, str | None]:client = RESTClient(base_url=BASE_URL,@@ -50,13 +99,80 @@ def validate_credentials(api_key: str) -> tuple[bool, str | None]:return True, None+def _client_config(api_key: str) -> ClientConfig:+ return {+ "base_url": BASE_URL,+ "auth": {"type": "bearer", "token": api_key},+ "allowed_hosts": [],+ "allow_redirects": False,+ "request_timeout": REQUEST_TIMEOUT,+ }+++def _workspace_child_resource(+ api_key: str,+ endpoint: str,+ team_id: int,+ job_id: str,+ should_use_incremental_field: bool,+ db_incremental_field_last_value: Any,+) -> Any:+ params: dict[str, Any] = {"workspace_id": {"type": "resolve", "resource": "workspaces", "field": "id"}}+ if endpoint == "cloud_usage_daily":+ params["days"] = cloud_usage_days(+ should_use_incremental_field, db_incremental_field_last_value, datetime.now(UTC).date()+ )+ child_endpoint: Endpoint = {**WORKSPACE_ENDPOINTS[endpoint], "params": params}+ if endpoint == "deleted_profiles":+ child_endpoint["paginator"] = DeletedProfilesPaginator()++ parent: EndpointResource = {+ "name": "workspaces",+ "table_name": "workspaces",+ "write_disposition": "replace",+ "endpoint": Endpoint(**ENDPOINTS["workspaces"]),+ "table_format": "delta",+ }+ child: EndpointResource = {+ "name": endpoint,+ "table_name": endpoint,+ "write_disposition": (+ {"disposition": "merge", "strategy": "upsert"} if should_use_incremental_field else "replace"+ ),+ "include_from_parent": ["id"],+ "endpoint": child_endpoint,+ "table_format": "delta",+ }+ if endpoint == "tags":+ # GoLogin returns a tag id as `_id` or as `id`.+ child["data_map"] = _normalize_tag++ return build_chained_resource(+ resources=[parent, child],+ child_name=endpoint,+ parent_name="workspaces",+ parent_field_renames={"id": "workspace_id"},+ client_config=_client_config(api_key),+ team_id=team_id,+ job_id=job_id,+ )++def gologin_source(api_key: str,endpoint: str,team_id: int,job_id: str,resumable_source_manager: ResumableSourceManager[GoLoginResumeConfig],+ should_use_incremental_field: bool = False,+ db_incremental_field_last_value: Any = None,) -> SourceResponse:+ if endpoint in WORKSPACE_ENDPOINTS:+ resource = _workspace_child_resource(+ api_key, endpoint, team_id, job_id, should_use_incremental_field, db_incremental_field_last_value+ )+ return SourceResponse(name=endpoint, items=lambda: resource, primary_keys=PRIMARY_KEYS[endpoint])+endpoint_config = ENDPOINTS[endpoint]initial_state: dict[str, Any] | None = Noneif endpoint == "profiles" and resumable_source_manager.can_resume():@@ -69,13 +185,7 @@ def gologin_source(resumable_source_manager.save_state(GoLoginResumeConfig(page=int(state["page"])))config: RESTAPIConfig = {- "client": {- "base_url": BASE_URL,- "auth": {"type": "bearer", "token": api_key},- "allowed_hosts": [],- "allow_redirects": False,- "request_timeout": REQUEST_TIMEOUT,- },+ "client": _client_config(api_key),"resources": [{"name": endpoint, "endpoint": endpoint_config, "write_disposition": "replace"}],}resource = rest_api_resource(diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/gologin/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/gologin/settings.pyindex e917d8454961..0f79d8467d23 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/gologin/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/gologin/settings.py@@ -1,10 +1,14 @@from typing import castfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import Endpoint+from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldTypeBASE_URL = "https://api.gologin.com"REQUEST_TIMEOUT = (10.0, 60.0)+# The workspace usage endpoint accepts a trailing window of 1 to 30 days.+CLOUD_USAGE_MAX_DAYS = 30+ENDPOINTS = cast(dict[str, Endpoint],{@@ -31,5 +35,66 @@ ENDPOINTS = cast(},)+# Tags, folders, deleted profiles and cloud usage are scoped to one workspace per request,+# so these endpoints fan out over `workspaces` and bind its id into `{workspace_id}`.+WORKSPACE_ENDPOINTS = cast(+ dict[str, Endpoint],+ {+ "tags": {+ "path": "/tags/all?workspace={workspace_id}",+ "data_selector": "tags",+ "data_selector_required": True,+ "paginator": "single_page",+ },+ # GET /folders is the legacy per-user listing with no response schema; the workspace+ # read documents its folders and covers every workspace.+ "folders": {+ "path": "/workspaces/{workspace_id}",+ "data_selector": "folders",+ "data_selector_required": True,+ "paginator": "single_page",+ },+ "deleted_profiles": {+ "path": "/deleted-profiles/v2?currentWorkspace={workspace_id}",+ "data_selector": "deletedProfilesList",+ "data_selector_required": True,+ },+ "cloud_usage_daily": {+ "path": "/cloud-usage/stats?workspaceId={workspace_id}",+ "data_selector": "days",+ "data_selector_required": True,+ "paginator": "single_page",+ },+ },+)++PRIMARY_KEYS: dict[str, list[str]] = {+ "tags": ["workspace_id", "id"],+ "folders": ["workspace_id", "id"],+ "deleted_profiles": ["workspace_id", "profileId"],+ "cloud_usage_daily": ["workspace_id", "date"],+}++INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {+ # GoLogin purges deleted profiles after 120 hours and has no time filter, so each run reads the+ # whole short list. Merging keeps deletions after GoLogin purges them.+ "deleted_profiles": [+ {+ "label": "deletedAt",+ "type": IncrementalFieldType.DateTime,+ "field": "deletedAt",+ "field_type": IncrementalFieldType.DateTime,+ }+ ],+ "cloud_usage_daily": [+ {+ "label": "date",+ "type": IncrementalFieldType.Date,+ "field": "date",+ "field_type": IncrementalFieldType.Date,+ }+ ],+}+AUTH_ERROR = "GoLogin rejected the API key. Check the key in Settings > API."PERMISSION_ERROR = "GoLogin denied access. Check the API key and your account permissions."diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/gologin/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/gologin/source.pyindex ce84a3d95453..9e73dc6c1895 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/gologin/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/gologin/source.py@@ -33,7 +33,9 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.gologin.gofrom products.warehouse_sources.backend.temporal.data_imports.sources.gologin.settings import (AUTH_ERROR,ENDPOINTS,+ INCREMENTAL_FIELDS,PERMISSION_ERROR,+ WORKSPACE_ENDPOINTS,)from products.warehouse_sources.backend.types import ExternalDataSourceType@@ -64,7 +66,9 @@ class GoLoginSource(ResumableSource[GoLoginSourceConfig, GoLoginResumeConfig]):force_refresh: bool = False,api_version: str | None = None,) -> list[SourceSchema]:- return build_endpoint_schemas(ENDPOINTS, {}, names)+ return build_endpoint_schemas(+ [*ENDPOINTS, *WORKSPACE_ENDPOINTS], INCREMENTAL_FIELDS, names, merge_only=INCREMENTAL_FIELDS.keys()+ )def validate_credentials(self,@@ -85,7 +89,13 @@ class GoLoginSource(ResumableSource[GoLoginSourceConfig, GoLoginResumeConfig]):inputs: SourceInputs,) -> SourceResponse:return gologin_source(- config.api_key, inputs.schema_name, inputs.team_id, inputs.job_id, resumable_source_manager+ config.api_key,+ inputs.schema_name,+ inputs.team_id,+ inputs.job_id,+ resumable_source_manager,+ should_use_incremental_field=inputs.should_use_incremental_field,+ db_incremental_field_last_value=inputs.db_incremental_field_last_value,)@propertydiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/gologin/tests/test_gologin.py b/products/warehouse_sources/backend/temporal/data_imports/sources/gologin/tests/test_gologin.pyindex aa0227981a37..9aa66d16aaa7 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/gologin/tests/test_gologin.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/gologin/tests/test_gologin.py@@ -1,4 +1,7 @@+from typing import Any+import pytest+import time_machinefrom requests.exceptions import HTTPError@@ -70,6 +73,125 @@ class TestGoLoginTransport:assert any(key in str(result.raised) and message for key, message in errors.items())assert result.paths == ["/browser/v2"]+ @pytest.mark.parametrize(+ "endpoint,path,workspace_query,child_bodies,expected_rows",+ [+ (+ "tags",+ "/tags/all",+ "workspace",+ [+ {"success": True, "tags": [{"_id": "tag-1", "title": "ads", "color": "blue"}]},+ {"success": True, "tags": [{"id": "tag-2", "title": "seo", "color": "pink"}]},+ ],+ [+ {"id": "tag-1", "title": "ads", "color": "blue", "workspace_id": "ws-1"},+ {"id": "tag-2", "title": "seo", "color": "pink", "workspace_id": "ws-2"},+ ],+ ),+ (+ "folders",+ None,+ None,+ [+ {"id": "ws-1", "folders": [{"id": "folder-1", "name": "Team A"}]},+ {"id": "ws-2", "folders": [{"id": "folder-2", "name": "Team B"}]},+ ],+ [+ {"id": "folder-1", "name": "Team A", "workspace_id": "ws-1"},+ {"id": "folder-2", "name": "Team B", "workspace_id": "ws-2"},+ ],+ ),+ (+ "cloud_usage_daily",+ "/cloud-usage/stats",+ "workspaceId",+ [+ {"days": [{"date": "2026-10-20", "launches": 3, "hours": 1.5}], "limits": {}},+ {"days": [{"date": "2026-10-20", "launches": 0, "hours": 0}], "limits": {}},+ ],+ [+ {"date": "2026-10-20", "launches": 3, "hours": 1.5, "workspace_id": "ws-1"},+ {"date": "2026-10-20", "launches": 0, "hours": 0, "workspace_id": "ws-2"},+ ],+ ),+ ],+ )+ def test_workspace_endpoints_fan_out_per_workspace(+ self,+ endpoint: str,+ path: str | None,+ workspace_query: str | None,+ child_bodies: list[dict[str, Any]],+ expected_rows: list[dict[str, Any]],+ ) -> None:+ result = self._driver().run(+ endpoint,+ [+ ScriptedResponse(json=[{"id": "ws-1", "name": "One"}, {"id": "ws-2", "name": "Two"}]),+ *(ScriptedResponse(json=body) for body in child_bodies),+ ],+ )++ assert result.raised is None+ assert result.rows == expected_rows+ assert result.paths[0] == "/workspaces"+ for workspace_id, request in zip(["ws-1", "ws-2"], result.requests[1:]):+ if workspace_query is None:+ assert request.path == f"/workspaces/{workspace_id}"+ else:+ assert request.path == path+ assert request.query[workspace_query] == (workspace_id,)++ def test_deleted_profiles_advance_offset_until_gologin_reports_no_more(self) -> None:+ result = self._driver().run(+ "deleted_profiles",+ [+ ScriptedResponse(json=[{"id": "ws-1"}]),+ ScriptedResponse(+ json={+ "deletedProfilesList": [{"profileId": "p-1"}, {"profileId": "p-2"}],+ "isMoreDeletedProfilesAvailable": True,+ }+ ),+ ScriptedResponse(+ json={"deletedProfilesList": [{"profileId": "p-3"}], "isMoreDeletedProfilesAvailable": False}+ ),+ ],+ )++ assert result.raised is None+ assert [row["profileId"] for row in result.rows] == ["p-1", "p-2", "p-3"]+ assert {row["workspace_id"] for row in result.rows} == {"ws-1"}+ assert [request.query for request in result.requests[1:]] == [+ {"currentWorkspace": ("ws-1",), "offset": ("0",)},+ {"currentWorkspace": ("ws-1",), "offset": ("2",)},+ ]++ @time_machine.travel("2026-10-21T12:00:00Z", tick=False)+ @pytest.mark.parametrize(+ "incremental,last_value,expected_days",+ [+ (False, "2026-10-18", "30"),+ (True, None, "30"),+ (True, "2026-10-18", "4"),+ (True, "2026-10-21", "1"),+ (True, "2026-01-01", "30"),+ ],+ )+ def test_cloud_usage_window_starts_at_the_watermark_day(+ self, incremental: bool, last_value: str | None, expected_days: str+ ) -> None:+ result = self._driver().run(+ "cloud_usage_daily",+ [ScriptedResponse(json=[{"id": "ws-1"}]), ScriptedResponse(json={"days": []})],+ should_use_incremental_field=incremental,+ db_incremental_field_last_value=last_value,+ )++ assert result.raised is None+ assert result.requests[1].query["days"] == (expected_days,)+def test_pipeline_refuses_redirect(self) -> None:result = self._driver().run("profiles",