Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(warehouse): add gologin tags, folders, deleted profiles and cloud usage tables (#121116)

Tom Owers2dbe9312784b
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 90c14178675d..955475096af6 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -5430,16 +5430,16 @@ Note: The swagger-ui URL recorded in the source (https://customapi.goldcast.io/s
## GoLogin — gaps
-Today (3): `profiles`, `proxy_devices`, `workspaces`
+Today (7): `cloud_usage_daily`, `deleted_profiles`, `folders`, `profiles`, `proxy_devices`, `tags`, `workspaces`
Diffed against: <https://api.gologin.com/docs-json>
-- [ ] `Folders (GET /folders)` — explain folder references on synced profiles and group profile activity by folder (high)
-- [ ] `Tags (GET /tags/all)` — resolve profile tag references for consistent analysis of browser profile groups (high)
-- [ ] `Workspace cloud usage statistics (GET /cloud-usage/stats)` — measure daily cloud launches, runtime minutes, and consumption against workspace limits (high)
+- [x] `Folders (GET /folders)` — explain folder references on synced profiles and group profile activity by folder (high)
+- [x] `Tags (GET /tags/all)` — resolve profile tag references for consistent analysis of browser profile groups (high)
+- [x] `Workspace cloud usage statistics (GET /cloud-usage/stats)` — measure daily cloud launches, runtime minutes, and consumption against workspace limits (high)
- [ ] `Profile cloud usage statistics (GET /cloud-usage/profile/{browserId}/stats)` — attribute cloud runtime to individual profiles and identify profiles with high resource consumption (medium)
- [ ] `Workspace members (GET /workspaces/{wid}, members collection)` — join workspace members and roles to profile activity for collaboration and access analysis (medium)
-- [ ] `Deleted profiles (GET /deleted-profiles/v2)` — track profile deletion and retain lifecycle coverage beyond the current active profile inventory (medium)
+- [x] `Deleted profiles (GET /deleted-profiles/v2)` — track profile deletion and retain lifecycle coverage beyond the current active profile inventory (medium)
- [ ] `GoLogin proxy traffic (GET /users-proxies/geolocation/traffic)` — monitor remaining proxy traffic allowances alongside cloud usage and profile activity (medium)
Note: Parsed all GET operations and response schemas in the official OpenAPI document. The profiles endpoint already covers all workspaces; workspace list responses omit the members available from workspace detail reads. Cloud usage needs workspace or profile fan-out, and workspace statistics cover at most 30 days per request. Excluded cookies, fingerprints, credentials, proxy configuration, and workspace settings.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/gologin/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/gologin/canonical_descriptions.py
index e376ee707e3d..05043963bb95 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/gologin/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/gologin/canonical_descriptions.py
@@ -42,4 +42,50 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"updatedAt": "Time when the device was last updated.",
},
},
+ "tags": {
+ "description": "Tags and custom statuses that can be assigned to profiles, one row per workspace and tag.",
+ "docs_url": "https://gologin.com/docs/api-reference/tags/get-all-tags",
+ "columns": {
+ "id": "Unique identifier of the tag.",
+ "workspace_id": "Identifier of the workspace that contains the tag.",
+ "title": "Name of the tag.",
+ "color": "Display color of the tag.",
+ "field": "Whether the entry is a tag (`tags`) or a custom status (`custom-status`).",
+ },
+ },
+ "folders": {
+ "description": "Profile folders in each workspace, with the current user's folder permissions.",
+ "docs_url": "https://gologin.com/docs/api-reference/workspace/get-workspace-info",
+ "columns": {
+ "id": "Unique identifier of the folder.",
+ "workspace_id": "Identifier of the workspace that contains the folder.",
+ "name": "Name of the folder. Profiles reference folders by this name.",
+ "permissions": "Actions the current user can take in the folder.",
+ "order": "Position of the folder in the folder list.",
+ "associatedProfiles": "Identifiers of the profiles in the folder.",
+ "isAllProfiles": "Whether the folder is the workspace's all-profiles folder.",
+ },
+ },
+ "deleted_profiles": {
+ "description": "Profiles deleted from a workspace. GoLogin keeps a deleted profile for 120 hours.",
+ "docs_url": "https://gologin.com/docs/api-reference/profile/get-list-of-deleted-profiles",
+ "columns": {
+ "profileId": "Identifier of the deleted profile.",
+ "workspace_id": "Identifier of the workspace the profile was deleted from.",
+ "name": "Name of the deleted profile.",
+ "deletedAt": "Time when the profile was deleted.",
+ "deletedByUserEmail": "Email of the user who deleted the profile.",
+ },
+ },
+ "cloud_usage_daily": {
+ "description": "Daily cloud browser usage per workspace.",
+ "docs_url": "https://gologin.com/docs/api-reference/cloud-usage/get-detailed-cloud-usage-statistics-for-a-workspace",
+ "columns": {
+ "date": "Day the usage applies to.",
+ "workspace_id": "Identifier of the workspace.",
+ "launches": "Number of cloud profile launches on the day.",
+ "hours": "Cloud browser hours used on the day.",
+ "maxConcurrentInstances": "Highest number of cloud profiles running at the same time on the day.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/gologin/gologin.py b/products/warehouse_sources/backend/temporal/data_imports/sources/gologin/gologin.py
index 2997eb5e5ed1..548af6885a95 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/gologin/gologin.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/gologin/gologin.py
@@ -1,26 +1,41 @@
-from typing import Any
+from datetime import UTC, date, datetime
+from typing import Any, Optional
+from requests import Request, Response
from requests.exceptions import HTTPError
from posthog.dataclasses import frozen
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.datetime_utils import parse_datetime_value
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (
RESTAPIConfig,
rest_api_resource,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.auth import BearerTokenAuth
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
+ build_chained_resource,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
+ BasePaginator,
SinglePagePaginator,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.rest_client import RESTClient
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import (
+ ClientConfig,
+ Endpoint,
+ EndpointResource,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.gologin.settings import (
AUTH_ERROR,
BASE_URL,
+ CLOUD_USAGE_MAX_DAYS,
ENDPOINTS,
PERMISSION_ERROR,
+ PRIMARY_KEYS,
REQUEST_TIMEOUT,
+ WORKSPACE_ENDPOINTS,
)
@@ -29,6 +44,40 @@ class GoLoginResumeConfig:
page: int
+class DeletedProfilesPaginator(BasePaginator):
+ """Advances `offset` by the rows received until GoLogin reports no more deleted profiles."""
+
+ def __init__(self) -> None:
+ super().__init__()
+ self.offset = 0
+
+ def init_request(self, request: Request) -> None:
+ request.params = {**(request.params or {}), "offset": self.offset}
+
+ def update_state(self, response: Response, data: Optional[list[Any]] = None) -> None:
+ received = len(data or [])
+ self.offset += received
+ self._has_next_page = received > 0 and response.json().get("isMoreDeletedProfilesAvailable") is True
+
+ def update_request(self, request: Request) -> None:
+ request.params = {**(request.params or {}), "offset": self.offset}
+
+
+def cloud_usage_days(should_use_incremental_field: bool, db_incremental_field_last_value: Any, today: date) -> int:
+ if should_use_incremental_field:
+ last_value = parse_datetime_value(db_incremental_field_last_value)
+ if last_value is not None:
+ # Re-read the watermark day, because it was still in progress when it last synced.
+ return min(max((today - last_value.date()).days + 1, 1), CLOUD_USAGE_MAX_DAYS)
+ return CLOUD_USAGE_MAX_DAYS
+
+
+def _normalize_tag(row: dict[str, Any]) -> dict[str, Any]:
+ if "id" not in row and "_id" in row:
+ row["id"] = row.pop("_id")
+ return row
+
+
def validate_credentials(api_key: str) -> tuple[bool, str | None]:
client = RESTClient(
base_url=BASE_URL,
@@ -50,13 +99,80 @@ def validate_credentials(api_key: str) -> tuple[bool, str | None]:
return True, None
+def _client_config(api_key: str) -> ClientConfig:
+ return {
+ "base_url": BASE_URL,
+ "auth": {"type": "bearer", "token": api_key},
+ "allowed_hosts": [],
+ "allow_redirects": False,
+ "request_timeout": REQUEST_TIMEOUT,
+ }
+
+
+def _workspace_child_resource(
+ api_key: str,
+ endpoint: str,
+ team_id: int,
+ job_id: str,
+ should_use_incremental_field: bool,
+ db_incremental_field_last_value: Any,
+) -> Any:
+ params: dict[str, Any] = {"workspace_id": {"type": "resolve", "resource": "workspaces", "field": "id"}}
+ if endpoint == "cloud_usage_daily":
+ params["days"] = cloud_usage_days(
+ should_use_incremental_field, db_incremental_field_last_value, datetime.now(UTC).date()
+ )
+ child_endpoint: Endpoint = {**WORKSPACE_ENDPOINTS[endpoint], "params": params}
+ if endpoint == "deleted_profiles":
+ child_endpoint["paginator"] = DeletedProfilesPaginator()
+
+ parent: EndpointResource = {
+ "name": "workspaces",
+ "table_name": "workspaces",
+ "write_disposition": "replace",
+ "endpoint": Endpoint(**ENDPOINTS["workspaces"]),
+ "table_format": "delta",
+ }
+ child: EndpointResource = {
+ "name": endpoint,
+ "table_name": endpoint,
+ "write_disposition": (
+ {"disposition": "merge", "strategy": "upsert"} if should_use_incremental_field else "replace"
+ ),
+ "include_from_parent": ["id"],
+ "endpoint": child_endpoint,
+ "table_format": "delta",
+ }
+ if endpoint == "tags":
+ # GoLogin returns a tag id as `_id` or as `id`.
+ child["data_map"] = _normalize_tag
+
+ return build_chained_resource(
+ resources=[parent, child],
+ child_name=endpoint,
+ parent_name="workspaces",
+ parent_field_renames={"id": "workspace_id"},
+ client_config=_client_config(api_key),
+ team_id=team_id,
+ job_id=job_id,
+ )
+
+
def gologin_source(
api_key: str,
endpoint: str,
team_id: int,
job_id: str,
resumable_source_manager: ResumableSourceManager[GoLoginResumeConfig],
+ should_use_incremental_field: bool = False,
+ db_incremental_field_last_value: Any = None,
) -> SourceResponse:
+ if endpoint in WORKSPACE_ENDPOINTS:
+ resource = _workspace_child_resource(
+ api_key, endpoint, team_id, job_id, should_use_incremental_field, db_incremental_field_last_value
+ )
+ return SourceResponse(name=endpoint, items=lambda: resource, primary_keys=PRIMARY_KEYS[endpoint])
+
endpoint_config = ENDPOINTS[endpoint]
initial_state: dict[str, Any] | None = None
if endpoint == "profiles" and resumable_source_manager.can_resume():
@@ -69,13 +185,7 @@ def gologin_source(
resumable_source_manager.save_state(GoLoginResumeConfig(page=int(state["page"])))
config: RESTAPIConfig = {
- "client": {
- "base_url": BASE_URL,
- "auth": {"type": "bearer", "token": api_key},
- "allowed_hosts": [],
- "allow_redirects": False,
- "request_timeout": REQUEST_TIMEOUT,
- },
+ "client": _client_config(api_key),
"resources": [{"name": endpoint, "endpoint": endpoint_config, "write_disposition": "replace"}],
}
resource = rest_api_resource(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/gologin/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/gologin/settings.py
index e917d8454961..0f79d8467d23 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/gologin/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/gologin/settings.py
@@ -1,10 +1,14 @@
from typing import cast
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import Endpoint
+from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType
BASE_URL = "https://api.gologin.com"
REQUEST_TIMEOUT = (10.0, 60.0)
+# The workspace usage endpoint accepts a trailing window of 1 to 30 days.
+CLOUD_USAGE_MAX_DAYS = 30
+
ENDPOINTS = cast(
dict[str, Endpoint],
{
@@ -31,5 +35,66 @@ ENDPOINTS = cast(
},
)
+# Tags, folders, deleted profiles and cloud usage are scoped to one workspace per request,
+# so these endpoints fan out over `workspaces` and bind its id into `{workspace_id}`.
+WORKSPACE_ENDPOINTS = cast(
+ dict[str, Endpoint],
+ {
+ "tags": {
+ "path": "/tags/all?workspace={workspace_id}",
+ "data_selector": "tags",
+ "data_selector_required": True,
+ "paginator": "single_page",
+ },
+ # GET /folders is the legacy per-user listing with no response schema; the workspace
+ # read documents its folders and covers every workspace.
+ "folders": {
+ "path": "/workspaces/{workspace_id}",
+ "data_selector": "folders",
+ "data_selector_required": True,
+ "paginator": "single_page",
+ },
+ "deleted_profiles": {
+ "path": "/deleted-profiles/v2?currentWorkspace={workspace_id}",
+ "data_selector": "deletedProfilesList",
+ "data_selector_required": True,
+ },
+ "cloud_usage_daily": {
+ "path": "/cloud-usage/stats?workspaceId={workspace_id}",
+ "data_selector": "days",
+ "data_selector_required": True,
+ "paginator": "single_page",
+ },
+ },
+)
+
+PRIMARY_KEYS: dict[str, list[str]] = {
+ "tags": ["workspace_id", "id"],
+ "folders": ["workspace_id", "id"],
+ "deleted_profiles": ["workspace_id", "profileId"],
+ "cloud_usage_daily": ["workspace_id", "date"],
+}
+
+INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {
+ # GoLogin purges deleted profiles after 120 hours and has no time filter, so each run reads the
+ # whole short list. Merging keeps deletions after GoLogin purges them.
+ "deleted_profiles": [
+ {
+ "label": "deletedAt",
+ "type": IncrementalFieldType.DateTime,
+ "field": "deletedAt",
+ "field_type": IncrementalFieldType.DateTime,
+ }
+ ],
+ "cloud_usage_daily": [
+ {
+ "label": "date",
+ "type": IncrementalFieldType.Date,
+ "field": "date",
+ "field_type": IncrementalFieldType.Date,
+ }
+ ],
+}
+
AUTH_ERROR = "GoLogin rejected the API key. Check the key in Settings > API."
PERMISSION_ERROR = "GoLogin denied access. Check the API key and your account permissions."
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/gologin/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/gologin/source.py
index ce84a3d95453..9e73dc6c1895 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/gologin/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/gologin/source.py
@@ -33,7 +33,9 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.gologin.go
from products.warehouse_sources.backend.temporal.data_imports.sources.gologin.settings import (
AUTH_ERROR,
ENDPOINTS,
+ INCREMENTAL_FIELDS,
PERMISSION_ERROR,
+ WORKSPACE_ENDPOINTS,
)
from products.warehouse_sources.backend.types import ExternalDataSourceType
@@ -64,7 +66,9 @@ class GoLoginSource(ResumableSource[GoLoginSourceConfig, GoLoginResumeConfig]):
force_refresh: bool = False,
api_version: str | None = None,
) -> list[SourceSchema]:
- return build_endpoint_schemas(ENDPOINTS, {}, names)
+ return build_endpoint_schemas(
+ [*ENDPOINTS, *WORKSPACE_ENDPOINTS], INCREMENTAL_FIELDS, names, merge_only=INCREMENTAL_FIELDS.keys()
+ )
def validate_credentials(
self,
@@ -85,7 +89,13 @@ class GoLoginSource(ResumableSource[GoLoginSourceConfig, GoLoginResumeConfig]):
inputs: SourceInputs,
) -> SourceResponse:
return gologin_source(
- config.api_key, inputs.schema_name, inputs.team_id, inputs.job_id, resumable_source_manager
+ config.api_key,
+ inputs.schema_name,
+ inputs.team_id,
+ inputs.job_id,
+ resumable_source_manager,
+ should_use_incremental_field=inputs.should_use_incremental_field,
+ db_incremental_field_last_value=inputs.db_incremental_field_last_value,
)
@property
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/gologin/tests/test_gologin.py b/products/warehouse_sources/backend/temporal/data_imports/sources/gologin/tests/test_gologin.py
index aa0227981a37..9aa66d16aaa7 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/gologin/tests/test_gologin.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/gologin/tests/test_gologin.py
@@ -1,4 +1,7 @@
+from typing import Any
+
import pytest
+import time_machine
from requests.exceptions import HTTPError
@@ -70,6 +73,125 @@ class TestGoLoginTransport:
assert any(key in str(result.raised) and message for key, message in errors.items())
assert result.paths == ["/browser/v2"]
+ @pytest.mark.parametrize(
+ "endpoint,path,workspace_query,child_bodies,expected_rows",
+ [
+ (
+ "tags",
+ "/tags/all",
+ "workspace",
+ [
+ {"success": True, "tags": [{"_id": "tag-1", "title": "ads", "color": "blue"}]},
+ {"success": True, "tags": [{"id": "tag-2", "title": "seo", "color": "pink"}]},
+ ],
+ [
+ {"id": "tag-1", "title": "ads", "color": "blue", "workspace_id": "ws-1"},
+ {"id": "tag-2", "title": "seo", "color": "pink", "workspace_id": "ws-2"},
+ ],
+ ),
+ (
+ "folders",
+ None,
+ None,
+ [
+ {"id": "ws-1", "folders": [{"id": "folder-1", "name": "Team A"}]},
+ {"id": "ws-2", "folders": [{"id": "folder-2", "name": "Team B"}]},
+ ],
+ [
+ {"id": "folder-1", "name": "Team A", "workspace_id": "ws-1"},
+ {"id": "folder-2", "name": "Team B", "workspace_id": "ws-2"},
+ ],
+ ),
+ (
+ "cloud_usage_daily",
+ "/cloud-usage/stats",
+ "workspaceId",
+ [
+ {"days": [{"date": "2026-10-20", "launches": 3, "hours": 1.5}], "limits": {}},
+ {"days": [{"date": "2026-10-20", "launches": 0, "hours": 0}], "limits": {}},
+ ],
+ [
+ {"date": "2026-10-20", "launches": 3, "hours": 1.5, "workspace_id": "ws-1"},
+ {"date": "2026-10-20", "launches": 0, "hours": 0, "workspace_id": "ws-2"},
+ ],
+ ),
+ ],
+ )
+ def test_workspace_endpoints_fan_out_per_workspace(
+ self,
+ endpoint: str,
+ path: str | None,
+ workspace_query: str | None,
+ child_bodies: list[dict[str, Any]],
+ expected_rows: list[dict[str, Any]],
+ ) -> None:
+ result = self._driver().run(
+ endpoint,
+ [
+ ScriptedResponse(json=[{"id": "ws-1", "name": "One"}, {"id": "ws-2", "name": "Two"}]),
+ *(ScriptedResponse(json=body) for body in child_bodies),
+ ],
+ )
+
+ assert result.raised is None
+ assert result.rows == expected_rows
+ assert result.paths[0] == "/workspaces"
+ for workspace_id, request in zip(["ws-1", "ws-2"], result.requests[1:]):
+ if workspace_query is None:
+ assert request.path == f"/workspaces/{workspace_id}"
+ else:
+ assert request.path == path
+ assert request.query[workspace_query] == (workspace_id,)
+
+ def test_deleted_profiles_advance_offset_until_gologin_reports_no_more(self) -> None:
+ result = self._driver().run(
+ "deleted_profiles",
+ [
+ ScriptedResponse(json=[{"id": "ws-1"}]),
+ ScriptedResponse(
+ json={
+ "deletedProfilesList": [{"profileId": "p-1"}, {"profileId": "p-2"}],
+ "isMoreDeletedProfilesAvailable": True,
+ }
+ ),
+ ScriptedResponse(
+ json={"deletedProfilesList": [{"profileId": "p-3"}], "isMoreDeletedProfilesAvailable": False}
+ ),
+ ],
+ )
+
+ assert result.raised is None
+ assert [row["profileId"] for row in result.rows] == ["p-1", "p-2", "p-3"]
+ assert {row["workspace_id"] for row in result.rows} == {"ws-1"}
+ assert [request.query for request in result.requests[1:]] == [
+ {"currentWorkspace": ("ws-1",), "offset": ("0",)},
+ {"currentWorkspace": ("ws-1",), "offset": ("2",)},
+ ]
+
+ @time_machine.travel("2026-10-21T12:00:00Z", tick=False)
+ @pytest.mark.parametrize(
+ "incremental,last_value,expected_days",
+ [
+ (False, "2026-10-18", "30"),
+ (True, None, "30"),
+ (True, "2026-10-18", "4"),
+ (True, "2026-10-21", "1"),
+ (True, "2026-01-01", "30"),
+ ],
+ )
+ def test_cloud_usage_window_starts_at_the_watermark_day(
+ self, incremental: bool, last_value: str | None, expected_days: str
+ ) -> None:
+ result = self._driver().run(
+ "cloud_usage_daily",
+ [ScriptedResponse(json=[{"id": "ws-1"}]), ScriptedResponse(json={"days": []})],
+ should_use_incremental_field=incremental,
+ db_incremental_field_last_value=last_value,
+ )
+
+ assert result.raised is None
+ assert result.requests[1].query["days"] == (expected_days,)
+
def test_pipeline_refuses_redirect(self) -> None:
result = self._driver().run(
"profiles",