Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(warehouse_sources): add lovable groups, scan findings, and analytics tables (#122648)

Tom Owersb803bf406351
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 83615f7d36e2..fc3ecc15b3e7 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -7509,10 +7509,10 @@ Today (5): `ProjectPiiLabels`, `Projects`, `ProjectSecurityScans`, `WorkspaceMem
Diffed against: <https://api.lovable.dev/v1/openapi.yaml>
-- [ ] `Workspace groups (GET /v1/workspaces/{workspace_id}/groups)` — resolve group audience targets on published projects and report access segmentation (high)
-- [ ] `Project analytics (GET /v1/projects/{project_id}/analytics)` — measure published website traffic and visitors over time (high)
-- [ ] `Security scan findings (GET /v1/projects/{project_id}/security-scans/{scan_id}/findings)` — analyze vulnerability details and remediation states absent from scan metadata (high)
-- [ ] `Workspace analytics (GET /v1/workspaces/{workspace_id}/analytics)` — measure workspace traffic, including historical visits to deleted projects (medium)
+- [x] `Workspace groups (GET /v1/workspaces/{workspace_id}/groups)` — resolve group audience targets on published projects and report access segmentation (high)
+- [x] `Project analytics (GET /v1/projects/{project_id}/analytics)` — measure published website traffic and visitors over time (high)
+- [x] `Security scan findings (GET /v1/projects/{project_id}/security-scans/{scan_id}/findings)` — analyze vulnerability details and remediation states absent from scan metadata (high)
+- [x] `Workspace analytics (GET /v1/workspaces/{workspace_id}/analytics)` — measure workspace traffic, including historical visits to deleted projects (medium)
- [ ] `Project analytics breakdowns (GET /v1/projects/{project_id}/analytics/breakdowns)` — analyze traffic by page, referring domain, device type, and country (medium)
- [ ] `Workspace security project inventory (GET /v1/workspaces/{workspace_id}/security-center/insights/projects)` — compare project review priorities, finding counts, and activity across a workspace (medium)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/lovable/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/lovable/canonical_descriptions.py
index 7fae2934d71e..01d404807520 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/lovable/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/lovable/canonical_descriptions.py
@@ -140,4 +140,63 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"found_at": "When the label was recorded.",
},
},
+ "WorkspaceGroups": {
+ "description": "A group of workspace members, used as an audience target when publishing a project.",
+ "docs_url": _DOCS_URL,
+ "columns": {
+ "id": "Group ID.",
+ "workspace_id": "Workspace the group belongs to.",
+ "name": "Group display name, unique within the workspace.",
+ "description": "Group description.",
+ "scim_external_id": "External ID assigned by the SCIM identity provider. Null for groups not provisioned by SCIM.",
+ "member_count": "Number of members in the group, including pending members and invited users.",
+ "created_at": "When the group was created.",
+ "updated_at": "When the group was last updated.",
+ },
+ },
+ "ProjectSecurityScanFindings": {
+ "description": "One finding in a project security scan snapshot. A scan repeats findings carried over from earlier scans.",
+ "docs_url": _DOCS_URL,
+ "columns": {
+ "project_id": "Project that was scanned.",
+ "workspace_id": "Workspace the project belongs to.",
+ "scan_id": "Security scan the finding belongs to.",
+ "id": "Finding ID. Stable across scans.",
+ "scanner_name": "Scanner that produced the finding.",
+ "status": "Review state: open, fixed, ignored, or suppressed.",
+ "level": "Severity reported by the scanner: error, warn, or info.",
+ "category": "Finding category.",
+ "name": "Short finding title.",
+ "description": "Summary of the finding.",
+ "created_at": "When the finding was first recorded.",
+ "updated_at": "When the finding was last updated.",
+ },
+ },
+ "ProjectAnalytics": {
+ "description": "Daily website traffic for a published project.",
+ "docs_url": _DOCS_URL,
+ "columns": {
+ "project_id": "Project the traffic belongs to.",
+ "workspace_id": "Workspace the project belongs to.",
+ "timestamp": "Start of the daily bucket, in UTC.",
+ "visitors": "Visitor sessions in the bucket.",
+ "pageviews": "Page views in the bucket.",
+ "pageviews_per_visit": "Page views per visit in the bucket.",
+ "session_duration": "Average session duration in seconds.",
+ "bounce_rate": "Bounce rate as a percentage from 0 to 100.",
+ },
+ },
+ "WorkspaceAnalytics": {
+ "description": "Daily website traffic across a workspace, including traffic from projects that were later deleted.",
+ "docs_url": _DOCS_URL,
+ "columns": {
+ "workspace_id": "Workspace the traffic belongs to.",
+ "timestamp": "Start of the daily bucket, in UTC.",
+ "visitors": "Visitor sessions in the bucket.",
+ "pageviews": "Page views in the bucket.",
+ "pageviews_per_visit": "Page views per visit in the bucket.",
+ "session_duration": "Average session duration in seconds.",
+ "bounce_rate": "Bounce rate as a percentage from 0 to 100.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/lovable/lovable.py b/products/warehouse_sources/backend/temporal/data_imports/sources/lovable/lovable.py
index d108b5f78e20..3fd6ae72006f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/lovable/lovable.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/lovable/lovable.py
@@ -1,10 +1,12 @@
from collections.abc import Iterator
+from datetime import UTC, datetime, time, timedelta
from typing import Any, Optional
from requests.exceptions import HTTPError
from posthog.dataclasses import frozen
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.datetime_utils import parse_datetime_value
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.auth import APIKeyAuth
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
@@ -16,6 +18,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res
from products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import validate_via_probe
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.lovable.settings import (
+ ANALYTICS_MAX_RANGE_DAYS,
LOVABLE_API_BASE_URL,
LOVABLE_URL_VERSION_SEGMENT,
LovableEndpointConfig,
@@ -34,20 +37,22 @@ REQUEST_TIMEOUT_SECONDS = 30.0
class LovableResumeConfig:
"""Where a resumed attempt picks the fan-out back up.
- `workspace_id`/`project_id` name the parent the checkpoint belongs to (both null for the
- account-scoped endpoint). A set `cursor` means "continue that parent's pages here"; a null one
+ `workspace_id`/`project_id`/`scan_id` name the parent the checkpoint belongs to (all null for
+ the account-scoped endpoint). A set `cursor` means "continue that parent's pages here"; a null one
means the parent finished, so the walk restarts at the parent after it.
"""
workspace_id: Optional[str] = None
project_id: Optional[str] = None
cursor: Optional[str] = None
+ scan_id: Optional[str] = None
@frozen
class _ParentRef:
workspace_id: Optional[str] = None
project_id: Optional[str] = None
+ scan_id: Optional[str] = None
def _version_headers(api_version: str) -> dict[str, str]:
@@ -75,20 +80,60 @@ def _paginator() -> JSONResponseCursorPaginator:
return JSONResponseCursorPaginator(cursor_path="pagination.next_cursor", cursor_param="cursor")
+def _endpoint_paginator(config: LovableEndpointConfig) -> JSONResponseCursorPaginator | SinglePagePaginator:
+ return SinglePagePaginator() if config.time_series else _paginator()
+
+
+def _data_selector(config: LovableEndpointConfig) -> str:
+ return "time_series" if config.time_series else "data"
+
+
def _path(config: LovableEndpointConfig, parent: _ParentRef) -> str:
return f"/{LOVABLE_URL_VERSION_SEGMENT}{config.path}".format(
workspace_id=parent.workspace_id or "",
project_id=parent.project_id or "",
+ scan_id=parent.scan_id or "",
)
def _params(config: LovableEndpointConfig, parent: _ParentRef) -> dict[str, Any]:
- params: dict[str, Any] = {"limit": config.page_size, **config.params}
+ params: dict[str, Any] = dict(config.params) if config.time_series else {"limit": config.page_size, **config.params}
if config.parent_id_param and parent.workspace_id:
params[config.parent_id_param] = parent.workspace_id
return params
+def _rfc3339(value: datetime) -> str:
+ return value.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+
+def analytics_window(should_use_incremental_field: bool, last_value: Any, now: datetime) -> dict[str, str]:
+ """`starts_at`/`ends_at` for an analytics request.
+
+ A full refresh reads the longest range the API allows. An incremental run starts at the UTC day
+ of the watermark, because the newest daily bucket is still filling up when it is first read.
+ """
+ earliest = datetime.combine(now.date() - timedelta(days=ANALYTICS_MAX_RANGE_DAYS - 1), time.min, tzinfo=UTC)
+ starts_at = earliest
+ last = parse_datetime_value(last_value) if should_use_incremental_field else None
+ if last is not None:
+ starts_at = max(earliest, datetime.combine(last.date(), time.min, tzinfo=UTC))
+ return {"starts_at": _rfc3339(starts_at), "ends_at": _rfc3339(now)}
+
+
+def _time_series_rows(time_series: dict[str, Any]) -> list[dict[str, Any]]:
+ """Pivot `{metric: {data: [{timestamp, value}]}}` into one row per bucket with a column per metric."""
+ rows: dict[str, dict[str, Any]] = {}
+ for metric, series in time_series.items():
+ if not isinstance(series, dict):
+ continue
+ for point in series.get("data") or []:
+ timestamp = point.get("timestamp")
+ if timestamp:
+ rows.setdefault(timestamp, {"timestamp": timestamp})[metric] = point.get("value")
+ return [rows[timestamp] for timestamp in sorted(rows)]
+
+
def _iter_rows(client: RESTClient, path: str, params: dict[str, Any]) -> Iterator[dict[str, Any]]:
for page in client.paginate(path=path, params=params, paginator=_paginator(), data_selector="data"):
yield from page
@@ -113,8 +158,18 @@ def _iter_parents(client: RESTClient, api_version: str, config: LovableEndpointC
workspace_ref = _ParentRef(workspace_id=workspace_id)
for project in _iter_rows(client, _path(projects, workspace_ref), _params(projects, workspace_ref)):
project_id = project.get("id")
- if project_id:
- yield _ParentRef(workspace_id=workspace_id, project_id=project_id)
+ if not project_id or (config.published_projects_only and not project.get("is_published")):
+ continue
+ project_ref = _ParentRef(workspace_id=workspace_id, project_id=project_id)
+ if config.scope == "project":
+ yield project_ref
+ continue
+
+ scans = endpoints["ProjectSecurityScans"]
+ for scan in _iter_rows(client, _path(scans, project_ref), _params(scans, project_ref)):
+ scan_id = scan.get("id")
+ if scan_id:
+ yield _ParentRef(workspace_id=workspace_id, project_id=project_id, scan_id=scan_id)
def resume_position(parents: list[_ParentRef], resume: Optional[LovableResumeConfig]) -> tuple[int, Optional[str]]:
@@ -126,7 +181,7 @@ def resume_position(parents: list[_ParentRef], resume: Optional[LovableResumeCon
if resume is None:
return 0, None
- checkpoint = _ParentRef(workspace_id=resume.workspace_id, project_id=resume.project_id)
+ checkpoint = _ParentRef(workspace_id=resume.workspace_id, project_id=resume.project_id, scan_id=resume.scan_id)
if checkpoint not in parents:
return 0, None
@@ -144,6 +199,8 @@ def _with_parent_ids(row: dict[str, Any], parent: _ParentRef) -> dict[str, Any]:
row.setdefault("workspace_id", parent.workspace_id)
if parent.project_id:
row.setdefault("project_id", parent.project_id)
+ if parent.scan_id:
+ row.setdefault("scan_id", parent.scan_id)
return row
@@ -152,6 +209,7 @@ def _iter_endpoint(
api_version: str,
config: LovableEndpointConfig,
resumable_source_manager: ResumableSourceManager[LovableResumeConfig],
+ extra_params: dict[str, Any],
) -> Iterator[list[dict[str, Any]]]:
parents = list(_iter_parents(client, api_version, config))
resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
@@ -166,20 +224,23 @@ def _iter_endpoint(
LovableResumeConfig(
workspace_id=parent.workspace_id,
project_id=parent.project_id,
+ scan_id=parent.scan_id,
cursor=str(next_cursor) if next_cursor else None,
)
)
for page in client.paginate(
path=_path(config, parent),
- params=_params(config, parent),
- paginator=_paginator(),
- data_selector="data",
+ params={**_params(config, parent), **extra_params},
+ paginator=_endpoint_paginator(config),
+ data_selector=_data_selector(config),
+ data_selector_required=config.time_series,
resume_hook=save_checkpoint,
initial_paginator_state={"cursor": cursor} if index == start_index and cursor else None,
):
- if page:
- yield [_with_parent_ids(row, parent) for row in page]
+ rows = [row for series in page for row in _time_series_rows(series)] if config.time_series else page
+ if rows:
+ yield [_with_parent_ids(row, parent) for row in rows]
cursor = None
@@ -193,13 +254,20 @@ def lovable_source(
api_version: str,
endpoint: str,
resumable_source_manager: ResumableSourceManager[LovableResumeConfig],
+ should_use_incremental_field: bool = False,
+ db_incremental_field_last_value: Any = None,
) -> SourceResponse:
config = version_config(api_version).endpoints[endpoint]
client = _client(api_key, api_version)
+ extra_params = (
+ analytics_window(should_use_incremental_field, db_incremental_field_last_value, datetime.now(UTC))
+ if config.time_series
+ else {}
+ )
return SourceResponse(
name=endpoint,
- items=lambda: _iter_endpoint(client, api_version, config, resumable_source_manager),
+ items=lambda: _iter_endpoint(client, api_version, config, resumable_source_manager, extra_params),
primary_keys=config.primary_keys,
partition_count=1,
partition_size=1,
@@ -233,10 +301,12 @@ def validate_credentials(api_key: str, api_version: str) -> tuple[bool, str | No
return False, "Could not connect to Lovable. Check the API key and try again."
-def _probe_status(client: RESTClient, path: str, params: dict[str, Any]) -> int | None:
+def _probe_status(client: RESTClient, path: str, params: dict[str, Any], data_selector: str) -> int | None:
"""Status of a single-row GET, or `None` when no response came back at all."""
try:
- for _ in client.paginate(path=path, params=params, paginator=SinglePagePaginator(), data_selector="data"):
+ for _ in client.paginate(
+ path=path, params=params, paginator=SinglePagePaginator(), data_selector=data_selector
+ ):
break
except HTTPError as error:
return error.response.status_code if error.response is not None else None
@@ -268,19 +338,36 @@ def check_endpoint_permissions(api_key: str, api_version: str, endpoints: list[s
# A saved schema whose endpoint left the catalog must not fail the whole schema picker.
known = {name: catalog[name] for name in endpoints if name in catalog}
project_id: str | None = None
- if workspace_id and any(config.scope == "project" for config in known.values()):
+ if workspace_id and any(config.scope in ("project", "scan") for config in known.values()):
project_id = _first_id(client, catalog["Projects"], _ParentRef(workspace_id=workspace_id))
+ scan_id: str | None = None
+ if project_id and any(config.scope == "scan" for config in known.values()):
+ scan_id = _first_id(
+ client, catalog["ProjectSecurityScans"], _ParentRef(workspace_id=workspace_id, project_id=project_id)
+ )
permissions: dict[str, str | None] = dict.fromkeys(endpoints)
+ now = datetime.now(UTC)
for name, config in known.items():
- parent = _ParentRef(workspace_id=workspace_id, project_id=project_id)
- # No workspace or project to probe against means nothing to report, because an empty
- # account is not a permission problem.
- if (config.scope == "workspace" and not workspace_id) or (config.scope == "project" and not project_id):
+ parent = _ParentRef(workspace_id=workspace_id, project_id=project_id, scan_id=scan_id)
+ # No parent to probe against means nothing to report, because an empty account is not a
+ # permission problem.
+ if (
+ (config.scope == "workspace" and not workspace_id)
+ or (config.scope == "project" and not project_id)
+ or (config.scope == "scan" and not scan_id)
+ ):
permissions[name] = None
continue
- status = _probe_status(client, _path(config, parent), {**_params(config, parent), "limit": 1})
+ probe_params = (
+ {"starts_at": _rfc3339(now - timedelta(days=1)), "ends_at": _rfc3339(now)}
+ if config.time_series
+ else {"limit": 1}
+ )
+ status = _probe_status(
+ client, _path(config, parent), {**_params(config, parent), **probe_params}, _data_selector(config)
+ )
if status == 402:
plan = config.minimum_plan or "paid"
permissions[name] = f"This table needs Lovable's {plan} plan or higher."
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/lovable/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/lovable/settings.py
index f5317fedf705..ce62723691c6 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/lovable/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/lovable/settings.py
@@ -3,11 +3,12 @@ from typing import Any, Literal
from posthog.dataclasses import frozen
-from products.warehouse_sources.backend.types import IncrementalField
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import incremental_field
+from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType
-# Where an endpoint's rows live: directly on the account, under one workspace, or under one
-# project. Anything but "account" is fanned out over the parents discovered at sync time.
-EndpointScope = Literal["account", "workspace", "project"]
+# Where an endpoint's rows live: directly on the account, under one workspace, one project, or one
+# project security scan. Anything but "account" is fanned out over the parents discovered at sync time.
+EndpointScope = Literal["account", "workspace", "project", "scan"]
LOVABLE_API_BASE_URL = "https://api.lovable.dev"
@@ -19,6 +20,9 @@ LOVABLE_API_VERSION_2026_09_11 = "2026-09-11"
LOVABLE_URL_VERSION_SEGMENT = "v1"
+# Analytics endpoints reject a date range longer than 365 days.
+ANALYTICS_MAX_RANGE_DAYS = 365
+
@frozen
class LovableEndpointConfig:
@@ -35,6 +39,11 @@ class LovableEndpointConfig:
# Lowest Lovable plan that may call the endpoint. Below it the API answers 402, so this is the
# reason surfaced per table in the schema picker rather than a sync that can only ever fail.
minimum_plan: str | None = None
+ # The endpoint returns one unpaginated `time_series` object over a required date range instead
+ # of a cursor page of rows.
+ time_series: bool = False
+ # Lovable answers 404 for an unpublished project, so only published projects are fanned out.
+ published_projects_only: bool = False
LOVABLE_ENDPOINTS: dict[str, LovableEndpointConfig] = {
@@ -131,6 +140,43 @@ LOVABLE_ENDPOINTS_2026_09_11: dict[str, LovableEndpointConfig] = {
minimum_plan="Business",
),
"ProjectPiiLabels": LOVABLE_ENDPOINTS["ProjectPiiLabels"],
+ "WorkspaceGroups": LovableEndpointConfig(
+ name="WorkspaceGroups",
+ path="/workspaces/{workspace_id}/groups",
+ scope="workspace",
+ primary_keys=["workspace_id", "id"],
+ partition_key="created_at",
+ minimum_plan="Business",
+ ),
+ "ProjectSecurityScanFindings": LovableEndpointConfig(
+ name="ProjectSecurityScanFindings",
+ path="/projects/{project_id}/security-scans/{scan_id}/findings",
+ scope="scan",
+ # Each scan is a full snapshot that repeats findings carried over from earlier scans under
+ # the same finding id, so the scan id is part of the key.
+ primary_keys=["project_id", "scan_id", "id"],
+ partition_key="created_at",
+ minimum_plan="Business",
+ ),
+ "ProjectAnalytics": LovableEndpointConfig(
+ name="ProjectAnalytics",
+ path="/projects/{project_id}/analytics",
+ scope="project",
+ primary_keys=["project_id", "timestamp"],
+ partition_key="timestamp",
+ minimum_plan="Business",
+ time_series=True,
+ published_projects_only=True,
+ ),
+ "WorkspaceAnalytics": LovableEndpointConfig(
+ name="WorkspaceAnalytics",
+ path="/workspaces/{workspace_id}/analytics",
+ scope="workspace",
+ primary_keys=["workspace_id", "timestamp"],
+ partition_key="timestamp",
+ minimum_plan="Business",
+ time_series=True,
+ ),
}
@@ -164,6 +210,12 @@ def version_config(api_version: str) -> LovableVersionConfig:
raise ValueError(f"Unsupported Lovable API version: {api_version!r}") from e
-# Every list endpoint paginates by opaque cursor with no timestamp filter, so none of these
-# tables can sync incrementally. A run either walks the collection or it skips rows.
-INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {}
+# Every list endpoint paginates by opaque cursor with no timestamp filter, so none of those tables
+# can sync incrementally. Only the analytics endpoints take a server-side `starts_at`.
+INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {
+ "ProjectAnalytics": [incremental_field("timestamp", IncrementalFieldType.DateTime)],
+ "WorkspaceAnalytics": [incremental_field("timestamp", IncrementalFieldType.DateTime)],
+}
+
+# An incremental analytics run reads the watermark day again, so append would duplicate that bucket.
+MERGE_ONLY_ENDPOINTS = frozenset(INCREMENTAL_FIELDS)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/lovable/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/lovable/source.py
index 529928ec25b2..5b115e9e4924 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/lovable/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/lovable/source.py
@@ -32,6 +32,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.lovable.se
INCREMENTAL_FIELDS,
LOVABLE_API_VERSION_2026_09_11,
LOVABLE_API_VERSION_V1,
+ MERGE_ONLY_ENDPOINTS,
version_config,
)
from products.warehouse_sources.backend.types import ExternalDataSourceType
@@ -76,7 +77,7 @@ class LovableSource(ResumableSource[LovableSourceConfig, LovableResumeConfig]):
) -> list[SourceSchema]:
# The table set differs by version, so discovery must follow the source's pin.
endpoints = tuple(version_config(self.resolve_api_version(api_version)).endpoints)
- return build_endpoint_schemas(endpoints, INCREMENTAL_FIELDS, names)
+ return build_endpoint_schemas(endpoints, INCREMENTAL_FIELDS, names, merge_only=MERGE_ONLY_ENDPOINTS)
def validate_credentials(
self,
@@ -106,6 +107,8 @@ class LovableSource(ResumableSource[LovableSourceConfig, LovableResumeConfig]):
api_version=self.resolve_api_version(inputs.api_version),
endpoint=inputs.schema_name,
resumable_source_manager=resumable_source_manager,
+ should_use_incremental_field=inputs.should_use_incremental_field,
+ db_incremental_field_last_value=inputs.db_incremental_field_last_value,
)
@property
@@ -116,8 +119,9 @@ class LovableSource(ResumableSource[LovableSourceConfig, LovableResumeConfig]):
label="Lovable",
caption=(
"Connect your Lovable account with an API key to sync workspaces, projects, "
- "members, credit history, and security findings. Member, collaborator, credit, and "
- "PII tables need Lovable's Enterprise plan, and security scans need Business or higher."
+ "members, groups, credit history, security findings, and website analytics. Member, "
+ "collaborator, credit, and PII tables need Lovable's Enterprise plan. Groups, security "
+ "scans, and analytics need Business or higher."
),
keywords=["app builder"],
docsUrl="https://posthog.com/docs/cdp/sources/lovable",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/lovable/tests/test_lovable.py b/products/warehouse_sources/backend/temporal/data_imports/sources/lovable/tests/test_lovable.py
index 5859cf801cbc..5ff3e1bbc166 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/lovable/tests/test_lovable.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/lovable/tests/test_lovable.py
@@ -1,3 +1,4 @@
+from datetime import UTC, datetime
from typing import Any
from parameterized import parameterized
@@ -14,6 +15,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.generated_
from products.warehouse_sources.backend.temporal.data_imports.sources.lovable.lovable import (
LovableResumeConfig,
_ParentRef,
+ analytics_window,
check_endpoint_permissions,
resume_position,
validate_credentials,
@@ -102,6 +104,96 @@ class TestLovableSourceTransport:
]
assert result.paths[2:] == ["/v1/projects/p-1/collaborators", "/v1/projects/p-2/collaborators"]
+ @parameterized.expand(
+ [
+ ("fresh_run_walks_every_scan", None, ["/v1/projects/p-1/security-scans/s-1/findings"], None),
+ (
+ "resume_continues_the_checkpointed_scan",
+ LovableResumeConfig(workspace_id="ws-1", project_id="p-1", scan_id="s-2", cursor="c1"),
+ [],
+ "c1",
+ ),
+ ]
+ )
+ def test_findings_fan_out_through_scans_and_stamp_the_scan(
+ self,
+ _name: str,
+ resume_state: LovableResumeConfig | None,
+ earlier_scan_paths: list[str],
+ expected_cursor: str | None,
+ ) -> None:
+ findings = [_page([{"id": "f-1"}])] if resume_state is None else []
+ result = _driver().run(
+ "ProjectSecurityScanFindings",
+ [
+ _page([{"id": "ws-1"}]),
+ _page([{"id": "p-1"}]),
+ _page([{"id": "s-1"}, {"id": "s-2"}]),
+ *findings,
+ _page([{"id": "f-1"}]),
+ ],
+ api_version="2026-09-11",
+ resume_state=resume_state,
+ )
+
+ assert result.raised is None
+ assert result.paths[3:] == [*earlier_scan_paths, "/v1/projects/p-1/security-scans/s-2/findings"]
+ assert result.requests[-1].param("cursor") == expected_cursor
+ assert [(row["project_id"], row["scan_id"], row["id"]) for row in result.rows][-1] == ("p-1", "s-2", "f-1")
+
+ def test_project_analytics_skips_unpublished_projects(self) -> None:
+ result = _driver().run(
+ "ProjectAnalytics",
+ [
+ _page([{"id": "ws-1"}]),
+ _page([{"id": "p-draft", "is_published": False}, {"id": "p-live", "is_published": True}]),
+ ScriptedResponse(json={"time_series": {}}),
+ ],
+ api_version="2026-09-11",
+ )
+
+ assert result.raised is None
+ assert result.paths[2:] == ["/v1/projects/p-live/analytics"]
+
+ def test_analytics_pivots_each_bucket_into_one_row(self) -> None:
+ result = _driver().run(
+ "WorkspaceAnalytics",
+ [
+ _page([{"id": "ws-1"}]),
+ ScriptedResponse(
+ json={
+ "time_series": {
+ "visitors": {
+ "label": "visitors",
+ "total": 5,
+ "data": [
+ {"timestamp": "2026-10-02T00:00:00Z", "value": 3},
+ {"timestamp": "2026-10-01T00:00:00Z", "value": 2},
+ ],
+ },
+ "bounce_rate": {
+ "label": "bounce_rate",
+ "total": 40.0,
+ "data": [{"timestamp": "2026-10-01T00:00:00Z", "value": 40.0}],
+ },
+ }
+ }
+ ),
+ ],
+ api_version="2026-09-11",
+ )
+
+ assert result.raised is None
+ assert result.rows == [
+ {"timestamp": "2026-10-01T00:00:00Z", "visitors": 2, "bounce_rate": 40.0, "workspace_id": "ws-1"},
+ {"timestamp": "2026-10-02T00:00:00Z", "visitors": 3, "workspace_id": "ws-1"},
+ ]
+ analytics_request = result.requests[1]
+ assert analytics_request.path == "/v1/workspaces/ws-1/analytics"
+ assert analytics_request.param("limit") is None
+ assert analytics_request.param("starts_at") is not None
+ assert analytics_request.param("ends_at") is not None
+
def test_parents_without_an_id_are_skipped(self) -> None:
result = _driver().run(
"WorkspaceMembers",
@@ -127,6 +219,10 @@ class TestLovableSourceTransport:
("2026-09-11", "WorkspaceMembers", ["workspace_id", "user_id"], None),
("2026-09-11", "ProjectSecurityScans", ["project_id", "id"], "started_at"),
("2026-09-11", "ProjectPiiLabels", ["project_id", "id"], "found_at"),
+ ("2026-09-11", "WorkspaceGroups", ["workspace_id", "id"], "created_at"),
+ ("2026-09-11", "ProjectSecurityScanFindings", ["project_id", "scan_id", "id"], "created_at"),
+ ("2026-09-11", "ProjectAnalytics", ["project_id", "timestamp"], "timestamp"),
+ ("2026-09-11", "WorkspaceAnalytics", ["workspace_id", "timestamp"], "timestamp"),
]
)
def test_primary_keys_and_partitioning_per_endpoint(
@@ -143,6 +239,30 @@ class TestLovableSourceTransport:
assert response.partition_mode == ("datetime" if partition_key else None)
+class TestAnalyticsWindow:
+ @parameterized.expand(
+ [
+ ("full_refresh_reads_the_longest_range", False, "2026-10-20T08:00:00Z", "2025-10-24T00:00:00Z"),
+ ("incremental_rereads_the_watermark_day", True, "2026-10-20T00:00:00Z", "2026-10-20T00:00:00Z"),
+ (
+ "incremental_accepts_a_datetime_watermark",
+ True,
+ datetime(2026, 10, 20, 15, 30, tzinfo=UTC),
+ "2026-10-20T00:00:00Z",
+ ),
+ ("old_watermark_is_clamped_to_the_api_limit", True, "2024-01-01T00:00:00Z", "2025-10-24T00:00:00Z"),
+ ("no_watermark_reads_the_longest_range", True, None, "2025-10-24T00:00:00Z"),
+ ]
+ )
+ def test_window(self, _name: str, incremental: bool, last_value: Any, expected_start: str) -> None:
+ now = datetime(2026, 10, 23, 12, 0, tzinfo=UTC)
+
+ assert analytics_window(incremental, last_value, now) == {
+ "starts_at": expected_start,
+ "ends_at": "2026-10-23T12:00:00Z",
+ }
+
+
class TestLovableResume:
@parameterized.expand(
[
@@ -241,11 +361,26 @@ class TestCheckEndpointPermissions:
assert [request.url for request in network.requests_log] == ["https://api.lovable.dev/v1/workspaces?limit=1"]
- def test_project_scoped_table_probes_a_project(self) -> None:
+ @parameterized.expand(
+ [
+ ("project", "v1", "ProjectSecurityScans", [], "/v1/projects/p-1/security-scans"),
+ ("time_series", "2026-09-11", "ProjectAnalytics", [], "/v1/projects/p-1/analytics"),
+ (
+ "scan",
+ "2026-09-11",
+ "ProjectSecurityScanFindings",
+ [_page([{"id": "s-1"}])],
+ "/v1/projects/p-1/security-scans/s-1/findings",
+ ),
+ ]
+ )
+ def test_nested_table_probes_its_parent(
+ self, _name: str, api_version: str, endpoint: str, scan_pages: list[ScriptedResponse], expected_path: str
+ ) -> None:
with scripted_network(
- [_page([{"id": "ws-1"}]), _page([{"id": "p-1"}]), _error(402)],
+ [_page([{"id": "ws-1"}]), _page([{"id": "p-1"}]), *scan_pages, _error(402)],
) as network:
- permissions = check_endpoint_permissions("lov_key", "v1", ["ProjectSecurityScans"])
+ permissions = check_endpoint_permissions("lov_key", api_version, [endpoint])
- assert permissions == {"ProjectSecurityScans": "This table needs Lovable's Business plan or higher."}
- assert network.requests_log[2].path == "/v1/projects/p-1/security-scans"
+ assert permissions == {endpoint: "This table needs Lovable's Business plan or higher."}
+ assert network.requests_log[-1].path == expected_path
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/lovable/tests/test_lovable_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/lovable/tests/test_lovable_source.py
index 6a87eb91dab7..3b2398ef256f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/lovable/tests/test_lovable_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/lovable/tests/test_lovable_source.py
@@ -10,6 +10,18 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.lovable.so
SOURCE_MODULE = "products.warehouse_sources.backend.temporal.data_imports.sources.lovable.source"
+DATED_ENDPOINTS = [
+ "Workspaces",
+ "Projects",
+ "WorkspaceMembers",
+ "ProjectSecurityScans",
+ "ProjectPiiLabels",
+ "WorkspaceGroups",
+ "ProjectSecurityScanFindings",
+ "ProjectAnalytics",
+ "WorkspaceAnalytics",
+]
+
class TestLovableSource:
def setup_method(self) -> None:
@@ -31,27 +43,21 @@ class TestLovableSource:
"ProjectPiiLabels",
],
),
- (
- "dated",
- "2026-09-11",
- ["Workspaces", "Projects", "WorkspaceMembers", "ProjectSecurityScans", "ProjectPiiLabels"],
- ),
- (
- "unpinned_uses_the_default",
- None,
- ["Workspaces", "Projects", "WorkspaceMembers", "ProjectSecurityScans", "ProjectPiiLabels"],
- ),
+ ("dated", "2026-09-11", DATED_ENDPOINTS),
+ ("unpinned_uses_the_default", None, DATED_ENDPOINTS),
]
)
- def test_get_schemas_lists_the_pinned_versions_endpoints_as_full_refresh(
+ def test_get_schemas_lists_the_pinned_versions_endpoints(
self, _name: str, api_version: str | None, expected: list[str]
) -> None:
schemas = self.source.get_schemas(self.config, team_id=1, api_version=api_version)
assert [schema.name for schema in schemas] == expected
- # No v1 list endpoint takes a timestamp filter, so nothing here can sync incrementally.
- assert all(not schema.supports_incremental and not schema.supports_append for schema in schemas)
- assert all(schema.incremental_fields == [] for schema in schemas)
+ # Only the analytics endpoints take a timestamp filter, so nothing else can sync incrementally.
+ incremental = {schema.name for schema in schemas if schema.supports_incremental}
+ assert incremental == {"ProjectAnalytics", "WorkspaceAnalytics"} & set(expected)
+ # Each analytics run rereads the watermark day, which append would duplicate.
+ assert all(not schema.supports_append for schema in schemas)
@parameterized.expand(
[
@@ -85,8 +91,10 @@ class TestLovableSource:
self, _name: str, pinned: str | None, expected_version: str, mock_lovable_source: mock.MagicMock
) -> None:
inputs = mock.MagicMock()
- inputs.schema_name = "Projects"
+ inputs.schema_name = "ProjectAnalytics"
inputs.api_version = pinned
+ inputs.should_use_incremental_field = True
+ inputs.db_incremental_field_last_value = "2026-10-01T00:00:00Z"
manager = mock.MagicMock()
self.source.source_for_pipeline(self.config, manager, inputs)
@@ -95,8 +103,10 @@ class TestLovableSource:
assert kwargs == {
"api_key": "lov_key",
"api_version": expected_version,
- "endpoint": "Projects",
+ "endpoint": "ProjectAnalytics",
"resumable_source_manager": manager,
+ "should_use_incremental_field": True,
+ "db_incremental_field_last_value": "2026-10-01T00:00:00Z",
}
@mock.patch(f"{SOURCE_MODULE}.validate_lovable_credentials", return_value=(True, None))