Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add missing plausible report tables (#126276)

Tom Owers0cacbb7ddd08
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 5ee3309d02a6..6f8a7e8d1540 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -9578,11 +9578,11 @@ Today (18): `browsers`, `cities`, `countries`, `devices`, `entry_pages`, `exit_p
Diffed against: <https://plausible.io/docs/stats-api>
-- [ ] `visit:channel` — Plausible's acquisition-channel dimension — the default breakdown in its own dashboard and not derivable from source or utm_medium (high)
-- [ ] `event:props:{key}` — custom event property breakdowns; the only way to analyze custom-event dimensions such as plan or signup method (high)
-- [ ] `event:hostname` — splits traffic across subdomains and multi-domain sites, which every other table silently merges (medium)
+- [x] `visit:channel` — Plausible's acquisition-channel dimension — the default breakdown in its own dashboard and not derivable from source or utm_medium (high)
+- [x] `event:props:{key}` — custom event property breakdowns; the only way to analyze custom-event dimensions such as plan or signup method (high)
+- [x] `event:hostname` — splits traffic across subdomains and multi-domain sites, which every other table silently merges (medium)
- [ ] `visit:browser_version` — browser version breakdown for compatibility analysis (low)
-- [ ] `visit:os_version` — OS version breakdown for compatibility analysis (low)
+- [x] `visit:os_version` — OS version breakdown for compatibility analysis (low)
- [ ] `visit:entry_page_hostname / visit:exit_page_hostname` — hostname-qualified entry and exit pages for multi-domain sites (low)
Note: Plausible's Stats API v2 is a single POST /api/v2/query endpoint; a PostHog 'table' is one breakdown dimension. Coverage is close to complete — 18 of the 24 documented dimensions are already exposed. Custom properties (event:props:\*) are open-ended per site, so implementing them would need dynamic table discovery rather than a fixed table.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/field_classification.csv b/products/warehouse_sources/backend/temporal/data_imports/sources/field_classification.csv
index 6c5a326bc6ba..4e1d3d7e5d58 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/field_classification.csv
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/field_classification.csv
@@ -966,7 +966,7 @@ PlanetScalePostgres,connection_string host port database user password verify_se
Planhat,api_key,,,,,planhat
PlanningCenter,,,,,,planning-center
PlatformSh,api_token platform,,,,,platform-sh
-Plausible,api_key site_id host,,,,,plausible
+Plausible,api_key site_id host,,,custom_property_keys,,plausible
Plivo,auth_id auth_token,,,,,plivo
Plunk,api_key base_url,,,,,plunk
PluralsightFlow,workspace api_key,,,,,pluralsight-flow
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/plausible.py b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/plausible.py
index 8088bf8ee7a0..2e2de478c025 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/plausible.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/plausible.py
@@ -9,3 +9,4 @@ class PlausibleSourceConfig(config.Config):
api_key: str
site_id: str
host: str | None = None
+ custom_property_keys: str | None = None
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/plausible/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/plausible/canonical_descriptions.py
index 09f6d1d20c27..ee788fb95a3b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/plausible/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/plausible/canonical_descriptions.py
@@ -72,6 +72,19 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"docs_url": _DOCS_URL,
"columns": {**_STANDARD_METRIC_COLUMNS, "utm_content": "Value of the utm_content query parameter."},
},
+ "channels": {
+ "description": "Daily traffic metrics broken down by acquisition channel.",
+ "docs_url": _DOCS_URL,
+ "columns": {
+ **_STANDARD_METRIC_COLUMNS,
+ "channel": "The acquisition channel Plausible assigned to the visit, for example Organic Search or Paid Social.",
+ },
+ },
+ "hostnames": {
+ "description": "Daily traffic metrics broken down by the hostname events were sent from.",
+ "docs_url": _DOCS_URL,
+ "columns": {**_EVENT_METRIC_COLUMNS, "hostname": "The hostname the event was sent from."},
+ },
"pages": {
"description": "Daily traffic metrics broken down by page path and hostname.",
"docs_url": _DOCS_URL,
@@ -124,6 +137,15 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"docs_url": _DOCS_URL,
"columns": {**_STANDARD_METRIC_COLUMNS, "os": "The visitor's operating system."},
},
+ "operating_system_versions": {
+ "description": "Daily traffic metrics broken down by operating system and version.",
+ "docs_url": _DOCS_URL,
+ "columns": {
+ **_STANDARD_METRIC_COLUMNS,
+ "os": "The visitor's operating system.",
+ "os_version": "The version of the visitor's operating system.",
+ },
+ },
"devices": {
"description": "Daily traffic metrics broken down by device type (screen size class).",
"docs_url": _DOCS_URL,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/plausible/plausible.py b/products/warehouse_sources/backend/temporal/data_imports/sources/plausible/plausible.py
index 0079bed35f18..65396ee7c08f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/plausible/plausible.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/plausible/plausible.py
@@ -16,9 +16,9 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.syn
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.plausible.settings import (
DEFAULT_BACKFILL_DAYS,
- PLAUSIBLE_ENDPOINTS,
REPORT_LOOKBACK_DAYS,
PlausibleEndpointConfig,
+ get_endpoint_config,
)
# Plausible Cloud; self-hosted instances override this via the source's host field.
@@ -207,7 +207,7 @@ def plausible_source(
should_use_incremental_field: bool = False,
db_incremental_field_last_value: Optional[Any] = None,
) -> SourceResponse:
- config = PLAUSIBLE_ENDPOINTS[endpoint]
+ config = get_endpoint_config(endpoint)
resume_config = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
window = _resolve_window(resume_config, should_use_incremental_field, db_incremental_field_last_value)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/plausible/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/plausible/settings.py
index 91a95d771958..299e0e0dc346 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/plausible/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/plausible/settings.py
@@ -38,6 +38,12 @@ SESSION_SCOPED_METRICS = frozenset({"bounce_rate", "visit_duration", "views_per_
# on it and the pipeline can slide the date window incrementally.
TIME_DIMENSION = "time:day"
+# Custom properties are open-ended per site and the Stats API can't list a site's keys, so the user
+# names them and each key syncs as its own table with this schema-name prefix.
+CUSTOM_PROPERTY_DIMENSION_PREFIX = "event:props:"
+CUSTOM_PROPERTY_ENDPOINT_PREFIX = "custom_property_"
+CUSTOM_PROPERTY_VALUE_COLUMN = "property_value"
+
_DATE_INCREMENTAL_FIELDS: list[IncrementalField] = [
{
"label": "date",
@@ -51,11 +57,14 @@ _DATE_INCREMENTAL_FIELDS: list[IncrementalField] = [
def dimension_to_column(dimension: str) -> str:
"""Map a Plausible query dimension to the flat column name we store it under.
- `time:day` becomes `date`; every other dimension uses the part after the colon
- (e.g. `visit:source` -> `source`, `event:page` -> `page`, `visit:os` -> `os`).
+ `time:day` becomes `date`; a custom property becomes `property_value`, because the key itself is
+ user input that could collide with a metric column; every other dimension uses the part after
+ the colon (e.g. `visit:source` -> `source`, `event:page` -> `page`, `visit:os` -> `os`).
"""
if dimension == TIME_DIMENSION:
return "date"
+ if dimension.startswith(CUSTOM_PROPERTY_DIMENSION_PREFIX):
+ return CUSTOM_PROPERTY_VALUE_COLUMN
return dimension.split(":", 1)[-1]
@@ -117,6 +126,9 @@ PLAUSIBLE_ENDPOINTS: dict[str, PlausibleEndpointConfig] = {
"utm_campaigns": PlausibleEndpointConfig(name="utm_campaigns", breakdown_dimensions=["visit:utm_campaign"]),
"utm_terms": PlausibleEndpointConfig(name="utm_terms", breakdown_dimensions=["visit:utm_term"]),
"utm_contents": PlausibleEndpointConfig(name="utm_contents", breakdown_dimensions=["visit:utm_content"]),
+ "channels": PlausibleEndpointConfig(name="channels", breakdown_dimensions=["visit:channel"]),
+ # Event-scoped on its own, so `__post_init__` drops the session metrics.
+ "hostnames": PlausibleEndpointConfig(name="hostnames", breakdown_dimensions=["event:hostname"]),
# Hostname pairs with the path so rows from different subdomains stay distinct in the composite
# primary key. It makes these breakdowns event-scoped, so `__post_init__` drops the session
# metrics from `pages` and keeps the session metrics on the two visit-scoped page breakdowns.
@@ -132,6 +144,12 @@ PLAUSIBLE_ENDPOINTS: dict[str, PlausibleEndpointConfig] = {
"cities": PlausibleEndpointConfig(name="cities", breakdown_dimensions=["visit:city"]),
"browsers": PlausibleEndpointConfig(name="browsers", breakdown_dimensions=["visit:browser"]),
"operating_systems": PlausibleEndpointConfig(name="operating_systems", breakdown_dimensions=["visit:os"]),
+ # A version number only means something next to its OS name, so both dimensions are kept.
+ "operating_system_versions": PlausibleEndpointConfig(
+ name="operating_system_versions",
+ breakdown_dimensions=["visit:os", "visit:os_version"],
+ should_sync_default=False,
+ ),
"devices": PlausibleEndpointConfig(name="devices", breakdown_dimensions=["visit:device"]),
# Goal conversions: `event:goal` is only valid with goal-compatible metrics, so it carries its
# own reduced metric set rather than the generic breakdown set.
@@ -144,6 +162,26 @@ PLAUSIBLE_ENDPOINTS: dict[str, PlausibleEndpointConfig] = {
ENDPOINTS = tuple(PLAUSIBLE_ENDPOINTS.keys())
-INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {
- name: config.incremental_fields for name, config in PLAUSIBLE_ENDPOINTS.items()
-}
+
+def parse_custom_property_keys(raw: str | None) -> list[str]:
+ keys: list[str] = []
+ for part in (raw or "").split(","):
+ key = part.strip()
+ if key and key not in keys:
+ keys.append(key)
+ return keys
+
+
+def custom_property_endpoint_name(key: str) -> str:
+ return f"{CUSTOM_PROPERTY_ENDPOINT_PREFIX}{key}"
+
+
+def get_endpoint_config(name: str) -> PlausibleEndpointConfig:
+ """Resolve a schema name to its report config, including the per-key custom property tables."""
+ if name in PLAUSIBLE_ENDPOINTS:
+ return PLAUSIBLE_ENDPOINTS[name]
+ if name.startswith(CUSTOM_PROPERTY_ENDPOINT_PREFIX):
+ key = name.removeprefix(CUSTOM_PROPERTY_ENDPOINT_PREFIX)
+ if key:
+ return PlausibleEndpointConfig(name=name, breakdown_dimensions=[f"{CUSTOM_PROPERTY_DIMENSION_PREFIX}{key}"])
+ raise KeyError(f"Unknown Plausible endpoint: {name}")
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/plausible/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/plausible/source.py
index 39239070d121..0a89bc986606 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/plausible/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/plausible/source.py
@@ -30,8 +30,9 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.plausible.
)
from products.warehouse_sources.backend.temporal.data_imports.sources.plausible.settings import (
ENDPOINTS,
- INCREMENTAL_FIELDS,
- PLAUSIBLE_ENDPOINTS,
+ custom_property_endpoint_name,
+ get_endpoint_config,
+ parse_custom_property_keys,
)
from products.warehouse_sources.backend.types import ExternalDataSourceType
@@ -133,6 +134,16 @@ Works with Plausible Cloud and self-hosted instances. Create an API key under **
placeholder="https://plausible.io",
secret=False,
),
+ SourceFieldInputConfig(
+ name="custom_property_keys",
+ sync_setting=True,
+ label="Custom property keys",
+ caption="Optional. A comma-separated list of the custom properties to sync, for example `plan, signup_method`. Each property syncs as its own table. Your Plausible plan needs to include custom properties.",
+ type=SourceFieldInputConfigType.TEXT,
+ required=False,
+ placeholder="plan, signup_method",
+ secret=False,
+ ),
],
),
)
@@ -154,18 +165,21 @@ Works with Plausible Cloud and self-hosted instances. Create an API key under **
api_version: str | None = None,
) -> list[SourceSchema]:
def _build_schema(endpoint: str) -> SourceSchema:
- incremental_fields = INCREMENTAL_FIELDS.get(endpoint)
+ endpoint_config = get_endpoint_config(endpoint)
return SourceSchema(
name=endpoint,
# date_range is a genuine server-side filter, so every report supports incremental
# syncing by sliding the window forward.
- supports_incremental=incremental_fields is not None,
- supports_append=incremental_fields is not None,
- incremental_fields=incremental_fields or [],
- should_sync_default=PLAUSIBLE_ENDPOINTS[endpoint].should_sync_default,
+ supports_incremental=True,
+ supports_append=True,
+ incremental_fields=endpoint_config.incremental_fields,
+ should_sync_default=endpoint_config.should_sync_default,
)
- schemas = [_build_schema(endpoint) for endpoint in ENDPOINTS]
+ custom_property_endpoints = [
+ custom_property_endpoint_name(key) for key in parse_custom_property_keys(config.custom_property_keys)
+ ]
+ schemas = [_build_schema(endpoint) for endpoint in [*ENDPOINTS, *custom_property_endpoints]]
if names is not None:
names_set = set(names)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/plausible/tests/test_plausible.py b/products/warehouse_sources/backend/temporal/data_imports/sources/plausible/tests/test_plausible.py
index da00fb598633..86d6aac7aaf0 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/plausible/tests/test_plausible.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/plausible/tests/test_plausible.py
@@ -141,6 +141,19 @@ class TestGetRows:
assert body["pagination"]["offset"] == 4
assert body["date_range"] == ["2024-01-01", "2024-01-31"]
+ def test_custom_property_table_queries_its_key_with_event_scoped_metrics(self):
+ result = _run(
+ [_page([_result(["2024-01-01", "pro"], [5, 6, 7, 8])])],
+ endpoint="custom_property_plan",
+ )
+
+ body = result.requests[0].json()
+ assert body["dimensions"] == ["time:day", "event:props:plan"]
+ assert body["metrics"] == ["visitors", "visits", "pageviews", "events"]
+ assert result.items[0] == [
+ {"date": "2024-01-01", "property_value": "pro", "visitors": 5, "visits": 6, "pageviews": 7, "events": 8}
+ ]
+
def test_http_error_raises(self):
result = _run([ScriptedResponse(status=400, json={"error": "bad query"})])
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/plausible/tests/test_plausible_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/plausible/tests/test_plausible_source.py
index 9cd04a2db8be..a4932a4f57dc 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/plausible/tests/test_plausible_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/plausible/tests/test_plausible_source.py
@@ -3,6 +3,8 @@ from typing import Any
import pytest
from unittest import mock
+from parameterized import parameterized
+
from products.warehouse_sources.backend.temporal.data_imports.sources.plausible.settings import (
ENDPOINTS,
PLAUSIBLE_ENDPOINTS,
@@ -12,11 +14,12 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.plausible.
_MODULE = "products.warehouse_sources.backend.temporal.data_imports.sources.plausible.source"
-def _config(host: str | None = None) -> mock.MagicMock:
+def _config(host: str | None = None, custom_property_keys: str | None = None) -> mock.MagicMock:
config = mock.MagicMock()
config.api_key = "key"
config.site_id = "example.com"
config.host = host
+ config.custom_property_keys = custom_property_keys
return config
@@ -37,6 +40,20 @@ class TestSourceConfig:
assert PlausibleSource().connection_host_fields == ["host"]
+class TestGetSchemas:
+ @parameterized.expand(
+ [
+ ("unset", None, []),
+ ("blank", " ", []),
+ ("messy", " plan, ,signup_method,plan ", ["custom_property_plan", "custom_property_signup_method"]),
+ ]
+ )
+ def test_lists_one_table_per_custom_property_key(self, _name, raw_keys, expected):
+ schemas = PlausibleSource().get_schemas(_config(custom_property_keys=raw_keys), team_id=1)
+
+ assert [s.name for s in schemas] == [*ENDPOINTS, *expected]
+
+
class TestValidateCredentials:
@mock.patch(f"{_MODULE}.validate_plausible_credentials")
def test_invalid_credentials_surface_message(self, mock_validate):