Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add 7 missing tables to the pluralsight flow source (#126267)

Tom Owers275252327638
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 3daae95fa533..5ee3309d02a6 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -9628,11 +9628,11 @@ Diffed against: <https://appfire.atlassian.net/wiki/spaces/FD/pages/1802600473>
- [x] `Pull Request Projects (GET /v3/customer/core/pull_request_projects/)` — resolve project IDs on pull requests and link their projects to repositories (high)
- [x] `Ticket Projects (GET /v3/customer/core/ticket_projects/)` — identify the issue tracker projects that contain synced tickets (high)
- [x] `Team Membership (GET /v3/customer/core/team_membership)` — connect users to teams for engineering activity analysis by team (high)
-- [ ] `DORA Metrics (GET /dora/build-release/change/lead_time/; GET /dora/build-release/deployment/frequency/; GET /dora/build-release/incident/change_failure_rate/; GET /dora/build-release/incident/time_to_restore/)` — measure deployment frequency, change failure rate, recovery time, and change lead time (high)
-- [ ] `Ticket Metrics (GET /ticket-delivery/metrics/)` — track Flow velocity, cycle time, queue time, backflow rate, and jitter (high)
+- [x] `DORA Metrics (GET /dora/build-release/change/lead_time/; GET /dora/build-release/deployment/frequency/; GET /dora/build-release/incident/change_failure_rate/; GET /dora/build-release/incident/time_to_restore/)` — measure deployment frequency, change failure rate, recovery time, and change lead time (high)
+- [x] `Ticket Metrics (GET /ticket-delivery/metrics/)` — track Flow velocity, cycle time, queue time, backflow rate, and jitter (high)
- [ ] `Pull Request Events (GET /v3/customer/core/pull_request_events/)` — analyze review approvals and pull request state changes over time (medium)
-- [ ] `Ticket Events (GET /v3/customer/core/ticket_events/)` — measure time in ticket states and identify workflow bottlenecks (medium)
-- [ ] `Pull Request Comments (GET /v3/customer/core/pull_request_comments/)` — analyze reviewer participation, discussion depth, and response times (medium)
+- [x] `Ticket Events (GET /v3/customer/core/ticket_events/)` — measure time in ticket states and identify workflow bottlenecks (medium)
+- [x] `Pull Request Comments (GET /v3/customer/core/pull_request_comments/)` — analyze reviewer participation, discussion depth, and response times (medium)
- [ ] `Ticket Comments (GET /v3/customer/core/ticket_comments)` — analyze collaboration and discussion volume on issue tracker tickets (medium)
Note: Parsed Customer API reference pages from the official page tree and their current Appfire markdown versions. Parsed live Swagger specifications at /dora/docs?format=openapi and /ticket-delivery/docs?format=openapi for the metric operations. Metric tables need date windows and Metrics API permission; core resources use the workspace host. Excluded integration configuration, targets, write operations, and duplicate metric versions.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/pluralsight_flow/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/pluralsight_flow/canonical_descriptions.py
index 063de8cf7bb6..4aa2da363e63 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/pluralsight_flow/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/pluralsight_flow/canonical_descriptions.py
@@ -196,4 +196,101 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"date_range": "The `[start:end]` date window the metrics were computed over.",
},
},
+ "PullRequestComments": {
+ "description": "A comment a user made on a pull request.",
+ "docs_url": "https://appfire.atlassian.net/wiki/spaces/FD/pages/1802502374",
+ "columns": {
+ "id": "Unique identifier for the pull request comment.",
+ "user_alias_id": "Identifier of the user alias that wrote the comment.",
+ "apex_user_id": "Identifier of the main user record for the comment author's aliases.",
+ "pull_request_id": "Identifier of the pull request the comment belongs to.",
+ "body": "Pull request comment body.",
+ "word_count": "Total number of words in the comment.",
+ "comment_robustness": "How robust the comment is: robust, regular, or trivial.",
+ "is_extension_comment": "Whether the comment is a follow-on comment (a second consecutive comment by the same author).",
+ "parent_id": "Identifier of the parent pull request comment.",
+ "reaction_time": "Time in seconds it took to respond to the comment.",
+ "created_at": "Date and time the comment was created.",
+ "updated_at": "Date and time the comment was updated.",
+ "is_reply": "Whether the comment was made in reply to another comment.",
+ "was_influential": "Whether the comment resulted in the submitter adding a follow-on commit.",
+ "comment_type": "Type of comment.",
+ },
+ },
+ "TicketEvents": {
+ "description": "A state change in a ticket's lifecycle in the issue-tracking system.",
+ "docs_url": "https://appfire.atlassian.net/wiki/spaces/FD/pages/1802568132",
+ "columns": {
+ "id": "Unique identifier for the ticket event.",
+ "user_alias_id": "Identifier of the user alias associated with the ticket event.",
+ "created_at": "Date and time the ticket event was created.",
+ "params": "Parameters for the ticket event.",
+ "ticket_id": "Identifier of the ticket.",
+ "to_state": "State the ticket moved to: N/A, Defined, In-Progress, Completed, or Accepted.",
+ "vendor": "Name of the issue-tracking system.",
+ },
+ },
+ "TicketMetrics": {
+ "description": (
+ "Aggregate ticket delivery metrics (tickets completed, velocity, cycle time, queue time, "
+ "backflow rate, jitter) for tickets moved to Done in the requested date range. One row per sync."
+ ),
+ "docs_url": "https://appfire.atlassian.net/wiki/spaces/FD/pages/1802141797",
+ "columns": {
+ "date_range": "The date window the metrics were computed over.",
+ "tickets_completed": "Number of tickets completed in the date range.",
+ "total_jitter": "Total jitter in the ticket completion times.",
+ "backflow_rate": "Rate of tickets that were reopened after completion.",
+ "median_cycle_time": "Median time taken to complete a ticket.",
+ "tickets_queue_time": "Total time tickets spent in the queue.",
+ "velocity": "Rate at which tickets are completed.",
+ "active_time": "Total time tickets spent actively being worked on.",
+ },
+ },
+ "DoraLeadTime": {
+ "description": "DORA lead time for changes (median, trend, performance) for the requested date range. One row per sync.",
+ "docs_url": "https://appfire.atlassian.net/wiki/spaces/FD/pages/1802338351",
+ "columns": {
+ "date_range": "The date window the metric was computed over.",
+ "median": "Median lead time for changes.",
+ "trend": "Change in the metric compared to the previous period.",
+ "performance": "DORA performance level for the metric.",
+ "timeseries": "The metric broken down by time period.",
+ },
+ },
+ "DoraDeploymentFrequency": {
+ "description": "DORA deployment frequency (total, weekly median, trend, performance) for the requested date range. One row per sync.",
+ "docs_url": "https://appfire.atlassian.net/wiki/spaces/FD/pages/1802338351",
+ "columns": {
+ "date_range": "The date window the metric was computed over.",
+ "weekly_median": "Median number of deployments per week.",
+ "total": "Total number of deployments.",
+ "trend": "Change in the metric compared to the previous period.",
+ "frequency": "Deployment frequency bucket, e.g. daily.",
+ "performance": "DORA performance level for the metric.",
+ "timeseries": "Deployment counts broken down by time period.",
+ },
+ },
+ "DoraChangeFailureRate": {
+ "description": "DORA change failure rate with its related trends, deployments and incidents for the requested date range. One row per sync.",
+ "docs_url": "https://appfire.atlassian.net/wiki/spaces/FD/pages/1802338351",
+ "columns": {
+ "date_range": "The date window the metric was computed over.",
+ "overall_failure_rate": "Change failure rate over the whole date range.",
+ "timeseries": "Failure rate, deployments and failures broken down by time period.",
+ "incidents": "Incidents counted as failures in the date range.",
+ },
+ },
+ "DoraTimeToRestore": {
+ "description": "DORA time to restore service with its related trends and incidents for the requested date range. One row per sync.",
+ "docs_url": "https://appfire.atlassian.net/wiki/spaces/FD/pages/1802338351",
+ "columns": {
+ "date_range": "The date window the metric was computed over.",
+ "time_to_restore": "Time to restore service in the date range.",
+ "trend": "Change in the metric compared to the previous period.",
+ "performance": "DORA performance level for the metric.",
+ "timeseries": "Incident counts and recovery time broken down by time period.",
+ "incidents": "Incidents resolved in the date range, with their time to restore.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/pluralsight_flow/pluralsight_flow.py b/products/warehouse_sources/backend/temporal/data_imports/sources/pluralsight_flow/pluralsight_flow.py
index b6969e2ce345..c7bb70928ec2 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/pluralsight_flow/pluralsight_flow.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/pluralsight_flow/pluralsight_flow.py
@@ -1,4 +1,5 @@
import re
+import json
from collections.abc import Callable
from datetime import UTC, date, datetime, timedelta
from typing import Any, Optional
@@ -29,14 +30,16 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.pluralsigh
DEFAULT_METRICS_LOOKBACK_DAYS,
INCREMENTAL_FIELDS,
MAX_PAGE_SIZE,
+ METRIC_DATE_RANGE_STYLES,
METRIC_ENDPOINT_PATHS,
METRIC_ENDPOINTS,
PARTITION_KEYS,
PRIMARY_KEYS,
TABLE_NAMES,
+ MetricDateRangeStyle,
)
-# Shared host for the Metrics API (Coding metrics, Collaboration/PR metrics) — unlike the Customer
+# Shared host for the Metrics API (Coding, Collaboration, Ticket and DORA metrics) — unlike the Customer
# API, it is not workspace-specific. Flow moved this off `flow-api.pluralsight.com` to
# `api.appfireflow.com` on 2025-07-23; the old host is no longer guaranteed to work.
METRICS_BASE_URL = "https://api.appfireflow.com"
@@ -119,10 +122,16 @@ def _core_resource(name: str, should_use_incremental_field: bool) -> EndpointRes
}
-def _default_metrics_date_range(today: Optional[date] = None) -> str:
+def _default_metrics_date_range(style: MetricDateRangeStyle = "bracket", today: Optional[date] = None) -> str:
end = today or datetime.now(UTC).date()
start = end - timedelta(days=DEFAULT_METRICS_LOOKBACK_DAYS)
- return f"[{start.isoformat()}:{end.isoformat()}]"
+ if style == "bracket":
+ return f"[{start.isoformat()}:{end.isoformat()}]"
+ start_at = f"{start.isoformat()}T00:00:00+00:00"
+ end_at = f"{end.isoformat()}T00:00:00+00:00"
+ if style == "json_list":
+ return json.dumps([start_at, end_at], separators=(",", ":"))
+ return f"{start_at},{end_at}"
def _stamp_date_range(date_range: str) -> Callable[[dict[str, Any]], dict[str, Any]]:
@@ -162,7 +171,7 @@ def pluralsight_flow_source(
metrics_config: RESTAPIConfig = {
"client": {"base_url": METRICS_BASE_URL, "auth": {"type": "bearer", "token": api_key}},
"resource_defaults": {},
- "resources": [_metrics_resource(endpoint, _default_metrics_date_range())],
+ "resources": [_metrics_resource(endpoint, _default_metrics_date_range(METRIC_DATE_RANGE_STYLES[endpoint]))],
}
resource = rest_api_resource(metrics_config, team_id, job_id, None)
return SourceResponse(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/pluralsight_flow/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/pluralsight_flow/settings.py
index 12de27e26d1c..e71b8e3b6070 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/pluralsight_flow/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/pluralsight_flow/settings.py
@@ -1,3 +1,5 @@
+from typing import Literal
+
from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import incremental_field
from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType
@@ -13,14 +15,21 @@ CORE_ENDPOINTS = (
"TeamMemberships",
"PullRequestProjects",
"TicketProjects",
+ "PullRequestComments",
+ "TicketEvents",
)
-# Aggregate report resources under the shared Metrics API (`https://api.appfireflow.com/collaboration/`).
+# Aggregate report resources under the shared Metrics API host (`https://api.appfireflow.com/`).
# Unlike the core endpoints, these return one summary object for a requested date range rather than
# a paginated list of records.
METRIC_ENDPOINTS = (
"CodingMetrics",
"CollaborationMetrics",
+ "TicketMetrics",
+ "DoraLeadTime",
+ "DoraDeploymentFrequency",
+ "DoraChangeFailureRate",
+ "DoraTimeToRestore",
)
ENDPOINTS = CORE_ENDPOINTS + METRIC_ENDPOINTS
@@ -38,12 +47,34 @@ CORE_ENDPOINT_PATHS: dict[str, str] = {
"TeamMemberships": "team_membership/",
"PullRequestProjects": "pull_request_projects/",
"TicketProjects": "ticket_projects/",
+ "PullRequestComments": "pull_request_comments/",
+ "TicketEvents": "ticket_events/",
}
# Path of each metrics endpoint under the shared collaboration host.
METRIC_ENDPOINT_PATHS: dict[str, str] = {
"CodingMetrics": "/collaboration/code/metrics/",
"CollaborationMetrics": "/collaboration/pullrequest/metrics/",
+ "TicketMetrics": "/ticket-delivery/metrics/",
+ "DoraLeadTime": "/dora/build-release/change/lead_time/",
+ "DoraDeploymentFrequency": "/dora/build-release/deployment/frequency/",
+ "DoraChangeFailureRate": "/dora/build-release/incident/change_failure_rate/",
+ "DoraTimeToRestore": "/dora/build-release/incident/time_to_restore/",
+}
+
+# Each metrics API family expects its `date_range` window in a different shape:
+# collaboration `[YYYY-MM-DD:YYYY-MM-DD]`, ticket delivery a JSON list of two offset datetimes,
+# and DORA two comma-separated offset datetimes.
+MetricDateRangeStyle = Literal["bracket", "json_list", "comma"]
+
+METRIC_DATE_RANGE_STYLES: dict[str, MetricDateRangeStyle] = {
+ "CodingMetrics": "bracket",
+ "CollaborationMetrics": "bracket",
+ "TicketMetrics": "json_list",
+ "DoraLeadTime": "comma",
+ "DoraDeploymentFrequency": "comma",
+ "DoraChangeFailureRate": "comma",
+ "DoraTimeToRestore": "comma",
}
# Every Customer API object exposes `id` as its primary key (confirmed for Users, Commits,
@@ -67,8 +98,15 @@ TABLE_NAMES: dict[str, str] = {
"TeamMemberships": "team_memberships",
"PullRequestProjects": "pull_request_projects",
"TicketProjects": "ticket_projects",
+ "PullRequestComments": "pull_request_comments",
+ "TicketEvents": "ticket_events",
"CodingMetrics": "coding_metrics",
"CollaborationMetrics": "collaboration_metrics",
+ "TicketMetrics": "ticket_metrics",
+ "DoraLeadTime": "dora_lead_time",
+ "DoraDeploymentFrequency": "dora_deployment_frequency",
+ "DoraChangeFailureRate": "dora_change_failure_rate",
+ "DoraTimeToRestore": "dora_time_to_restore",
}
# Stable datetime field to partition on, for endpoints where one is confirmed by the API docs.
@@ -83,11 +121,13 @@ PARTITION_KEYS: dict[str, str] = {
"UserAliases": "created_at",
"PullRequestProjects": "created_at",
"TicketProjects": "created_at",
+ "PullRequestComments": "created_at",
+ "TicketEvents": "created_at",
}
# Only endpoints with a documented `<field>__gte`/`<field>__lt`-filterable timestamp get an entry.
-# Repos, TeamMemberships, PullRequestProjects, TicketProjects and the metrics endpoints are full refresh:
-# the first four document no date filter, and the metrics endpoints already require an explicit
+# Repos, TeamMemberships, PullRequestProjects, TicketProjects, PullRequestComments, TicketEvents and the
+# metrics endpoints are full refresh: the core ones document no date filter, and the metrics endpoints already require an explicit
# date-range window on every call.
INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {
"Users": [incremental_field("last_activity_at", IncrementalFieldType.DateTime)],
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/pluralsight_flow/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/pluralsight_flow/source.py
index 5b93d33a7b73..4c54b3fa6707 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/pluralsight_flow/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/pluralsight_flow/source.py
@@ -55,8 +55,8 @@ class PluralsightFlowSource(ResumableSource[PluralsightFlowSourceConfig, Plurals
return {
"401 Client Error": "Your Flow API key is invalid or expired. Generate a new key and reconnect.",
"403 Client Error": (
- "Flow rejected this request. Coding metrics and Collaboration metrics need the "
- "Metrics API permission on your API key."
+ "Flow rejected this request. The Coding, Collaboration, Ticket and DORA metrics "
+ "tables need the Metrics API permission on your API key."
),
}
@@ -125,9 +125,9 @@ class PluralsightFlowSource(ResumableSource[PluralsightFlowSourceConfig, Plurals
category=DataWarehouseSourceCategory.ENGINEERING___MONITORING,
label="Pluralsight Flow",
caption=(
- "Import commits, pull requests, tickets, and coding/collaboration metrics from "
- "Pluralsight Flow (formerly GitPrime). Generate an API key in Flow under "
- "Settings > API keys. Coding metrics and Collaboration metrics need the "
+ "Import commits, pull requests, tickets, and coding, collaboration, ticket and DORA "
+ "metrics from Pluralsight Flow (formerly GitPrime). Generate an API key in Flow under "
+ "Settings > API keys. The Coding, Collaboration, Ticket and DORA metrics tables need the "
"**Metrics API** permission on that key; the other tables only need the key itself."
),
docsUrl="https://posthog.com/docs/cdp/sources/pluralsight-flow",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/pluralsight_flow/tests/test_pluralsight_flow.py b/products/warehouse_sources/backend/temporal/data_imports/sources/pluralsight_flow/tests/test_pluralsight_flow.py
index 6ce87b2c0d5e..d8cd0ce04f71 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/pluralsight_flow/tests/test_pluralsight_flow.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/pluralsight_flow/tests/test_pluralsight_flow.py
@@ -1,3 +1,4 @@
+import re
from datetime import date
import pytest
@@ -101,6 +102,8 @@ class TestPluralsightFlowSourceCoreEndpoints:
("TeamMemberships", "/v3/customer/core/team_membership/"),
("PullRequestProjects", "/v3/customer/core/pull_request_projects/"),
("TicketProjects", "/v3/customer/core/ticket_projects/"),
+ ("PullRequestComments", "/v3/customer/core/pull_request_comments/"),
+ ("TicketEvents", "/v3/customer/core/ticket_events/"),
],
)
def test_full_refresh_endpoint_sends_no_date_filter(self, endpoint: str, path: str) -> None:
@@ -117,15 +120,39 @@ class TestPluralsightFlowSourceCoreEndpoints:
class TestPluralsightFlowSourceMetricsEndpoints:
- def test_single_object_response_yields_one_row_with_date_range(self) -> None:
+ @pytest.mark.parametrize(
+ ("endpoint", "path", "date_range_pattern"),
+ [
+ (
+ "CollaborationMetrics",
+ "/collaboration/pullrequest/metrics/",
+ r"^\[\d{4}-\d{2}-\d{2}:\d{4}-\d{2}-\d{2}\]$",
+ ),
+ (
+ "TicketMetrics",
+ "/ticket-delivery/metrics/",
+ r'^\["\d{4}-\d{2}-\d{2}T00:00:00\+00:00","\d{4}-\d{2}-\d{2}T00:00:00\+00:00"\]$',
+ ),
+ (
+ "DoraChangeFailureRate",
+ "/dora/build-release/incident/change_failure_rate/",
+ r"^\d{4}-\d{2}-\d{2}T00:00:00\+00:00,\d{4}-\d{2}-\d{2}T00:00:00\+00:00$",
+ ),
+ ],
+ )
+ def test_single_object_response_yields_one_row_with_date_range(
+ self, endpoint: str, path: str, date_range_pattern: str
+ ) -> None:
result = SourceDriver(
PluralsightFlowSource(), PluralsightFlowSourceConfig(workspace="acme", api_key="test-key")
- ).run("CollaborationMetrics", [ScriptedResponse(json={"time_to_merge": {"average": 3.5, "trend": 0.1}})])
+ ).run(endpoint, [ScriptedResponse(json={"metric": {"average": 3.5, "trend": 0.1}})])
assert result.raised is None
+ assert result.paths == [path]
+ assert re.match(date_range_pattern, result.requests[0].param("date_range") or "")
assert len(result.rows) == 1
- assert result.rows[0]["time_to_merge"] == {"average": 3.5, "trend": 0.1}
- assert "date_range" in result.rows[0]
+ assert result.rows[0]["metric"] == {"average": 3.5, "trend": 0.1}
+ assert result.rows[0]["date_range"] == result.requests[0].param("date_range")
assert result.saved_states == []