Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add four more tables to the Plain source (#126265)

Tom Owers9447e038a696
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 6f8a7e8d1540..da288ec760e9 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -9519,12 +9519,13 @@ Diffed against: <https://core-api.uk.plain.com/graphql/v1/schema.graphql>
- [x] `users` — lookup resolving assignee and actor IDs carried on threads and timeline entries (high)
- [x] `labelTypes` — lookup resolving the label IDs attached to synced threads (high)
- [x] `tenants` — multi-tenant grouping dimension for customers and threads (high)
-- [ ] `customerSurveys` — CSAT responses, the headline support-quality metric (high)
+- [x] `customerSurveys` — CSAT survey definitions: template, targeting and send settings, not responses (high)
+- [ ] CSAT survey responses — individual responses and scores, the headline support-quality metric; the schema has no list query for them (high)
- [ ] `tiers` — SLA tier per company/tenant, needed to segment thread response times (medium)
- [ ] `tasks` — work items attached to threads, needed for workload analysis (medium)
-- [ ] `discussions` — internal discussion threads separate from customer-facing threads (medium)
-- [ ] `customerGroups` — lookup resolving group IDs on customers (medium)
-- [ ] `workflowExecutions` — automation run history, for measuring deflection and routing (medium)
+- [x] `discussions` — internal discussion threads separate from customer-facing threads (medium)
+- [x] `customerGroups` — lookup resolving group IDs on customers (medium)
+- [x] `workflowExecutions` — automation run history, for measuring deflection and routing (medium)
- [ ] `threadFieldSchemas` — lookup defining the custom thread fields whose values appear on threads (medium)
- [ ] `knowledgeGaps` — AI knowledge-gap records for content prioritization (low)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/plain/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/plain/canonical_descriptions.py
index e42062c257f8..4fcb4453448d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/plain/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/plain/canonical_descriptions.py
@@ -138,4 +138,75 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"updatedAt": "Time at which the label type was last updated.",
},
},
+ "customer_groups": {
+ "description": "A customer group in Plain, used to segment customers (for example by plan or region).",
+ "docs_url": "https://core-api.uk.plain.com/graphql/v1/schema.graphql",
+ "columns": {
+ "id": "Unique identifier for the customer group.",
+ "name": "The customer group's name.",
+ "key": "Unique, human-readable identifier for the group, used in API calls and integrations.",
+ "color": "The color shown for the group in the Plain UI.",
+ "externalId": "Your own identifier for the group, if set.",
+ "createdAt": "Time at which the customer group was created.",
+ "updatedAt": "Time at which the customer group was last updated.",
+ },
+ },
+ "customer_surveys": {
+ "description": "A customer survey configured in Plain, such as a CSAT survey sent after a thread is resolved. One row per survey definition, not per response.",
+ "docs_url": "https://core-api.uk.plain.com/graphql/v1/schema.graphql",
+ "columns": {
+ "id": "Unique identifier for the customer survey.",
+ "name": "The survey's name.",
+ "templateType": "The survey type from its template, for example CSAT.",
+ "templateQuestionText": "The question text shown to the customer when the survey is sent.",
+ "conditionsOperator": "How the targeting conditions combine: AND requires every condition, OR requires at least one.",
+ "isEnabled": "Whether the survey is active. Disabled surveys are never sent.",
+ "responseDelayMinutes": "Minutes after a thread is resolved before the survey is sent.",
+ "customerIntervalDays": "Minimum days before the same customer can be sent this survey again.",
+ "order": "Zero-based display and evaluation order of the survey relative to other surveys.",
+ "createdAt": "Time at which the survey was created.",
+ "updatedAt": "Time at which the survey was last updated.",
+ },
+ },
+ "discussions": {
+ "description": "An internal discussion in Plain: a Slack, email or Sidekick Agent conversation held by your team, separate from the customer-facing thread.",
+ "docs_url": "https://core-api.uk.plain.com/graphql/v1/schema.graphql",
+ "columns": {
+ "id": "Unique identifier for the discussion.",
+ "threadId": "ID of the support thread the discussion is attached to. Null for discussions not started from a thread.",
+ "title": "The discussion's title.",
+ "status": "Lifecycle of the discussion: OPEN until resolved, then RESOLVED.",
+ "agentStatus": "What the agent working on the discussion is doing. UNKNOWN when no agent is involved.",
+ "agentId": "ID of the machine user agent that was asked. Null means Plain's Sidekick Agent.",
+ "agentSessionId": "ID of the Sidekick Agent session backing the discussion, if any.",
+ "channelType": "Where the discussion happens: Slack, email, or an agent session.",
+ "visibility": "Who can see the discussion: WORKSPACE for all members, USER for the creator only.",
+ "sourceType": "What the discussion was started from: THREAD, PAGE, ENTITY or CHART.",
+ "sourceEntityId": "ID of the entity (for example a company or tenant) the discussion was started from, if any.",
+ "sourceEntityType": "Type of the entity in sourceEntityId, if set.",
+ "sourcePageLink": "Path of the page the discussion was started from, for PAGE and CHART sources.",
+ "resolvedAt": "Time at which the discussion was resolved, if applicable.",
+ "lastActivityAt": "Time of the most recent message in the discussion. Null when no messages exist yet.",
+ "createdAt": "Time at which the discussion was created.",
+ "updatedAt": "Time at which the discussion was last updated.",
+ },
+ },
+ "workflow_executions": {
+ "description": "One run of a Plain workflow automation, across every workflow in the workspace.",
+ "docs_url": "https://core-api.uk.plain.com/graphql/v1/schema.graphql",
+ "columns": {
+ "id": "Unique identifier for the workflow execution.",
+ "workflowId": "ID of the workflow that ran.",
+ "triggeredBy": "The event type that triggered the run, for example thread.thread_created.",
+ "executionStatus": "State of the run: QUEUED, RUNNING, WAITING, SUCCESS, FAILED or CANCELLED.",
+ "errorMessage": "Error message when the run failed.",
+ "entityType": "Type of the entity the run was triggered for: THREAD or WORKSPACE.",
+ "entityId": "ID of the entity the run was triggered for, such as a thread ID.",
+ "startedAt": "Time at which the run began processing. While queued, the time it was enqueued.",
+ "completedAt": "Time at which the run reached SUCCESS, FAILED or CANCELLED.",
+ "executionDurationMs": "How long the run took, in milliseconds, excluding time spent queued.",
+ "manuallyTriggeredByType": "Actor type that triggered the run manually, if it was triggered manually.",
+ "manuallyTriggeredById": "ID of the actor that triggered the run manually, if it was triggered manually.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/plain/plain.py b/products/warehouse_sources/backend/temporal/data_imports/sources/plain/plain.py
index 8ce266ecea82..88d0d9e704df 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/plain/plain.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/plain/plain.py
@@ -33,11 +33,16 @@ _MESSAGE_INFO_FIELDS: list[tuple[str, str]] = [
_ACTOR_FIELDS = ("createdBy", "updatedBy", "statusChangedBy", "markedAsSpamBy", "actor")
# Null on most rows, so always split them into Type/Id columns rather than keep a null struct column.
-_NULLABLE_ACTOR_FIELDS = ("deletedBy", "archivedBy")
-
-_REFERENCE_ID_FIELDS: list[tuple[str, str]] = [
- ("accountOwner", "accountOwnerId"),
- ("parentLabelType", "parentLabelTypeId"),
+_NULLABLE_ACTOR_FIELDS = ("deletedBy", "archivedBy", "manuallyTriggeredBy")
+
+# (nested object, key inside it, flat column) — a null object still yields a null column.
+_NESTED_VALUE_FIELDS: list[tuple[str, str, str]] = [
+ ("accountOwner", "id", "accountOwnerId"),
+ ("parentLabelType", "id", "parentLabelTypeId"),
+ ("agent", "id", "agentId"),
+ ("channelDetails", "channelType", "channelType"),
+ ("template", "type", "templateType"),
+ ("template", "questionText", "templateQuestionText"),
]
@@ -142,10 +147,13 @@ def _flatten_node(node: dict[str, Any]) -> dict[str, Any]:
actor.get("userId") or actor.get("machineUserId") or actor.get("customerId") or actor.get("systemId")
)
- for src_field, dst_field in _REFERENCE_ID_FIELDS:
- if src_field in flattened:
- reference = flattened.pop(src_field)
- flattened[dst_field] = reference.get("id") if isinstance(reference, dict) else None
+ nested_sources = {src_field for src_field, _, _ in _NESTED_VALUE_FIELDS if src_field in flattened}
+ for src_field, key, dst_field in _NESTED_VALUE_FIELDS:
+ if src_field in nested_sources:
+ nested = flattened[src_field]
+ flattened[dst_field] = nested.get(key) if isinstance(nested, dict) else None
+ for src_field in nested_sources:
+ del flattened[src_field]
for src_field, dst_field in _MESSAGE_INFO_FIELDS:
if src_field in flattened:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/plain/queries.py b/products/warehouse_sources/backend/temporal/data_imports/sources/plain/queries.py
index 7bed3e1194fd..e98155b11a58 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/plain/queries.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/plain/queries.py
@@ -424,6 +424,182 @@ query PaginatedTenants($first: Int!, $after: String, $filter: TenantsFilter) {
}
}"""
+CUSTOMER_GROUPS_QUERY = """
+query PaginatedCustomerGroups($first: Int!, $after: String) {
+ customer_groups: customerGroups(first: $first, after: $after) {
+ edges {
+ node {
+ id
+ name
+ key
+ color
+ externalId
+ createdAt {
+ iso8601
+ }
+ createdBy {
+ actorType: __typename
+ ... on UserActor { userId }
+ ... on MachineUserActor { machineUserId }
+ ... on SystemActor { systemId }
+ }
+ updatedAt {
+ iso8601
+ }
+ updatedBy {
+ actorType: __typename
+ ... on UserActor { userId }
+ ... on MachineUserActor { machineUserId }
+ ... on SystemActor { systemId }
+ }
+ }
+ }
+ pageInfo {
+ hasNextPage
+ endCursor
+ }
+ }
+}"""
+
+CUSTOMER_SURVEYS_QUERY = """
+query PaginatedCustomerSurveys($first: Int!, $after: String) {
+ customer_surveys: customerSurveys(first: $first, after: $after) {
+ edges {
+ node {
+ id
+ name
+ template {
+ ... on CsatCustomerSurveyTemplate {
+ type
+ questionText
+ }
+ }
+ conditionsOperator
+ isEnabled
+ responseDelayMinutes
+ customerIntervalDays
+ order
+ createdAt {
+ iso8601
+ }
+ createdBy {
+ actorType: __typename
+ ... on UserActor { userId }
+ ... on MachineUserActor { machineUserId }
+ ... on SystemActor { systemId }
+ }
+ updatedAt {
+ iso8601
+ }
+ updatedBy {
+ actorType: __typename
+ ... on UserActor { userId }
+ ... on MachineUserActor { machineUserId }
+ ... on SystemActor { systemId }
+ }
+ }
+ }
+ pageInfo {
+ hasNextPage
+ endCursor
+ }
+ }
+}"""
+
+DISCUSSIONS_QUERY = """
+query PaginatedDiscussions($first: Int!, $after: String) {
+ discussions(first: $first, after: $after, sortBy: { field: CREATED_AT, direction: ASC }) {
+ edges {
+ node {
+ id
+ threadId
+ title
+ status
+ agentStatus
+ agent {
+ id
+ }
+ agentSessionId
+ channelDetails {
+ channelType: __typename
+ }
+ visibility
+ sourceType
+ sourceEntityId
+ sourceEntityType
+ sourcePageLink
+ resolvedAt {
+ iso8601
+ }
+ lastActivityAt {
+ iso8601
+ }
+ createdAt {
+ iso8601
+ }
+ createdBy {
+ actorType: __typename
+ ... on UserActor { userId }
+ ... on MachineUserActor { machineUserId }
+ ... on CustomerActor { customerId }
+ ... on DeletedCustomerActor { customerId }
+ ... on SystemActor { systemId }
+ }
+ updatedAt {
+ iso8601
+ }
+ updatedBy {
+ actorType: __typename
+ ... on UserActor { userId }
+ ... on MachineUserActor { machineUserId }
+ ... on CustomerActor { customerId }
+ ... on DeletedCustomerActor { customerId }
+ ... on SystemActor { systemId }
+ }
+ }
+ }
+ pageInfo {
+ hasNextPage
+ endCursor
+ }
+ }
+}"""
+
+# `workflowExecutions` needs a workflowId; the workspace-wide query lists every workflow's runs in one walk.
+WORKFLOW_EXECUTIONS_QUERY = """
+query PaginatedWorkflowExecutions($first: Int!, $after: String) {
+ workflow_executions: workflowExecutionsForWorkspace(first: $first, after: $after) {
+ edges {
+ node {
+ id
+ workflowId
+ triggeredBy
+ executionStatus
+ errorMessage
+ entityType
+ entityId
+ startedAt {
+ iso8601
+ }
+ completedAt {
+ iso8601
+ }
+ executionDurationMs
+ manuallyTriggeredBy {
+ actorType: __typename
+ ... on UserActor { userId }
+ ... on MachineUserActor { machineUserId }
+ ... on SystemActor { systemId }
+ }
+ }
+ }
+ pageInfo {
+ hasNextPage
+ endCursor
+ }
+ }
+}"""
+
VIEWER_QUERY = "{ myWorkspace { id name } }"
QUERIES: dict[str, str] = {
@@ -434,4 +610,8 @@ QUERIES: dict[str, str] = {
"tenants": TENANTS_QUERY,
"users": USERS_QUERY,
"label_types": LABEL_TYPES_QUERY,
+ "customer_groups": CUSTOMER_GROUPS_QUERY,
+ "customer_surveys": CUSTOMER_SURVEYS_QUERY,
+ "discussions": DISCUSSIONS_QUERY,
+ "workflow_executions": WORKFLOW_EXECUTIONS_QUERY,
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/plain/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/plain/settings.py
index 3aa57319f511..5e1457c5ae11 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/plain/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/plain/settings.py
@@ -75,6 +75,24 @@ PLAIN_ENDPOINTS: dict[str, PlainEndpointConfig] = {
"label_types": PlainEndpointConfig(
incremental_fields=[],
),
+ # No list query below accepts an updatedAt filter. `discussions` only filters on lastActivityAt,
+ # which a status change such as resolving does not move, so it stays full refresh too.
+ "customer_groups": PlainEndpointConfig(
+ incremental_fields=[],
+ ),
+ "customer_surveys": PlainEndpointConfig(
+ incremental_fields=[],
+ ),
+ "discussions": PlainEndpointConfig(
+ incremental_fields=[],
+ ),
+ # Executions have no createdAt, and startedAt moves from enqueue to start time, so no stable partition key.
+ "workflow_executions": PlainEndpointConfig(
+ incremental_fields=[],
+ partition_mode=None,
+ partition_format=None,
+ partition_keys=None,
+ ),
}
ENDPOINTS = tuple(PLAIN_ENDPOINTS.keys())
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/plain/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/plain/source.py
index fb352f22af28..c337f55480b2 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/plain/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/plain/source.py
@@ -72,6 +72,10 @@ Make sure to grant the following read permissions:
- machineUser:read
- labelType:read
- tenant:read
+- customerGroup:read
+- customerSurvey:read
+- threadDiscussion:read
+- workflowRule:read
""",
iconPath="/static/services/plain.png",
fields=cast(
@@ -103,7 +107,7 @@ Make sure to grant the following read permissions:
# Our queries read assignee (user:read), label (label:read, labelType:read), company
# (company:read), tenant (tenant:read), and machine-user actor (machineUser:read) data on top
# of the basics, so name every required scope — retrying a key missing one never succeeds.
- "403 Client Error": "Access forbidden. Grant your Plain API key these read permissions, then reconnect: customer:read, thread:read, timeline:read, user:read, label:read, labelType:read, company:read, tenant:read, machineUser:read.",
+ "403 Client Error": "Access forbidden. Grant your Plain API key these read permissions, then reconnect: customer:read, thread:read, timeline:read, user:read, label:read, labelType:read, company:read, tenant:read, machineUser:read, customerGroup:read, customerSurvey:read, threadDiscussion:read, workflowRule:read.",
}
def get_retryable_errors(self) -> set[str]:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/plain/tests/test_plain.py b/products/warehouse_sources/backend/temporal/data_imports/sources/plain/tests/test_plain.py
index 2c79d212420d..e5468fa2afd0 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/plain/tests/test_plain.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/plain/tests/test_plain.py
@@ -111,13 +111,45 @@ class TestFlattenNode:
},
{"parentLabelTypeId": "lt_1", "archivedByType": "UserActor", "archivedById": "u_1"},
),
+ (
+ {"id": "disc_1", "agent": None, "channelDetails": None},
+ {"agentId": None, "channelType": None},
+ ),
+ (
+ {
+ "id": "disc_2",
+ "agent": {"id": "mu_1"},
+ "channelDetails": {"channelType": "ThreadDiscussionSlackChannelDetails"},
+ },
+ {"agentId": "mu_1", "channelType": "ThreadDiscussionSlackChannelDetails"},
+ ),
+ (
+ {"id": "cs_1", "template": {"type": "CSAT", "questionText": "How did we do?"}},
+ {"templateType": "CSAT", "templateQuestionText": "How did we do?"},
+ ),
+ (
+ {"id": "we_1", "manuallyTriggeredBy": None},
+ {"manuallyTriggeredByType": None, "manuallyTriggeredById": None},
+ ),
],
)
def test_flattens_nullable_references_into_stable_columns(self, node, expected):
result = _flatten_node(node)
assert {key: result[key] for key in expected} == expected
- assert not {"accountOwner", "parentLabelType", "deletedBy", "archivedBy"} & result.keys()
+ assert (
+ not {
+ "accountOwner",
+ "parentLabelType",
+ "deletedBy",
+ "archivedBy",
+ "agent",
+ "channelDetails",
+ "template",
+ "manuallyTriggeredBy",
+ }
+ & result.keys()
+ )
class TestFlattenTimelineEntry:
@@ -413,6 +445,10 @@ class TestLookupEndpoints:
("tenants", "PaginatedTenants"),
("users", "PaginatedUsers"),
("label_types", "PaginatedLabelTypes"),
+ ("customer_groups", "PaginatedCustomerGroups"),
+ ("customer_surveys", "PaginatedCustomerSurveys"),
+ ("discussions", "PaginatedDiscussions"),
+ ("workflow_executions", "PaginatedWorkflowExecutions"),
],
)
def test_walks_pages_until_has_next_page_is_false(self, endpoint, operation):