Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(warehouse): add four pylon endpoint tables (#126783)

Tom Owersb415dda9674c
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
index 8bb831e8d24f..9df86fc192a3 100644
--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
@@ -878,8 +878,6 @@
1 products/warehouse_sources/backend/temporal/data_imports/sources/productboard/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/pulumi_cloud/pulumi_cloud.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/pulumi_cloud/settings.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/pylon/pylon.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/pylon/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/pypi/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/qualaroo/qualaroo.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/qualaroo/settings.py
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 96e7ec00eb9d..aaaa194689ba 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -9860,14 +9860,14 @@ Today (13): `accounts`, `contacts`, `custom_fields`, `issue_statuses`, `issues`,
Diffed against: <https://api.usepylon.com/openapi.json>
-- [ ] `GET /issues/{id}/messages` — the actual conversation messages behind each issue - response times, volume, and content analysis all need this (high)
-- [ ] `GET /surveys/{id}/responses` — CSAT survey responses, the headline support-quality metric (high)
-- [ ] `GET /knowledge-bases/{id}/articles` — article inventory per knowledge base; the knowledge_bases table alone has no content rows (high)
+- [x] `GET /issues/{id}/messages` — the actual conversation messages behind each issue - response times, volume, and content analysis all need this (high)
+- [x] `GET /surveys/{id}/responses` — CSAT survey responses, the headline support-quality metric (high)
+- [x] `GET /knowledge-bases/{id}/articles` — article inventory per knowledge base; the knowledge_bases table alone has no content rows (high)
- [ ] `GET /surveys` — survey definitions that give the responses their questions and scales (medium)
- [ ] `GET /accounts/{account_id}/relationships` — account hierarchy (parent/child orgs), needed to roll ticket volume up to a customer group (medium)
- [ ] `GET /issues/{id}/threads` — per-issue thread breakdown across Slack/email channels (medium)
- [ ] `GET /custom-objects/{type}` — customer-defined object records already referenced from issues and accounts (medium)
-- [ ] `GET /activity-types` — lookup resolving activity type IDs used across issues and accounts (medium)
+- [x] `GET /activity-types` — lookup resolving activity type IDs used across issues and accounts (medium)
- [ ] `GET /knowledge-bases/{id}/collections` — collection grouping for knowledge base articles (medium)
- [ ] `GET /issues/{id}/followers` — who is watching each ticket - useful for internal-involvement analysis (medium)
- [ ] `GET /tasks/{id}/comments` — comment history on the tasks already synced (low)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/pylon/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/pylon/canonical_descriptions.py
index ae2287c1f2c6..2e19725b40ab 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/pylon/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/pylon/canonical_descriptions.py
@@ -200,4 +200,73 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"is_archived": "Indicates if the issue status is archived.",
},
},
+ "activity_types": {
+ "description": "Custom activity type definitions configured in Pylon, used to label activities on accounts and issues.",
+ "docs_url": "https://docs.usepylon.com/pylon-docs/developer/api/api-reference/accounts/activities",
+ "columns": {
+ "id": "The ID of the activity type.",
+ "slug": "The slug of the activity type.",
+ "label": "The label of the activity type.",
+ "icon_url": "The icon URL of the activity type.",
+ },
+ },
+ "knowledge_base_articles": {
+ "description": "Articles in each Pylon knowledge base, with their published and draft content.",
+ "docs_url": "https://docs.usepylon.com/pylon-docs/developer/api/api-reference/knowledge-base",
+ "columns": {
+ "id": "The ID of the article.",
+ "knowledge_base_id": "The ID of the knowledge base the article belongs to.",
+ "collection_id": "The ID of the collection this article belongs to, if any.",
+ "author_user_id": "The ID of the user who authored the article.",
+ "title": "The name of the article.",
+ "slug": "The slug of the article.",
+ "identifier": "The identifier of the article.",
+ "url": "The URL of the article.",
+ "is_published": "Whether the article is published.",
+ "is_unlisted": "Whether the article is accessible only via a direct link.",
+ "tags": "Tags assigned to the article.",
+ "current_published_content_html": "The current published content of the article in HTML.",
+ "current_published_content_markdown": "The current published content of the article in markdown.",
+ "current_draft_content_html": "The current draft content of the article in HTML, which may differ from the published version.",
+ "current_draft_content_markdown": "The current draft content of the article in markdown, which may differ from the published version.",
+ "created_at": "The time the article was created.",
+ "last_edited_at": "The time the article content was last edited.",
+ "last_published_at": "The time the article was last published.",
+ },
+ },
+ "survey_responses": {
+ "description": "Responses to Pylon surveys (such as CSAT and NPS), one row per submitted response.",
+ "docs_url": "https://docs.usepylon.com/pylon-docs/developer/api/api-reference/surveys",
+ "columns": {
+ "id": "The unique identifier of the survey response.",
+ "survey_id": "The ID of the survey template associated with this response.",
+ "survey_instance_id": "The ID of the individual survey delivery that produced this response.",
+ "survey_type": "The type of survey associated with this response (for example nps, csat, or custom).",
+ "account_id": "The ID of the account associated with the response.",
+ "contact_id": "The ID of the contact who submitted the response.",
+ "issue_id": "The ID of the issue associated with this response, when applicable.",
+ "answers": "The answers submitted in this response, each with its prompt, question type, and value.",
+ "submitted_at": "When the response was submitted.",
+ "excluded_at": "When the response was excluded from analytics.",
+ },
+ },
+ "issue_messages": {
+ "description": "Messages on each Pylon issue, including customer replies, agent replies, and internal notes.",
+ "docs_url": "https://docs.usepylon.com/pylon-docs/developer/api/api-reference/messages",
+ "columns": {
+ "id": "The top-level Pylon message ID.",
+ "issue_id": "The ID of the issue the message belongs to.",
+ "author": "The author of the message (a user or a contact).",
+ "author_unverified": "Whether the author identity of the message has not been verified.",
+ "message_html": "The HTML body of the message.",
+ "is_private": "Whether the message is an internal note that the requester cannot see.",
+ "source": "The source of the message.",
+ "thread_id": "The Pylon internal thread ID. Set only for internal notes.",
+ "file_urls": "Signed download URLs of the files in the message, if any.",
+ "reactions": "The emoji reactions on the message, grouped by emoji.",
+ "email_info": "Email metadata for messages sent over email.",
+ "slack_info": "Slack metadata for messages sent over Slack.",
+ "timestamp": "The time at which the message was created.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/pylon/pylon.py b/products/warehouse_sources/backend/temporal/data_imports/sources/pylon/pylon.py
index 8d07fe464102..9136b3886a5e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/pylon/pylon.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/pylon/pylon.py
@@ -2,7 +2,7 @@ import dataclasses
from collections.abc import Iterator
from datetime import UTC, date, datetime, timedelta
from typing import Any, Optional
-from urllib.parse import urlencode
+from urllib.parse import quote, urlencode
import requests
from structlog.types import FilteringBoundLogger
@@ -31,7 +31,7 @@ class PylonRetryableError(Exception):
pass
[email protected](frozen=False)
class PylonResumeConfig:
# Next-page cursor within the current page scroll.
cursor: str | None = None
@@ -39,6 +39,13 @@ class PylonResumeConfig:
window_start: str | None = None
# The custom-fields object_type currently being fanned out (fan-out endpoints only).
object_type: str | None = None
+ # Child endpoints: the cursor that fetched the current parent page, the parent being read, and
+ # whether every child page of that parent has already been yielded.
+ parent_cursor: str | None = None
+ parent_id: str | None = None
+ parent_completed: bool = False
+ # RFC3339 cutoff that every child request of an incremental run is bounded by.
+ upper_bound: str | None = None
def base_url_for_token(api_token: str) -> str:
@@ -61,6 +68,14 @@ def _format_rfc3339(dt: datetime) -> str:
return utc_dt.strftime("%Y-%m-%dT%H:%M:%SZ")
+def _format_rfc3339_precise(dt: datetime) -> str:
+ """Like `_format_rfc3339`, but keeps sub-second precision so a watermark doesn't re-match its own row."""
+ utc_dt = dt.replace(tzinfo=UTC) if dt.tzinfo is None else dt.astimezone(UTC)
+ if utc_dt.microsecond:
+ return utc_dt.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
+ return utc_dt.strftime("%Y-%m-%dT%H:%M:%SZ")
+
+
def _parse_rfc3339(value: str) -> datetime:
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
return parsed if parsed.tzinfo is not None else parsed.replace(tzinfo=UTC)
@@ -281,6 +296,118 @@ def _get_windowed_rows(
manager.save_state(PylonResumeConfig(cursor=None, window_start=_format_rfc3339(window_start)))
+def _iter_parent_pages(
+ session: requests.Session,
+ headers: dict[str, str],
+ base_url: str,
+ config: PylonEndpointConfig,
+ logger: FilteringBoundLogger,
+ resume: PylonResumeConfig | None,
+) -> Iterator[tuple[list[str], PylonResumeConfig]]:
+ """Yield (parent ids, position) per parent page, where position locates that page for resume."""
+ assert config.parent is not None
+ parent = config.parent
+
+ windows: list[dict[str, Any]] = [{}]
+ if parent.windowed:
+ now = datetime.now(UTC)
+ if resume is not None and resume.window_start:
+ window_start = _parse_rfc3339(resume.window_start)
+ else:
+ window_start = now - timedelta(days=config.default_lookback_days or ISSUES_MAX_WINDOW_DAYS)
+ windows = []
+ while window_start < now:
+ window_end = min(window_start + timedelta(days=ISSUES_MAX_WINDOW_DAYS), now)
+ windows.append({"start_time": _format_rfc3339(window_start), "end_time": _format_rfc3339(window_end)})
+ window_start = window_end
+
+ page_cursor = resume.parent_cursor if resume else None
+ for window in windows:
+ params = dict(window)
+ if parent.limit:
+ params["limit"] = parent.limit
+ for items, next_cursor in _iter_pages(session, headers, base_url, parent.path, params, logger, page_cursor):
+ parent_ids = [str(item["id"]) for item in items if item.get("id")]
+ yield parent_ids, PylonResumeConfig(window_start=window.get("start_time"), parent_cursor=page_cursor)
+ page_cursor = next_cursor
+ page_cursor = None
+
+
+def _get_child_rows(
+ session: requests.Session,
+ headers: dict[str, str],
+ base_url: str,
+ config: PylonEndpointConfig,
+ logger: FilteringBoundLogger,
+ manager: ResumableSourceManager[PylonResumeConfig],
+ resume: PylonResumeConfig | None,
+ should_use_incremental_field: bool,
+ db_incremental_field_last_value: Any,
+) -> Iterator[list[dict[str, Any]]]:
+ assert config.parent is not None
+ id_field = config.parent.id_field
+
+ params: dict[str, Any] = {}
+ if config.limit:
+ params["limit"] = config.limit
+ if config.incremental_param and should_use_incremental_field and db_incremental_field_last_value:
+ params[config.incremental_param] = _format_rfc3339_precise(_to_datetime(db_incremental_field_last_value))
+ upper_bound: str | None = None
+ if config.incremental_upper_param and should_use_incremental_field:
+ # Every parent is read up to the same instant. Otherwise a row added to an already-read parent
+ # falls below the newer watermark a later parent sets, and the next run never reads it.
+ upper_bound = resume.upper_bound if resume and resume.upper_bound else _format_rfc3339(datetime.now(UTC))
+ params[config.incremental_upper_param] = upper_bound
+
+ resume_parent_id = resume.parent_id if resume else None
+ for parent_ids, position in _iter_parent_pages(session, headers, base_url, config, logger, resume):
+ resume_cursor: str | None = None
+ # The saved parent is only looked for on the resumed parent page. If it has since been deleted,
+ # the whole page is re-read and merge dedupes on the primary key.
+ if resume is not None and resume_parent_id is not None and resume_parent_id in parent_ids:
+ index = parent_ids.index(resume_parent_id)
+ if resume.parent_completed:
+ parent_ids = parent_ids[index + 1 :]
+ else:
+ parent_ids = parent_ids[index:]
+ resume_cursor = resume.cursor
+ resume_parent_id = None
+
+ for parent_id in parent_ids:
+ path = config.path.format(parent_id=quote(parent_id, safe=""))
+ yielded = False
+ try:
+ for items, next_cursor in _iter_pages(session, headers, base_url, path, params, logger, resume_cursor):
+ manager.save_state(
+ dataclasses.replace(
+ position,
+ parent_id=parent_id,
+ cursor=next_cursor,
+ parent_completed=next_cursor is None,
+ upper_bound=upper_bound,
+ )
+ )
+ if items:
+ yield [{**item, id_field: item.get(id_field) or parent_id} for item in items]
+ yielded = True
+ except requests.HTTPError as e:
+ # The parent was deleted after the parent list was read.
+ if e.response is None or e.response.status_code != 404:
+ raise
+ logger.warning(f"Pylon: parent {parent_id} of {config.name} no longer exists, skipping")
+ manager.save_state(
+ dataclasses.replace(
+ position, parent_id=parent_id, cursor=None, parent_completed=True, upper_bound=upper_bound
+ )
+ )
+ resume_cursor = None
+ if not yielded:
+ manager.safe_point()
+
+ if not parent_ids:
+ manager.safe_point()
+
+
def get_rows(
api_token: str,
endpoint: str,
@@ -310,6 +437,18 @@ def get_rows(
should_use_incremental_field,
db_incremental_field_last_value,
)
+ elif config.parent:
+ yield from _get_child_rows(
+ session,
+ headers,
+ base_url,
+ config,
+ logger,
+ resumable_source_manager,
+ resume,
+ should_use_incremental_field,
+ db_incremental_field_last_value,
+ )
elif config.fan_out_object_types:
yield from _get_fan_out_rows(session, headers, base_url, config, logger, resumable_source_manager, resume)
else:
@@ -342,7 +481,8 @@ def pylon_source(
partition_mode="datetime" if config.partition_key else None,
partition_format="month" if config.partition_key else None,
partition_keys=[config.partition_key] if config.partition_key else None,
- # Issues windows are processed strictly ascending; the other endpoints are full-refresh cursor
- # scrolls where sort_mode is not used to checkpoint a watermark.
- sort_mode="asc",
+ # Issues windows are processed strictly ascending; the other top-level endpoints are full-refresh
+ # cursor scrolls where sort_mode is not used to checkpoint a watermark. Child endpoints read one
+ # series per parent, so their rows are not ordered across the table.
+ sort_mode=None if config.parent else "asc",
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/pylon/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/pylon/settings.py
index ecb12212bb10..2207072211b6 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/pylon/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/pylon/settings.py
@@ -4,7 +4,18 @@ from typing import Optional
from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType
-@dataclass
+@dataclass(frozen=True)
+class PylonParentConfig:
+ path: str
+ # Column stamped onto each child row with the parent's id. Child ids are only documented as unique
+ # within their parent, so this column is part of the child's composite primary key.
+ id_field: str
+ # The parent list needs the issues-style <=30-day start_time/end_time window.
+ windowed: bool = False
+ limit: Optional[int] = None
+
+
+@dataclass(frozen=False)
class PylonEndpointConfig:
name: str
path: str
@@ -17,8 +28,8 @@ class PylonEndpointConfig:
# expose no stable timestamp (contacts, users, teams, tags, ...).
partition_key: Optional[str] = None
incremental_fields: list[IncrementalField] = field(default_factory=list)
- # Only issues exposes a genuine server-side time filter (start_time/end_time), so it is the only
- # endpoint that can sync incrementally. Everything else is cursor-only and ships full refresh.
+ # Only issues (start_time/end_time) and survey responses (submitted_after) expose a genuine
+ # server-side time filter, so only they sync incrementally. Everything else ships full refresh.
supports_incremental: bool = False
# `GET /issues` requires an RFC3339 start_time/end_time window capped at 30 days, so its backfill
# is walked forward one <=30-day window at a time rather than as a single cursor scroll.
@@ -34,6 +45,12 @@ class PylonEndpointConfig:
# over each object type and stamp it onto every row. The id is only unique within an object type,
# hence the composite primary key.
fan_out_object_types: Optional[list[str]] = None
+ # Child endpoints (`path` holds a `{parent_id}` placeholder) are fetched once per parent row.
+ parent: Optional[PylonParentConfig] = None
+ # Server-side "newer than" filter on a child endpoint, sent with the incremental watermark.
+ incremental_param: Optional[str] = None
+ # Server-side "older than" filter on a child endpoint, sent with one cutoff for the whole run.
+ incremental_upper_param: Optional[str] = None
should_sync_default: bool = True
@@ -45,6 +62,9 @@ ISSUES_MAX_WINDOW_DAYS = 30
CUSTOM_FIELD_OBJECT_TYPES = ["account", "issue", "contact", "task", "project", "meeting", "opportunity"]
+ISSUES_PARENT = PylonParentConfig(path="/issues", id_field="issue_id", windowed=True, limit=2000)
+
+
def _datetime_field(name: str) -> IncrementalField:
return {
"label": name,
@@ -127,6 +147,42 @@ PYLON_ENDPOINTS: dict[str, PylonEndpointConfig] = {
# IssueStatus objects have no `id`; `slug` is the stable unique key.
primary_keys=["slug"],
),
+ "activity_types": PylonEndpointConfig(
+ name="activity_types",
+ path="/activity-types",
+ ),
+ "knowledge_base_articles": PylonEndpointConfig(
+ name="knowledge_base_articles",
+ path="/knowledge-bases/{parent_id}/articles",
+ primary_keys=["knowledge_base_id", "id"],
+ partition_key="created_at",
+ limit=999,
+ parent=PylonParentConfig(path="/knowledge-bases", id_field="knowledge_base_id"),
+ ),
+ "survey_responses": PylonEndpointConfig(
+ name="survey_responses",
+ path="/surveys/{parent_id}/responses",
+ primary_keys=["survey_id", "id"],
+ partition_key="submitted_at",
+ limit=999,
+ parent=PylonParentConfig(path="/surveys", id_field="survey_id"),
+ supports_incremental=True,
+ incremental_param="submitted_after",
+ incremental_upper_param="submitted_before",
+ incremental_fields=[_datetime_field("submitted_at")],
+ ),
+ "issue_messages": PylonEndpointConfig(
+ name="issue_messages",
+ path="/issues/{parent_id}/messages",
+ primary_keys=["issue_id", "id"],
+ partition_key="timestamp",
+ limit=1000,
+ # The endpoint has no time filter, and new messages land on old issues, so this is a full
+ # refresh over every issue in the lookback. It costs one request per issue, so it's opt-in.
+ parent=ISSUES_PARENT,
+ default_lookback_days=365,
+ should_sync_default=False,
+ ),
}
ENDPOINTS = tuple(PYLON_ENDPOINTS.keys())
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/pylon/tests/test_pylon.py b/products/warehouse_sources/backend/temporal/data_imports/sources/pylon/tests/test_pylon.py
index 6f02e02ee18d..4e1da88897c4 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/pylon/tests/test_pylon.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/pylon/tests/test_pylon.py
@@ -8,6 +8,13 @@ from unittest.mock import MagicMock
from parameterized import parameterized
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
+ ScriptedResponse,
+ SourceDriver,
+ route,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing.fake_network import RecordedRequest
+from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.pylon import PylonSourceConfig
from products.warehouse_sources.backend.temporal.data_imports.sources.pylon import pylon
from products.warehouse_sources.backend.temporal.data_imports.sources.pylon.pylon import (
PYLON_EU_BASE_URL,
@@ -20,6 +27,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.pylon.pylo
pylon_source,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.pylon.settings import PYLON_ENDPOINTS
+from products.warehouse_sources.backend.temporal.data_imports.sources.pylon.source import PylonSource
def _page(items: list[dict[str, Any]], cursor: str | None = None) -> dict[str, Any]:
@@ -247,3 +255,138 @@ class TestPylonSourceResponse:
assert response.partition_keys == [partition_key]
assert response.partition_mode == "datetime"
assert response.partition_format == "month"
+
+
+def _driver() -> SourceDriver:
+ return SourceDriver(PylonSource(), PylonSourceConfig(api_token="pylon_api_token"))
+
+
+def _scripted_page(items: list[dict[str, Any]], cursor: str | None = None) -> ScriptedResponse:
+ return ScriptedResponse(json=_page(items, cursor))
+
+
+class TestChildEndpoints:
+ @parameterized.expand(
+ [
+ ("incremental", True, "2026-01-02T03:04:05Z", None, "2026-01-02T03:04:05Z", "2026-06-23T00:00:00Z"),
+ (
+ "resumed_incremental_keeps_cutoff",
+ True,
+ "2026-01-02T03:04:05Z",
+ "2026-06-01T00:00:00Z",
+ "2026-01-02T03:04:05Z",
+ "2026-06-01T00:00:00Z",
+ ),
+ (
+ "incremental_keeps_subsecond_watermark",
+ True,
+ "2026-01-02T03:04:05.123456Z",
+ None,
+ "2026-01-02T03:04:05.123456Z",
+ "2026-06-23T00:00:00Z",
+ ),
+ ("full_refresh", False, None, None, None, None),
+ ]
+ )
+ @time_machine.travel("2026-06-23T00:00:00Z", tick=False)
+ def test_survey_responses_walk_every_survey_to_its_last_page(
+ self,
+ _name: str,
+ should_use_incremental_field: bool,
+ last_value: str | None,
+ saved_cutoff: str | None,
+ expected_filter: str | None,
+ expected_cutoff: str | None,
+ ) -> None:
+ result = _driver().run(
+ "survey_responses",
+ route(
+ {
+ "/surveys": [ScriptedResponse(json={"data": [{"id": "s1"}, {"id": "s2"}], "request_id": "r"})],
+ "/surveys/s1/responses": [
+ _scripted_page([{"id": "r1"}], cursor="c2"),
+ _scripted_page([{"id": "r2", "survey_id": "s1"}]),
+ ],
+ "/surveys/s2/responses": [_scripted_page([{"id": "r3"}])],
+ }
+ ),
+ incremental_field="submitted_at",
+ should_use_incremental_field=should_use_incremental_field,
+ db_incremental_field_last_value=last_value,
+ resume_state=PylonResumeConfig(upper_bound=saved_cutoff) if saved_cutoff else None,
+ )
+
+ assert result.raised is None
+ assert result.paths == ["/surveys", "/surveys/s1/responses", "/surveys/s1/responses", "/surveys/s2/responses"]
+ assert result.params("cursor") == [None, None, "c2", None]
+ assert result.params("submitted_after")[1:] == [expected_filter] * 3
+ assert result.params("submitted_before")[1:] == [expected_cutoff] * 3
+ assert [(row["survey_id"], row["id"]) for row in result.rows] == [("s1", "r1"), ("s1", "r2"), ("s2", "r3")]
+ # One request series per survey, so rows are not ordered across the table.
+ assert result.response is not None
+ assert result.response.sort_mode is None
+
+ @parameterized.expand(
+ [
+ (
+ "mid_parent",
+ False,
+ ["/knowledge-bases", "/knowledge-bases/kb2/articles", "/knowledge-bases/kb3/articles"],
+ ["c2", None],
+ ),
+ ("parent_completed", True, ["/knowledge-bases", "/knowledge-bases/kb3/articles"], [None]),
+ ]
+ )
+ def test_articles_resume_at_the_saved_knowledge_base(
+ self, _name: str, parent_completed: bool, expected_paths: list[str], expected_cursors: list[str | None]
+ ) -> None:
+ result = _driver().run(
+ "knowledge_base_articles",
+ route(
+ {
+ "/knowledge-bases": [_scripted_page([{"id": "kb1"}, {"id": "kb2"}, {"id": "kb3"}])],
+ "/knowledge-bases/kb2/articles": [_scripted_page([{"id": "a2"}])],
+ "/knowledge-bases/kb3/articles": [_scripted_page([{"id": "a3"}])],
+ }
+ ),
+ resume_state=PylonResumeConfig(cursor="c2", parent_id="kb2", parent_completed=parent_completed),
+ )
+
+ assert result.raised is None
+ assert result.paths == expected_paths
+ assert result.params("cursor")[1:] == expected_cursors
+ assert all(row["knowledge_base_id"] in {"kb2", "kb3"} for row in result.rows)
+
+ def test_skips_a_parent_deleted_after_the_parent_list_was_read(self) -> None:
+ result = _driver().run(
+ "knowledge_base_articles",
+ route(
+ {
+ "/knowledge-bases": [_scripted_page([{"id": "kb1"}, {"id": "kb2"}])],
+ "/knowledge-bases/kb1/articles": [ScriptedResponse(status=404, json={"errors": ["not found"]})],
+ "/knowledge-bases/kb2/articles": [_scripted_page([{"id": "a2"}])],
+ }
+ ),
+ )
+
+ assert result.raised is None
+ assert [(row["knowledge_base_id"], row["id"]) for row in result.rows] == [("kb2", "a2")]
+
+ @time_machine.travel("2026-06-23T00:00:00Z", tick=False)
+ def test_issue_messages_fan_out_over_every_issues_window(self) -> None:
+ def respond(request: RecordedRequest) -> ScriptedResponse:
+ if request.path == "/issues":
+ if request.param("start_time") == "2026-06-18T00:00:00Z":
+ return _scripted_page([{"id": "i1"}])
+ return _scripted_page([])
+ assert request.path == "/issues/i1/messages"
+ return _scripted_page([{"id": "m1"}, {"id": "m2"}])
+
+ result = _driver().run("issue_messages", respond)
+
+ assert result.raised is None
+ issue_requests = [request for request in result.requests if request.path == "/issues"]
+ assert len(issue_requests) == 13
+ assert issue_requests[0].param("start_time") == "2025-06-23T00:00:00Z"
+ assert issue_requests[-1].param("end_time") == "2026-06-23T00:00:00Z"
+ assert [(row["issue_id"], row["id"]) for row in result.rows] == [("i1", "m1"), ("i1", "m2")]