Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add 4 missing tables to the Bitrise source (#119164)

Tom Owers2b98355654b3
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index d81f4b4a58a4..bd47e247fd25 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -1667,7 +1667,7 @@ pull request in one walk, so it costs one request per page instead of one per pu
## Bitrise — gaps
-Today (8): `apps`, `artifacts`, `branches`, `builds`, `organization_members`, `organizations`, `pipelines`, `workflows`
+Today (12): `activities`, `apps`, `archived_builds`, `artifacts`, `available_stacks`, `branches`, `build_requests`, `builds`, `organization_members`, `organizations`, `pipelines`, `workflows`
Diffed against: <https://api-docs.bitrise.io/docs/swagger.json>
@@ -1675,10 +1675,10 @@ Diffed against: <https://api-docs.bitrise.io/docs/swagger.json>
- [x] `/organizations/{org-slug}/members` — org membership records for attributing builds to people (medium)
- [x] `/organizations` — lookup resolving the org that owns each app we already sync (medium)
- [x] `/apps/{app-slug}/branches` — branch lookup for grouping build history (medium)
-- [ ] `/apps/{app-slug}/archived-builds` — extends build history past the live builds window (medium)
-- [ ] `/apps/{app-slug}/build-requests` — queued/pending build requests reveal CI backlog (low)
-- [ ] `/me/activities` — account-level activity event feed (low)
-- [ ] `/available-stacks` — lookup for the stack identifier carried on builds (low)
+- [x] `/apps/{app-slug}/archived-builds` — extends build history past the live builds window (medium)
+- [x] `/apps/{app-slug}/build-requests` — queued/pending build requests reveal CI backlog (low)
+- [x] `/me/activities` — account-level activity event feed (low)
+- [x] `/available-stacks` — lookup for the stack identifier carried on builds (low)
## BlandAI — **thin**
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bitrise/bitrise.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bitrise/bitrise.py
index ab9615043065..7e9ea974a8d1 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bitrise/bitrise.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bitrise/bitrise.py
@@ -29,6 +29,8 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.typ
BITRISE_BASE_URL = "https://api.bitrise.io/v0.1"
# Bitrise list endpoints cap `limit` at 50.
PAGE_LIMIT = 50
+# The archived builds endpoint takes no paging anchor and returns at most 1000 builds per app.
+ARCHIVED_BUILDS_LIMIT = 1000
# Safety overlap subtracted from the incremental watermark: builds that were still running (or on
# hold) at the last sync keep mutating (status, finished_at) after their trigger time, so each run
# re-pulls a trailing window and merge dedupes the re-fetched rows on the primary key.
@@ -40,11 +42,12 @@ class BitriseResumeConfig:
# Fan-out bookmark: retained only so pre-migration saved state still parses via
# `dataclass(**saved)`. New runs never populate it; fan-out resume now lives in `fanout_state`.
app_slug: str | None = None
- # `next` paging anchor for the flat apps listing. None means "start at the first page".
+ # `next` paging anchor for the flat apps and activities listings. None means "start at the
+ # first page".
next: str | None = None
# Framework fan-out resume snapshot ({"completed": [...], "current": ..., "child_state": ...})
# for the single-hop parent-app -> child-endpoint dependent resources (builds, pipelines,
- # workflows, branches).
+ # workflows, branches, archived builds, build requests).
fanout_state: dict[str, Any] | None = None
@@ -132,6 +135,18 @@ def _apps_resource() -> EndpointResource:
}
+def _activities_resource() -> EndpointResource:
+ return {
+ "name": "activities",
+ "endpoint": {
+ "path": "/me/activities",
+ "params": {"limit": PAGE_LIMIT},
+ "data_selector": "data",
+ "paginator": _cursor_paginator(),
+ },
+ }
+
+
def _organizations_resource() -> EndpointResource:
# The organizations listing takes no params and returns every org in one body.
return {
@@ -155,15 +170,16 @@ def _builds_child_params(after: int | None) -> dict[str, Any]:
return params
-def _apps_source(
+def _flat_cursor_source(
api_token: str,
team_id: int,
job_id: str,
resumable_source_manager: ResumableSourceManager[BitriseResumeConfig],
+ resource: EndpointResource,
) -> Resource:
rest_config: RESTAPIConfig = {
"client": _client_config(api_token),
- "resources": [_apps_resource()],
+ "resources": [resource],
}
initial_paginator_state: Optional[dict[str, Any]] = None
@@ -321,6 +337,58 @@ def _app_name_list_source(
return _single_hop_fanout_source(api_token, team_id, job_id, resumable_source_manager, child_resource)
+def _single_page_app_child_source(
+ api_token: str,
+ team_id: int,
+ job_id: str,
+ resumable_source_manager: ResumableSourceManager[BitriseResumeConfig],
+ name: str,
+ path: str,
+ extra_params: dict[str, Any],
+) -> Resource:
+ params: dict[str, Any] = {"app_slug": {"type": "resolve", "resource": "apps", "field": "slug"}, **extra_params}
+ child_resource: EndpointResource = {
+ "name": name,
+ "include_from_parent": ["slug"],
+ "endpoint": {
+ "path": path,
+ "params": params,
+ "data_selector": "data",
+ "paginator": SinglePagePaginator(),
+ "response_actions": [{"status_code": 404, "action": "ignore"}],
+ },
+ "data_map": rename_parent_fields("apps", {"slug": "app_slug"}),
+ }
+ return _single_hop_fanout_source(api_token, team_id, job_id, resumable_source_manager, child_resource)
+
+
+def _explode_available_stacks(row: dict[str, Any]) -> list[dict[str, Any]]:
+ """Explode the `{"<stack id>": {...}}` map into one row per stack, keeping the map key as `id`."""
+ return [{**stack, "id": stack.get("id") or stack_id} for stack_id, stack in row.items() if isinstance(stack, dict)]
+
+
+def _available_stacks_source(
+ api_token: str,
+ team_id: int,
+ job_id: str,
+) -> Resource:
+ resource: EndpointResource = {
+ "name": "available_stacks",
+ "endpoint": {
+ "path": "/available-stacks",
+ # The body is a bare map keyed by stack id, so take it whole and let data_map explode it.
+ "data_selector": "$",
+ "paginator": SinglePagePaginator(),
+ },
+ "data_map": _explode_available_stacks,
+ }
+ rest_config: RESTAPIConfig = {
+ "client": _client_config(api_token),
+ "resources": [resource],
+ }
+ return rest_api_resource(rest_config, team_id, job_id, None)
+
+
def _organizations_source(
api_token: str,
team_id: int,
@@ -462,7 +530,7 @@ def bitrise_source(
after = _build_after_param(should_use_incremental_field, db_incremental_field_last_value)
if endpoint == "apps":
- resource = _apps_source(api_token, team_id, job_id, resumable_source_manager)
+ resource = _flat_cursor_source(api_token, team_id, job_id, resumable_source_manager, _apps_resource())
elif endpoint == "builds":
resource = _builds_source(api_token, team_id, job_id, resumable_source_manager, after)
elif endpoint == "pipelines":
@@ -483,6 +551,30 @@ def bitrise_source(
resource = _organization_members_source(api_token, team_id, job_id)
elif endpoint == "artifacts":
resource = _artifacts_source(api_token, team_id, job_id, after)
+ elif endpoint == "archived_builds":
+ resource = _single_page_app_child_source(
+ api_token,
+ team_id,
+ job_id,
+ resumable_source_manager,
+ name="archived_builds",
+ path="/apps/{app_slug}/archived-builds",
+ extra_params={"limit": ARCHIVED_BUILDS_LIMIT},
+ )
+ elif endpoint == "build_requests":
+ resource = _single_page_app_child_source(
+ api_token,
+ team_id,
+ job_id,
+ resumable_source_manager,
+ name="build_requests",
+ path="/apps/{app_slug}/build-requests",
+ extra_params={},
+ )
+ elif endpoint == "activities":
+ resource = _flat_cursor_source(api_token, team_id, job_id, resumable_source_manager, _activities_resource())
+ elif endpoint == "available_stacks":
+ resource = _available_stacks_source(api_token, team_id, job_id)
else:
raise ValueError(f"Unknown Bitrise endpoint: {endpoint}")
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bitrise/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bitrise/canonical_descriptions.py
index 16a3cbbc0378..1530a6549b61 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bitrise/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bitrise/canonical_descriptions.py
@@ -136,4 +136,68 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"is_public_page_enabled": "Whether a public install page is enabled for the artifact.",
},
},
+ "archived_builds": {
+ "description": "A build that has moved out of the live builds listing into the archive, one row per app and build. "
+ "Bitrise returns at most 1000 archived builds per app.",
+ "docs_url": "https://api-docs.bitrise.io/#/builds",
+ "columns": {
+ "app_slug": "Identifier of the app the build belongs to (added by PostHog during sync).",
+ "slug": "Unique identifier of the build.",
+ "build_number": "Sequential build number within the app.",
+ "status": "Numeric build status (0 running, 1 successful, 2 failed, 3 aborted with failure, 4 aborted with success).",
+ "status_text": "Human-readable build status (e.g. success, error, aborted).",
+ "abort_reason": "Reason the build was aborted, if it was.",
+ "branch": "Git branch the build ran on.",
+ "tag": "Git tag that triggered the build, if any.",
+ "commit_hash": "Hash of the commit the build ran against.",
+ "commit_message": "Message of the commit the build ran against.",
+ "triggered_workflow": "Name of the Bitrise workflow the build ran.",
+ "triggered_by": "What triggered the build (e.g. a webhook, a schedule, or a manual start).",
+ "started_on_worker_at": "When the build started running on a worker.",
+ "finished_at": "When the build finished.",
+ "stack_identifier": "Identifier of the stack the build ran on.",
+ "machine_type_id": "Identifier of the machine type the build ran on.",
+ "credit_cost": "Credits the build consumed.",
+ "build_artifacts": "Artifacts the build produced.",
+ "original_build_params": "Parameters the build was started with.",
+ },
+ },
+ "build_requests": {
+ "description": "An open build request of an app that waits for approval before it starts a build.",
+ "docs_url": "https://api-docs.bitrise.io/#/build-request",
+ "columns": {
+ "app_slug": "Identifier of the app the build request belongs to (added by PostHog during sync).",
+ "slug": "Unique identifier of the build request.",
+ "created_at": "When the build request was created.",
+ "build_params": "Parameters the build will start with.",
+ "pull_request_url": "URL of the pull request that created the build request.",
+ },
+ },
+ "activities": {
+ "description": "An activity event of the Bitrise account that owns the API token.",
+ "docs_url": "https://api-docs.bitrise.io/#/activity",
+ "columns": {
+ "slug": "Unique identifier of the activity event.",
+ "created_at": "When the activity event happened.",
+ "title": "Title of the activity event.",
+ "description": "Description of the activity event.",
+ "event_stype": "Type of the activity event.",
+ "repository_title": "Title of the app the activity event relates to.",
+ "target_path_string": "Path in the Bitrise web app that the activity event links to.",
+ },
+ },
+ "available_stacks": {
+ "description": "A build stack (operating system image with preinstalled tools) that builds can run on.",
+ "docs_url": "https://api-docs.bitrise.io/#/configuration",
+ "columns": {
+ "id": "Unique identifier of the stack, as carried on builds in stack_identifier.",
+ "title": "Human-readable title of the stack.",
+ "description": "Description of the stack.",
+ "os": "Operating system of the stack (linux or macos).",
+ "os_version": "Version number of the operating system.",
+ "status": "Current status of the stack (stable, edge, or frozen).",
+ "removal_date": "Date on which the stack stops being available. Usually set only for frozen stacks.",
+ "stack_report": "URL of the report that lists the preinstalled tools of the stack.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bitrise/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bitrise/settings.py
index 2ddf4f88a02d..68fc29440e6d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bitrise/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bitrise/settings.py
@@ -17,8 +17,9 @@ class BitriseEndpointConfig:
# the given time), which gives an honest incremental sync on `triggered_at`. The pipelines endpoint
# takes the same filter as an RFC3339 date. Artifacts inherit the builds filter through their parent
# build fan-out, keyed on the injected `build_triggered_at`. Apps, workflows, branches,
-# organizations and organization members are small listings with no server-side time filter, so
-# they stay full refresh.
+# organizations, organization members, build requests, activities and stacks have no server-side
+# time filter, so they stay full refresh. Archived builds take the same `after` filter as builds, but
+# their rows carry no `triggered_at` to watermark on, so they stay full refresh too.
BITRISE_ENDPOINTS: dict[str, BitriseEndpointConfig] = {
"apps": BitriseEndpointConfig(
name="apps",
@@ -85,6 +86,26 @@ BITRISE_ENDPOINTS: dict[str, BitriseEndpointConfig] = {
# A member slug identifies the Bitrise account, which can belong to several organizations.
primary_keys=["org_slug", "slug"],
),
+ "archived_builds": BitriseEndpointConfig(
+ name="archived_builds",
+ primary_keys=["app_slug", "slug"],
+ partition_key="started_on_worker_at",
+ ),
+ "build_requests": BitriseEndpointConfig(
+ name="build_requests",
+ primary_keys=["app_slug", "slug"],
+ partition_key="created_at",
+ ),
+ "activities": BitriseEndpointConfig(
+ name="activities",
+ partition_key="created_at",
+ # A full refresh pages through the whole account activity history on every run.
+ should_sync_default=False,
+ ),
+ "available_stacks": BitriseEndpointConfig(
+ name="available_stacks",
+ primary_keys=["id"],
+ ),
}
ENDPOINTS = tuple(BITRISE_ENDPOINTS.keys())
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bitrise/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bitrise/source.py
index d3ecb3a6ede0..f8717bf133d9 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bitrise/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bitrise/source.py
@@ -105,6 +105,12 @@ You can create a personal access token in your [Bitrise security settings](https
)
if endpoint == "branches":
return "Only branches that already have builds on Bitrise are listed"
+ if endpoint == "archived_builds":
+ return "The Bitrise API returns at most 1000 archived builds per app"
+ if endpoint == "build_requests":
+ return "Only open build requests are listed"
+ if endpoint == "activities":
+ return "Activity events of the account that owns the API token. Disabled by default because every sync reads the full history"
return None
def _build_schema(endpoint: str) -> SourceSchema:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bitrise/tests/test_bitrise.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bitrise/tests/test_bitrise.py
index 868f75ab9528..c5dd3c2c0678 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bitrise/tests/test_bitrise.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bitrise/tests/test_bitrise.py
@@ -92,18 +92,27 @@ class TestValidateCredentials:
assert {request.path for request in network.requests_log} == {"/v0.1/me"}
-class TestApps:
- def test_paginates_via_next_anchor_and_saves_state(self) -> None:
+class TestFlatCursorListings:
+ @pytest.mark.parametrize(
+ "endpoint, path",
+ [
+ ("apps", "/v0.1/apps"),
+ ("activities", "/v0.1/me/activities"),
+ ],
+ )
+ def test_paginates_via_next_anchor_and_saves_state(self, endpoint: str, path: str) -> None:
result = _driver().run(
- "apps",
+ endpoint,
[
- _response({"data": [{"slug": "app1"}], "paging": {"next": "anchor1"}}),
- _response({"data": [{"slug": "app2"}], "paging": {}}),
+ _response({"data": [{"slug": "row1"}], "paging": {"next": "anchor1"}}),
+ _response({"data": [{"slug": "row2"}], "paging": {}}),
],
)
assert result.raised is None
- assert [row["slug"] for row in result.rows] == ["app1", "app2"]
+ assert [row["slug"] for row in result.rows] == ["row1", "row2"]
+ assert result.paths == [path, path]
+ assert result.params("limit") == ["50", "50"]
# The second page carries the anchor returned by the first.
assert result.params("next") == [None, "anchor1"]
# State saved once, after the first page yielded and only while more pages remain.
@@ -210,6 +219,76 @@ class TestBranches:
assert result.paths[1] == "/v0.1/apps/app1/branches"
+class TestSinglePageAppChildren:
+ @pytest.mark.parametrize(
+ "endpoint, path, expected_limit",
+ [
+ ("archived_builds", "/v0.1/apps/{app}/archived-builds", "1000"),
+ ("build_requests", "/v0.1/apps/{app}/build-requests", None),
+ ],
+ )
+ def test_fans_out_over_apps_with_one_request_per_app(
+ self, endpoint: str, path: str, expected_limit: str | None
+ ) -> None:
+ result = _driver().run(
+ endpoint,
+ [
+ _response({"data": [{"slug": "app1"}, {"slug": "app2"}, {"slug": "gone"}], "paging": {}}),
+ # A paging anchor must not trigger a second request: the endpoint takes no anchor param.
+ _response({"data": [{"slug": "r1"}], "paging": {"next": "ignored"}}),
+ _response({"data": [{"slug": "r2"}, {"slug": "r3"}]}),
+ # An app deleted between enumeration and its fetch must not fail the sync.
+ _response({"message": "Not Found"}, status_code=404),
+ ],
+ )
+
+ assert result.raised is None
+ assert [(row["slug"], row["app_slug"]) for row in result.rows] == [
+ ("r1", "app1"),
+ ("r2", "app2"),
+ ("r3", "app2"),
+ ]
+ assert result.paths[1:] == [path.format(app=app) for app in ("app1", "app2", "gone")]
+ assert [request.param("limit") for request in result.requests[1:]] == [expected_limit] * 3
+ assert all(request.param("app_slug") is None for request in result.requests[1:])
+
+
+class TestAvailableStacks:
+ @pytest.mark.parametrize(
+ "body, expected_rows",
+ [
+ (
+ {
+ "linux-docker-android-22.04": {
+ "id": "linux-docker-android-22.04",
+ "os": "linux",
+ "status": "stable",
+ },
+ "osx-xcode-16.0.x": {"id": "osx-xcode-16.0.x", "os": "macos", "removal-date": "2027-01-01"},
+ },
+ [
+ {"id": "linux-docker-android-22.04", "os": "linux", "status": "stable"},
+ {"id": "osx-xcode-16.0.x", "os": "macos", "removal-date": "2027-01-01"},
+ ],
+ ),
+ # The map key stands in for a missing or empty `id`.
+ (
+ {"osx-xcode-edge": {"title": "Xcode edge"}, "linux-edge": {"id": "", "os": "linux"}},
+ [{"title": "Xcode edge", "id": "osx-xcode-edge"}, {"id": "linux-edge", "os": "linux"}],
+ ),
+ # Non-object values are not stacks.
+ ({"osx-xcode-16.0.x": {"id": "osx-xcode-16.0.x"}, "note": "x"}, [{"id": "osx-xcode-16.0.x"}]),
+ ({}, []),
+ ],
+ )
+ def test_explodes_stack_map_into_rows(self, body: dict[str, Any], expected_rows: list[dict[str, Any]]) -> None:
+ result = _driver().run("available_stacks", [_response(body)])
+
+ assert result.raised is None
+ assert result.rows == expected_rows
+ assert result.paths == ["/v0.1/available-stacks"]
+
+
class TestOrganizationMembers:
def test_fans_out_over_organizations_and_injects_org_slug(self) -> None:
result = _driver().run(