:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add 4 missing tables to the Bitrise source (#119164)
Tom Owers2b98355654b3
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex d81f4b4a58a4..bd47e247fd25 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -1667,7 +1667,7 @@ pull request in one walk, so it costs one request per page instead of one per pu## Bitrise — gaps-Today (8): `apps`, `artifacts`, `branches`, `builds`, `organization_members`, `organizations`, `pipelines`, `workflows`+Today (12): `activities`, `apps`, `archived_builds`, `artifacts`, `available_stacks`, `branches`, `build_requests`, `builds`, `organization_members`, `organizations`, `pipelines`, `workflows`Diffed against: <https://api-docs.bitrise.io/docs/swagger.json>@@ -1675,10 +1675,10 @@ Diffed against: <https://api-docs.bitrise.io/docs/swagger.json>- [x] `/organizations/{org-slug}/members` — org membership records for attributing builds to people (medium)- [x] `/organizations` — lookup resolving the org that owns each app we already sync (medium)- [x] `/apps/{app-slug}/branches` — branch lookup for grouping build history (medium)-- [ ] `/apps/{app-slug}/archived-builds` — extends build history past the live builds window (medium)-- [ ] `/apps/{app-slug}/build-requests` — queued/pending build requests reveal CI backlog (low)-- [ ] `/me/activities` — account-level activity event feed (low)-- [ ] `/available-stacks` — lookup for the stack identifier carried on builds (low)+- [x] `/apps/{app-slug}/archived-builds` — extends build history past the live builds window (medium)+- [x] `/apps/{app-slug}/build-requests` — queued/pending build requests reveal CI backlog (low)+- [x] `/me/activities` — account-level activity event feed (low)+- [x] `/available-stacks` — lookup for the stack identifier carried on builds (low)## BlandAI — **thin**diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bitrise/bitrise.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bitrise/bitrise.pyindex ab9615043065..7e9ea974a8d1 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bitrise/bitrise.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bitrise/bitrise.py@@ -29,6 +29,8 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.typBITRISE_BASE_URL = "https://api.bitrise.io/v0.1"# Bitrise list endpoints cap `limit` at 50.PAGE_LIMIT = 50+# The archived builds endpoint takes no paging anchor and returns at most 1000 builds per app.+ARCHIVED_BUILDS_LIMIT = 1000# Safety overlap subtracted from the incremental watermark: builds that were still running (or on# hold) at the last sync keep mutating (status, finished_at) after their trigger time, so each run# re-pulls a trailing window and merge dedupes the re-fetched rows on the primary key.@@ -40,11 +42,12 @@ class BitriseResumeConfig:# Fan-out bookmark: retained only so pre-migration saved state still parses via# `dataclass(**saved)`. New runs never populate it; fan-out resume now lives in `fanout_state`.app_slug: str | None = None- # `next` paging anchor for the flat apps listing. None means "start at the first page".+ # `next` paging anchor for the flat apps and activities listings. None means "start at the+ # first page".next: str | None = None# Framework fan-out resume snapshot ({"completed": [...], "current": ..., "child_state": ...})# for the single-hop parent-app -> child-endpoint dependent resources (builds, pipelines,- # workflows, branches).+ # workflows, branches, archived builds, build requests).fanout_state: dict[str, Any] | None = None@@ -132,6 +135,18 @@ def _apps_resource() -> EndpointResource:}+def _activities_resource() -> EndpointResource:+ return {+ "name": "activities",+ "endpoint": {+ "path": "/me/activities",+ "params": {"limit": PAGE_LIMIT},+ "data_selector": "data",+ "paginator": _cursor_paginator(),+ },+ }++def _organizations_resource() -> EndpointResource:# The organizations listing takes no params and returns every org in one body.return {@@ -155,15 +170,16 @@ def _builds_child_params(after: int | None) -> dict[str, Any]:return params-def _apps_source(+def _flat_cursor_source(api_token: str,team_id: int,job_id: str,resumable_source_manager: ResumableSourceManager[BitriseResumeConfig],+ resource: EndpointResource,) -> Resource:rest_config: RESTAPIConfig = {"client": _client_config(api_token),- "resources": [_apps_resource()],+ "resources": [resource],}initial_paginator_state: Optional[dict[str, Any]] = None@@ -321,6 +337,58 @@ def _app_name_list_source(return _single_hop_fanout_source(api_token, team_id, job_id, resumable_source_manager, child_resource)+def _single_page_app_child_source(+ api_token: str,+ team_id: int,+ job_id: str,+ resumable_source_manager: ResumableSourceManager[BitriseResumeConfig],+ name: str,+ path: str,+ extra_params: dict[str, Any],+) -> Resource:+ params: dict[str, Any] = {"app_slug": {"type": "resolve", "resource": "apps", "field": "slug"}, **extra_params}+ child_resource: EndpointResource = {+ "name": name,+ "include_from_parent": ["slug"],+ "endpoint": {+ "path": path,+ "params": params,+ "data_selector": "data",+ "paginator": SinglePagePaginator(),+ "response_actions": [{"status_code": 404, "action": "ignore"}],+ },+ "data_map": rename_parent_fields("apps", {"slug": "app_slug"}),+ }+ return _single_hop_fanout_source(api_token, team_id, job_id, resumable_source_manager, child_resource)+++def _explode_available_stacks(row: dict[str, Any]) -> list[dict[str, Any]]:+ """Explode the `{"<stack id>": {...}}` map into one row per stack, keeping the map key as `id`."""+ return [{**stack, "id": stack.get("id") or stack_id} for stack_id, stack in row.items() if isinstance(stack, dict)]+++def _available_stacks_source(+ api_token: str,+ team_id: int,+ job_id: str,+) -> Resource:+ resource: EndpointResource = {+ "name": "available_stacks",+ "endpoint": {+ "path": "/available-stacks",+ # The body is a bare map keyed by stack id, so take it whole and let data_map explode it.+ "data_selector": "$",+ "paginator": SinglePagePaginator(),+ },+ "data_map": _explode_available_stacks,+ }+ rest_config: RESTAPIConfig = {+ "client": _client_config(api_token),+ "resources": [resource],+ }+ return rest_api_resource(rest_config, team_id, job_id, None)++def _organizations_source(api_token: str,team_id: int,@@ -462,7 +530,7 @@ def bitrise_source(after = _build_after_param(should_use_incremental_field, db_incremental_field_last_value)if endpoint == "apps":- resource = _apps_source(api_token, team_id, job_id, resumable_source_manager)+ resource = _flat_cursor_source(api_token, team_id, job_id, resumable_source_manager, _apps_resource())elif endpoint == "builds":resource = _builds_source(api_token, team_id, job_id, resumable_source_manager, after)elif endpoint == "pipelines":@@ -483,6 +551,30 @@ def bitrise_source(resource = _organization_members_source(api_token, team_id, job_id)elif endpoint == "artifacts":resource = _artifacts_source(api_token, team_id, job_id, after)+ elif endpoint == "archived_builds":+ resource = _single_page_app_child_source(+ api_token,+ team_id,+ job_id,+ resumable_source_manager,+ name="archived_builds",+ path="/apps/{app_slug}/archived-builds",+ extra_params={"limit": ARCHIVED_BUILDS_LIMIT},+ )+ elif endpoint == "build_requests":+ resource = _single_page_app_child_source(+ api_token,+ team_id,+ job_id,+ resumable_source_manager,+ name="build_requests",+ path="/apps/{app_slug}/build-requests",+ extra_params={},+ )+ elif endpoint == "activities":+ resource = _flat_cursor_source(api_token, team_id, job_id, resumable_source_manager, _activities_resource())+ elif endpoint == "available_stacks":+ resource = _available_stacks_source(api_token, team_id, job_id)else:raise ValueError(f"Unknown Bitrise endpoint: {endpoint}")diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bitrise/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bitrise/canonical_descriptions.pyindex 16a3cbbc0378..1530a6549b61 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bitrise/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bitrise/canonical_descriptions.py@@ -136,4 +136,68 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"is_public_page_enabled": "Whether a public install page is enabled for the artifact.",},},+ "archived_builds": {+ "description": "A build that has moved out of the live builds listing into the archive, one row per app and build. "+ "Bitrise returns at most 1000 archived builds per app.",+ "docs_url": "https://api-docs.bitrise.io/#/builds",+ "columns": {+ "app_slug": "Identifier of the app the build belongs to (added by PostHog during sync).",+ "slug": "Unique identifier of the build.",+ "build_number": "Sequential build number within the app.",+ "status": "Numeric build status (0 running, 1 successful, 2 failed, 3 aborted with failure, 4 aborted with success).",+ "status_text": "Human-readable build status (e.g. success, error, aborted).",+ "abort_reason": "Reason the build was aborted, if it was.",+ "branch": "Git branch the build ran on.",+ "tag": "Git tag that triggered the build, if any.",+ "commit_hash": "Hash of the commit the build ran against.",+ "commit_message": "Message of the commit the build ran against.",+ "triggered_workflow": "Name of the Bitrise workflow the build ran.",+ "triggered_by": "What triggered the build (e.g. a webhook, a schedule, or a manual start).",+ "started_on_worker_at": "When the build started running on a worker.",+ "finished_at": "When the build finished.",+ "stack_identifier": "Identifier of the stack the build ran on.",+ "machine_type_id": "Identifier of the machine type the build ran on.",+ "credit_cost": "Credits the build consumed.",+ "build_artifacts": "Artifacts the build produced.",+ "original_build_params": "Parameters the build was started with.",+ },+ },+ "build_requests": {+ "description": "An open build request of an app that waits for approval before it starts a build.",+ "docs_url": "https://api-docs.bitrise.io/#/build-request",+ "columns": {+ "app_slug": "Identifier of the app the build request belongs to (added by PostHog during sync).",+ "slug": "Unique identifier of the build request.",+ "created_at": "When the build request was created.",+ "build_params": "Parameters the build will start with.",+ "pull_request_url": "URL of the pull request that created the build request.",+ },+ },+ "activities": {+ "description": "An activity event of the Bitrise account that owns the API token.",+ "docs_url": "https://api-docs.bitrise.io/#/activity",+ "columns": {+ "slug": "Unique identifier of the activity event.",+ "created_at": "When the activity event happened.",+ "title": "Title of the activity event.",+ "description": "Description of the activity event.",+ "event_stype": "Type of the activity event.",+ "repository_title": "Title of the app the activity event relates to.",+ "target_path_string": "Path in the Bitrise web app that the activity event links to.",+ },+ },+ "available_stacks": {+ "description": "A build stack (operating system image with preinstalled tools) that builds can run on.",+ "docs_url": "https://api-docs.bitrise.io/#/configuration",+ "columns": {+ "id": "Unique identifier of the stack, as carried on builds in stack_identifier.",+ "title": "Human-readable title of the stack.",+ "description": "Description of the stack.",+ "os": "Operating system of the stack (linux or macos).",+ "os_version": "Version number of the operating system.",+ "status": "Current status of the stack (stable, edge, or frozen).",+ "removal_date": "Date on which the stack stops being available. Usually set only for frozen stacks.",+ "stack_report": "URL of the report that lists the preinstalled tools of the stack.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bitrise/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bitrise/settings.pyindex 2ddf4f88a02d..68fc29440e6d 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bitrise/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bitrise/settings.py@@ -17,8 +17,9 @@ class BitriseEndpointConfig:# the given time), which gives an honest incremental sync on `triggered_at`. The pipelines endpoint# takes the same filter as an RFC3339 date. Artifacts inherit the builds filter through their parent# build fan-out, keyed on the injected `build_triggered_at`. Apps, workflows, branches,-# organizations and organization members are small listings with no server-side time filter, so-# they stay full refresh.+# organizations, organization members, build requests, activities and stacks have no server-side+# time filter, so they stay full refresh. Archived builds take the same `after` filter as builds, but+# their rows carry no `triggered_at` to watermark on, so they stay full refresh too.BITRISE_ENDPOINTS: dict[str, BitriseEndpointConfig] = {"apps": BitriseEndpointConfig(name="apps",@@ -85,6 +86,26 @@ BITRISE_ENDPOINTS: dict[str, BitriseEndpointConfig] = {# A member slug identifies the Bitrise account, which can belong to several organizations.primary_keys=["org_slug", "slug"],),+ "archived_builds": BitriseEndpointConfig(+ name="archived_builds",+ primary_keys=["app_slug", "slug"],+ partition_key="started_on_worker_at",+ ),+ "build_requests": BitriseEndpointConfig(+ name="build_requests",+ primary_keys=["app_slug", "slug"],+ partition_key="created_at",+ ),+ "activities": BitriseEndpointConfig(+ name="activities",+ partition_key="created_at",+ # A full refresh pages through the whole account activity history on every run.+ should_sync_default=False,+ ),+ "available_stacks": BitriseEndpointConfig(+ name="available_stacks",+ primary_keys=["id"],+ ),}ENDPOINTS = tuple(BITRISE_ENDPOINTS.keys())diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bitrise/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bitrise/source.pyindex d3ecb3a6ede0..f8717bf133d9 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bitrise/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bitrise/source.py@@ -105,6 +105,12 @@ You can create a personal access token in your [Bitrise security settings](https)if endpoint == "branches":return "Only branches that already have builds on Bitrise are listed"+ if endpoint == "archived_builds":+ return "The Bitrise API returns at most 1000 archived builds per app"+ if endpoint == "build_requests":+ return "Only open build requests are listed"+ if endpoint == "activities":+ return "Activity events of the account that owns the API token. Disabled by default because every sync reads the full history"return Nonedef _build_schema(endpoint: str) -> SourceSchema:diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bitrise/tests/test_bitrise.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bitrise/tests/test_bitrise.pyindex 868f75ab9528..c5dd3c2c0678 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bitrise/tests/test_bitrise.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bitrise/tests/test_bitrise.py@@ -92,18 +92,27 @@ class TestValidateCredentials:assert {request.path for request in network.requests_log} == {"/v0.1/me"}-class TestApps:- def test_paginates_via_next_anchor_and_saves_state(self) -> None:+class TestFlatCursorListings:+ @pytest.mark.parametrize(+ "endpoint, path",+ [+ ("apps", "/v0.1/apps"),+ ("activities", "/v0.1/me/activities"),+ ],+ )+ def test_paginates_via_next_anchor_and_saves_state(self, endpoint: str, path: str) -> None:result = _driver().run(- "apps",+ endpoint,[- _response({"data": [{"slug": "app1"}], "paging": {"next": "anchor1"}}),- _response({"data": [{"slug": "app2"}], "paging": {}}),+ _response({"data": [{"slug": "row1"}], "paging": {"next": "anchor1"}}),+ _response({"data": [{"slug": "row2"}], "paging": {}}),],)assert result.raised is None- assert [row["slug"] for row in result.rows] == ["app1", "app2"]+ assert [row["slug"] for row in result.rows] == ["row1", "row2"]+ assert result.paths == [path, path]+ assert result.params("limit") == ["50", "50"]# The second page carries the anchor returned by the first.assert result.params("next") == [None, "anchor1"]# State saved once, after the first page yielded and only while more pages remain.@@ -210,6 +219,76 @@ class TestBranches:assert result.paths[1] == "/v0.1/apps/app1/branches"+class TestSinglePageAppChildren:+ @pytest.mark.parametrize(+ "endpoint, path, expected_limit",+ [+ ("archived_builds", "/v0.1/apps/{app}/archived-builds", "1000"),+ ("build_requests", "/v0.1/apps/{app}/build-requests", None),+ ],+ )+ def test_fans_out_over_apps_with_one_request_per_app(+ self, endpoint: str, path: str, expected_limit: str | None+ ) -> None:+ result = _driver().run(+ endpoint,+ [+ _response({"data": [{"slug": "app1"}, {"slug": "app2"}, {"slug": "gone"}], "paging": {}}),+ # A paging anchor must not trigger a second request: the endpoint takes no anchor param.+ _response({"data": [{"slug": "r1"}], "paging": {"next": "ignored"}}),+ _response({"data": [{"slug": "r2"}, {"slug": "r3"}]}),+ # An app deleted between enumeration and its fetch must not fail the sync.+ _response({"message": "Not Found"}, status_code=404),+ ],+ )++ assert result.raised is None+ assert [(row["slug"], row["app_slug"]) for row in result.rows] == [+ ("r1", "app1"),+ ("r2", "app2"),+ ("r3", "app2"),+ ]+ assert result.paths[1:] == [path.format(app=app) for app in ("app1", "app2", "gone")]+ assert [request.param("limit") for request in result.requests[1:]] == [expected_limit] * 3+ assert all(request.param("app_slug") is None for request in result.requests[1:])+++class TestAvailableStacks:+ @pytest.mark.parametrize(+ "body, expected_rows",+ [+ (+ {+ "linux-docker-android-22.04": {+ "id": "linux-docker-android-22.04",+ "os": "linux",+ "status": "stable",+ },+ "osx-xcode-16.0.x": {"id": "osx-xcode-16.0.x", "os": "macos", "removal-date": "2027-01-01"},+ },+ [+ {"id": "linux-docker-android-22.04", "os": "linux", "status": "stable"},+ {"id": "osx-xcode-16.0.x", "os": "macos", "removal-date": "2027-01-01"},+ ],+ ),+ # The map key stands in for a missing or empty `id`.+ (+ {"osx-xcode-edge": {"title": "Xcode edge"}, "linux-edge": {"id": "", "os": "linux"}},+ [{"title": "Xcode edge", "id": "osx-xcode-edge"}, {"id": "linux-edge", "os": "linux"}],+ ),+ # Non-object values are not stacks.+ ({"osx-xcode-16.0.x": {"id": "osx-xcode-16.0.x"}, "note": "x"}, [{"id": "osx-xcode-16.0.x"}]),+ ({}, []),+ ],+ )+ def test_explodes_stack_map_into_rows(self, body: dict[str, Any], expected_rows: list[dict[str, Any]]) -> None:+ result = _driver().run("available_stacks", [_response(body)])++ assert result.raised is None+ assert result.rows == expected_rows+ assert result.paths == ["/v0.1/available-stacks"]++class TestOrganizationMembers:def test_fans_out_over_organizations_and_injects_org_slug(self) -> None:result = _driver().run(