:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(warehouse-sources): read ynab month categories from the plan export (#121404)
posthog[bot]6daee097310d
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/ynab/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/ynab/settings.pyindex b7c36740cde1..b8b2fbc73c96 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/ynab/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/ynab/settings.py@@ -19,8 +19,8 @@ class YnabEndpoint:default_incremental_field: str | None = Nonepage_size: int = 0fanout: DependentEndpointConfig | None = None- # Binds {month} to each month in the plan's first_month..last_month range.- per_month: bool = False+ # Emits one row per category in each month of the plan export's `months`.+ categories_by_month: bool = FalsePLAN_FANOUT = DependentEndpointConfig(@@ -67,20 +67,14 @@ ENDPOINTS: dict[str, YnabEndpoint] = {),"month_categories": YnabEndpoint(name="month_categories",- path="plans/{plan_id}/months/{month}",- data_selector="data.month.categories",+ # The plan export carries every month's categories in one request. A request per month+ # spends YNAB's hourly per-token limit on a plan with a few years of history.+ path="plans/{plan_id}",+ data_selector="data.plan.months",primary_keys=("plan_id", "month", "id"),partition_key="month",- per_month=True,- fanout=DependentEndpointConfig(- parent_name="plans",- resolve_param="plan_id",- resolve_field="id",- include_from_parent=["id", "month"],- parent_field_renames={"id": "plan_id", "month": "month"},- # A month inside the plan's range that YNAB has no record of returns 404.- child_response_actions=[{"status_code": 404, "action": "ignore"}],- ),+ categories_by_month=True,+ fanout=PLAN_FANOUT,),"money_movements": YnabEndpoint(name="money_movements",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/ynab/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/ynab/source.pyindex 17672a89f4d4..715f6209e360 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/ynab/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/ynab/source.py@@ -25,6 +25,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.ynab.settifrom products.warehouse_sources.backend.temporal.data_imports.sources.ynab.ynab import (AUTH_ERROR,PERMISSION_ERROR,+ RATE_LIMIT_ERROR,YnabResumeConfig,validate_credentials,ynab_source,@@ -119,6 +120,11 @@ class YnabSource(ResumableSource[YnabSourceConfig, YnabResumeConfig]):"403 Client Error: Forbidden for url: https://api.ynab.com/": PERMISSION_ERROR,}+ def get_retry_exhausted_errors(self) -> dict[str, str]:+ # The limit is per token over a rolling hour, so the activity's own retries, seconds apart,+ # cannot outlast it. Every table in the source shares that one token.+ return {"HTTP 429 for https://api.ynab.com/": RATE_LIMIT_ERROR}+def get_canonical_descriptions(self) -> CanonicalDescriptions:from products.warehouse_sources.backend.temporal.data_imports.sources.ynab.canonical_descriptions import (CANONICAL_DESCRIPTIONS,diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/ynab/tests/test_ynab.py b/products/warehouse_sources/backend/temporal/data_imports/sources/ynab/tests/test_ynab.pyindex e9813fd40078..eebf36c65c5e 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/ynab/tests/test_ynab.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/ynab/tests/test_ynab.py@@ -162,88 +162,63 @@ def test_plans_list_and_empty_account(driver: SourceDriver) -> None:assert accounts.saved_states == []-def _month_detail(categories: list[dict[str, Any]]) -> ScriptedResponse:- return ScriptedResponse(json={"data": {"month": {"categories": categories}}})+def _plan_export(months: list[dict[str, Any]]) -> ScriptedResponse:+ return ScriptedResponse(json={"data": {"plan": {"id": "unused", "months": months}, "server_knowledge": 42}})@pytest.mark.parametrize(- "plan,resume_state,expected_rows",+ "resume_state,expected_rows",[(- {"id": "plan-1", "first_month": "2023-11-01", "last_month": "2024-01-01"},None,- [("plan-1", "2023-11-01"), ("plan-1", "2023-12-01"), ("plan-1", "2024-01-01")],+ [("plan-1", "2023-12-01", "category-1"), ("plan-1", "2024-01-01", "category-1")],),(- {"id": "plan-1", "first_month": "2023-11-01", "last_month": "2024-01-01"},- YnabResumeConfig(completed=["plans/plan-1/months/2023-11-01"]),- [("plan-1", "2023-12-01"), ("plan-1", "2024-01-01")],+ YnabResumeConfig(completed=["plans/plan-1"]),+ [],),- ({"id": "plan-1", "first_month": "2024-01-01", "last_month": "2024-01-01"}, None, [("plan-1", "2024-01-01")]),- ({"id": "plan-1", "first_month": None, "last_month": None}, None, []),- ({"id": "plan-1"}, None, []),],)-def test_month_categories_cover_each_plan_month(- plan: dict[str, Any],+def test_month_categories_take_one_request_per_plan(resume_state: YnabResumeConfig | None,- expected_rows: list[tuple[str, str]],+ expected_rows: list[tuple[str, str, str]],driver: SourceDriver,) -> None:category = {"id": "category-1", "budgeted": 1000, "activity": -500, "balance": 500}+ # Enough months that a request per month would spend YNAB's hourly limit of 200 on its own.+ history = [+ {"month": f"{year}-{month:02d}-01", "categories": []} for year in range(2005, 2023) for month in range(1, 13)+ ]result = driver.run("month_categories",route({- "/v1/plans": [ScriptedResponse(json={"data": {"plans": [plan, {"id": "plan-2"}]}})],- **{- f"/v1/plans/plan-1/months/{month}": [_month_detail([category])]- for month in ["2023-11-01", "2023-12-01", "2024-01-01"]- },+ "/v1/plans": [ScriptedResponse(json={"data": {"plans": [{"id": "plan-1"}, {"id": "plan-2"}]}})],+ "/v1/plans/plan-1": [+ _plan_export(+ [+ {"month": "2023-12-01", "income": 0, "categories": [category]},+ {"month": "2024-01-01", "income": 0, "categories": [category]},+ ]+ )+ ],+ "/v1/plans/plan-2": [_plan_export(history)],}),resume_state=resume_state,)rows = [row for batch in result.items for row in batch]- assert [(row["plan_id"], row["month"]) for row in rows] == expected_rows- assert all(row["id"] == "category-1" and row["balance"] == 500 for row in rows)+ assert [(row["plan_id"], row["month"], row["id"]) for row in rows] == expected_rows+ assert all(row["balance"] == 500 and "categories" not in row for row in rows)assert result.response is not None and result.response.primary_keys is not Noneassert len({tuple(row[key] for key in result.response.primary_keys) for row in rows}) == len(rows)- assert result.paths == ["/v1/plans", *[f"/v1/plans/plan-1/months/{month}" for _, month in expected_rows]]- assert all(dict(request.query) == {} for request in result.requests)- if expected_rows:- assert result.saved_states[-1].completed[-1] == f"plans/plan-1/months/{expected_rows[-1][1]}"---def test_month_categories_skip_missing_month(driver: SourceDriver) -> None:- result = driver.run(- "month_categories",- route(- {- "/v1/plans": [- ScriptedResponse(- json={- "data": {- "plans": [{"id": "plan-1", "first_month": "2024-01-01", "last_month": "2024-02-01"}]- }- }- )- ],- "/v1/plans/plan-1/months/2024-01-01": [- ScriptedResponse(status=404, json={"error": {"id": "404.2", "name": "resource_not_found"}})- ],- "/v1/plans/plan-1/months/2024-02-01": [_month_detail([{"id": "category-1"}])],- }- ),+ expected_paths = (+ ["/v1/plans", "/v1/plans/plan-2"] if resume_state else ["/v1/plans", "/v1/plans/plan-1", "/v1/plans/plan-2"])-- assert result.raised is None- assert result.items == [[{"id": "category-1", "plan_id": "plan-1", "month": "2024-02-01"}]]- assert result.saved_states[-1].completed == [- "plans/plan-1/months/2024-01-01",- "plans/plan-1/months/2024-02-01",- ]+ assert result.paths == expected_paths+ assert all(dict(request.query) == {} for request in result.requests)+ assert result.saved_states[-1].completed == ["plans/plan-1", "plans/plan-2"]@pytest.mark.parametrize("groups", [[], [{"id": "group-1", "categories": []}]])@@ -305,9 +280,11 @@ def test_sync_errors_keep_auth_terminal_and_outages_retryable(status: int, driveresult = driver.run("plans", always(ScriptedResponse(status=status, json={"error": {"id": str(status)}})))exception = RESTClientRetryableError if status in (429, 500) else HTTPErrorassert isinstance(result.raised, exception)- mappings = YnabSource().get_non_retryable_errors()- matched = [message for pattern, message in mappings.items() if pattern in str(result.raised)]- assert bool(matched) is (status in (401, 403))+ source = YnabSource()+ terminal = [pattern for pattern in source.get_non_retryable_errors() if pattern in str(result.raised)]+ assert bool(terminal) is (status in (401, 403))+ exhausted = [pattern for pattern in source.get_retry_exhausted_errors() if pattern in str(result.raised)]+ assert bool(exhausted) is (status == 429)assert result.saved_states == []diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/ynab/ynab.py b/products/warehouse_sources/backend/temporal/data_imports/sources/ynab/ynab.pyindex eebeeff64f4b..ce0cf6fae976 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/ynab/ynab.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/ynab/ynab.py@@ -1,5 +1,4 @@-from datetime import date-from typing import Any+from typing import Any, castfrom requests.exceptions import HTTPError@@ -16,6 +15,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.resfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (SinglePagePaginator,)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.resource import Resourcefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.rest_client import RESTClientfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import ClientConfigfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager@@ -25,6 +25,11 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.ynab.settiAUTH_ERROR = "Your YNAB personal access token is invalid or expired. Create a new token and reconnect."PERMISSION_ERROR = "Your YNAB token cannot access this resource. Check its permissions and reconnect."+RATE_LIMIT_ERROR = (+ "YNAB's request limit for your personal access token ran out before this sync finished. "+ "YNAB allows 200 requests per hour for each token, shared by every table in this source and any other app "+ "that uses the token. Set these tables to sync less often, such as every 6 hours, or turn off tables you don't need."+)@frozen@@ -32,18 +37,10 @@ class YnabResumeConfig:completed: list[str]-def _plan_months(plan: dict[str, Any]) -> list[dict[str, Any]]:- if not plan.get("first_month") or not plan.get("last_month"):- return []- month = date.fromisoformat(plan["first_month"]).replace(day=1)- last_month = date.fromisoformat(plan["last_month"]).replace(day=1)- months: list[dict[str, Any]] = []- while month <= last_month:- months.append({"id": plan["id"], "month": month.isoformat()})- month = (- month.replace(year=month.year + 1, month=1) if month.month == 12 else month.replace(month=month.month + 1)- )- return months+def _month_category_rows(month: dict[str, Any]) -> list[dict[str, Any]]:+ return [+ {**category, "plan_id": month["plan_id"], "month": month["month"]} for category in month.get("categories") or []+ ]def validate_credentials(api_key: str, api_version: str, schema_name: str | None) -> tuple[bool, str | None]:@@ -104,15 +101,11 @@ def ynab_source(# Wildcard selectors have no matches when every category group is empty."data_selector_required": endpoint != "categories",},- child_params_extra=(- {"month": {"type": "resolve", "resource": "plans", "field": "month"}}- if endpoint_config.per_month- else None- ),- parent_data_map=_plan_months if endpoint_config.per_month else None,resume_hook=save_checkpoint,initial_paginator_state={"completed": resume.completed} if resume else None,)+ if endpoint_config.categories_by_month:+ resource = cast(Resource, resource).add_map(_month_category_rows)else:config: RESTAPIConfig = {"client": client_config,