Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(warehouse-sources): read ynab month categories from the plan export (#121404)

posthog[bot]6daee097310d
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/ynab/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/ynab/settings.py
index b7c36740cde1..b8b2fbc73c96 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/ynab/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/ynab/settings.py
@@ -19,8 +19,8 @@ class YnabEndpoint:
default_incremental_field: str | None = None
page_size: int = 0
fanout: DependentEndpointConfig | None = None
- # Binds {month} to each month in the plan's first_month..last_month range.
- per_month: bool = False
+ # Emits one row per category in each month of the plan export's `months`.
+ categories_by_month: bool = False
PLAN_FANOUT = DependentEndpointConfig(
@@ -67,20 +67,14 @@ ENDPOINTS: dict[str, YnabEndpoint] = {
),
"month_categories": YnabEndpoint(
name="month_categories",
- path="plans/{plan_id}/months/{month}",
- data_selector="data.month.categories",
+ # The plan export carries every month's categories in one request. A request per month
+ # spends YNAB's hourly per-token limit on a plan with a few years of history.
+ path="plans/{plan_id}",
+ data_selector="data.plan.months",
primary_keys=("plan_id", "month", "id"),
partition_key="month",
- per_month=True,
- fanout=DependentEndpointConfig(
- parent_name="plans",
- resolve_param="plan_id",
- resolve_field="id",
- include_from_parent=["id", "month"],
- parent_field_renames={"id": "plan_id", "month": "month"},
- # A month inside the plan's range that YNAB has no record of returns 404.
- child_response_actions=[{"status_code": 404, "action": "ignore"}],
- ),
+ categories_by_month=True,
+ fanout=PLAN_FANOUT,
),
"money_movements": YnabEndpoint(
name="money_movements",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/ynab/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/ynab/source.py
index 17672a89f4d4..715f6209e360 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/ynab/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/ynab/source.py
@@ -25,6 +25,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.ynab.setti
from products.warehouse_sources.backend.temporal.data_imports.sources.ynab.ynab import (
AUTH_ERROR,
PERMISSION_ERROR,
+ RATE_LIMIT_ERROR,
YnabResumeConfig,
validate_credentials,
ynab_source,
@@ -119,6 +120,11 @@ class YnabSource(ResumableSource[YnabSourceConfig, YnabResumeConfig]):
"403 Client Error: Forbidden for url: https://api.ynab.com/": PERMISSION_ERROR,
}
+ def get_retry_exhausted_errors(self) -> dict[str, str]:
+ # The limit is per token over a rolling hour, so the activity's own retries, seconds apart,
+ # cannot outlast it. Every table in the source shares that one token.
+ return {"HTTP 429 for https://api.ynab.com/": RATE_LIMIT_ERROR}
+
def get_canonical_descriptions(self) -> CanonicalDescriptions:
from products.warehouse_sources.backend.temporal.data_imports.sources.ynab.canonical_descriptions import (
CANONICAL_DESCRIPTIONS,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/ynab/tests/test_ynab.py b/products/warehouse_sources/backend/temporal/data_imports/sources/ynab/tests/test_ynab.py
index e9813fd40078..eebf36c65c5e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/ynab/tests/test_ynab.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/ynab/tests/test_ynab.py
@@ -162,88 +162,63 @@ def test_plans_list_and_empty_account(driver: SourceDriver) -> None:
assert accounts.saved_states == []
-def _month_detail(categories: list[dict[str, Any]]) -> ScriptedResponse:
- return ScriptedResponse(json={"data": {"month": {"categories": categories}}})
+def _plan_export(months: list[dict[str, Any]]) -> ScriptedResponse:
+ return ScriptedResponse(json={"data": {"plan": {"id": "unused", "months": months}, "server_knowledge": 42}})
@pytest.mark.parametrize(
- "plan,resume_state,expected_rows",
+ "resume_state,expected_rows",
[
(
- {"id": "plan-1", "first_month": "2023-11-01", "last_month": "2024-01-01"},
None,
- [("plan-1", "2023-11-01"), ("plan-1", "2023-12-01"), ("plan-1", "2024-01-01")],
+ [("plan-1", "2023-12-01", "category-1"), ("plan-1", "2024-01-01", "category-1")],
),
(
- {"id": "plan-1", "first_month": "2023-11-01", "last_month": "2024-01-01"},
- YnabResumeConfig(completed=["plans/plan-1/months/2023-11-01"]),
- [("plan-1", "2023-12-01"), ("plan-1", "2024-01-01")],
+ YnabResumeConfig(completed=["plans/plan-1"]),
+ [],
),
- ({"id": "plan-1", "first_month": "2024-01-01", "last_month": "2024-01-01"}, None, [("plan-1", "2024-01-01")]),
- ({"id": "plan-1", "first_month": None, "last_month": None}, None, []),
- ({"id": "plan-1"}, None, []),
],
)
-def test_month_categories_cover_each_plan_month(
- plan: dict[str, Any],
+def test_month_categories_take_one_request_per_plan(
resume_state: YnabResumeConfig | None,
- expected_rows: list[tuple[str, str]],
+ expected_rows: list[tuple[str, str, str]],
driver: SourceDriver,
) -> None:
category = {"id": "category-1", "budgeted": 1000, "activity": -500, "balance": 500}
+ # Enough months that a request per month would spend YNAB's hourly limit of 200 on its own.
+ history = [
+ {"month": f"{year}-{month:02d}-01", "categories": []} for year in range(2005, 2023) for month in range(1, 13)
+ ]
result = driver.run(
"month_categories",
route(
{
- "/v1/plans": [ScriptedResponse(json={"data": {"plans": [plan, {"id": "plan-2"}]}})],
- **{
- f"/v1/plans/plan-1/months/{month}": [_month_detail([category])]
- for month in ["2023-11-01", "2023-12-01", "2024-01-01"]
- },
+ "/v1/plans": [ScriptedResponse(json={"data": {"plans": [{"id": "plan-1"}, {"id": "plan-2"}]}})],
+ "/v1/plans/plan-1": [
+ _plan_export(
+ [
+ {"month": "2023-12-01", "income": 0, "categories": [category]},
+ {"month": "2024-01-01", "income": 0, "categories": [category]},
+ ]
+ )
+ ],
+ "/v1/plans/plan-2": [_plan_export(history)],
}
),
resume_state=resume_state,
)
rows = [row for batch in result.items for row in batch]
- assert [(row["plan_id"], row["month"]) for row in rows] == expected_rows
- assert all(row["id"] == "category-1" and row["balance"] == 500 for row in rows)
+ assert [(row["plan_id"], row["month"], row["id"]) for row in rows] == expected_rows
+ assert all(row["balance"] == 500 and "categories" not in row for row in rows)
assert result.response is not None and result.response.primary_keys is not None
assert len({tuple(row[key] for key in result.response.primary_keys) for row in rows}) == len(rows)
- assert result.paths == ["/v1/plans", *[f"/v1/plans/plan-1/months/{month}" for _, month in expected_rows]]
- assert all(dict(request.query) == {} for request in result.requests)
- if expected_rows:
- assert result.saved_states[-1].completed[-1] == f"plans/plan-1/months/{expected_rows[-1][1]}"
-
-
-def test_month_categories_skip_missing_month(driver: SourceDriver) -> None:
- result = driver.run(
- "month_categories",
- route(
- {
- "/v1/plans": [
- ScriptedResponse(
- json={
- "data": {
- "plans": [{"id": "plan-1", "first_month": "2024-01-01", "last_month": "2024-02-01"}]
- }
- }
- )
- ],
- "/v1/plans/plan-1/months/2024-01-01": [
- ScriptedResponse(status=404, json={"error": {"id": "404.2", "name": "resource_not_found"}})
- ],
- "/v1/plans/plan-1/months/2024-02-01": [_month_detail([{"id": "category-1"}])],
- }
- ),
+ expected_paths = (
+ ["/v1/plans", "/v1/plans/plan-2"] if resume_state else ["/v1/plans", "/v1/plans/plan-1", "/v1/plans/plan-2"]
)
-
- assert result.raised is None
- assert result.items == [[{"id": "category-1", "plan_id": "plan-1", "month": "2024-02-01"}]]
- assert result.saved_states[-1].completed == [
- "plans/plan-1/months/2024-01-01",
- "plans/plan-1/months/2024-02-01",
- ]
+ assert result.paths == expected_paths
+ assert all(dict(request.query) == {} for request in result.requests)
+ assert result.saved_states[-1].completed == ["plans/plan-1", "plans/plan-2"]
@pytest.mark.parametrize("groups", [[], [{"id": "group-1", "categories": []}]])
@@ -305,9 +280,11 @@ def test_sync_errors_keep_auth_terminal_and_outages_retryable(status: int, drive
result = driver.run("plans", always(ScriptedResponse(status=status, json={"error": {"id": str(status)}})))
exception = RESTClientRetryableError if status in (429, 500) else HTTPError
assert isinstance(result.raised, exception)
- mappings = YnabSource().get_non_retryable_errors()
- matched = [message for pattern, message in mappings.items() if pattern in str(result.raised)]
- assert bool(matched) is (status in (401, 403))
+ source = YnabSource()
+ terminal = [pattern for pattern in source.get_non_retryable_errors() if pattern in str(result.raised)]
+ assert bool(terminal) is (status in (401, 403))
+ exhausted = [pattern for pattern in source.get_retry_exhausted_errors() if pattern in str(result.raised)]
+ assert bool(exhausted) is (status == 429)
assert result.saved_states == []
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/ynab/ynab.py b/products/warehouse_sources/backend/temporal/data_imports/sources/ynab/ynab.py
index eebeeff64f4b..ce0cf6fae976 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/ynab/ynab.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/ynab/ynab.py
@@ -1,5 +1,4 @@
-from datetime import date
-from typing import Any
+from typing import Any, cast
from requests.exceptions import HTTPError
@@ -16,6 +15,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
SinglePagePaginator,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.resource import Resource
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.rest_client import RESTClient
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import ClientConfig
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
@@ -25,6 +25,11 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.ynab.setti
AUTH_ERROR = "Your YNAB personal access token is invalid or expired. Create a new token and reconnect."
PERMISSION_ERROR = "Your YNAB token cannot access this resource. Check its permissions and reconnect."
+RATE_LIMIT_ERROR = (
+ "YNAB's request limit for your personal access token ran out before this sync finished. "
+ "YNAB allows 200 requests per hour for each token, shared by every table in this source and any other app "
+ "that uses the token. Set these tables to sync less often, such as every 6 hours, or turn off tables you don't need."
+)
@frozen
@@ -32,18 +37,10 @@ class YnabResumeConfig:
completed: list[str]
-def _plan_months(plan: dict[str, Any]) -> list[dict[str, Any]]:
- if not plan.get("first_month") or not plan.get("last_month"):
- return []
- month = date.fromisoformat(plan["first_month"]).replace(day=1)
- last_month = date.fromisoformat(plan["last_month"]).replace(day=1)
- months: list[dict[str, Any]] = []
- while month <= last_month:
- months.append({"id": plan["id"], "month": month.isoformat()})
- month = (
- month.replace(year=month.year + 1, month=1) if month.month == 12 else month.replace(month=month.month + 1)
- )
- return months
+def _month_category_rows(month: dict[str, Any]) -> list[dict[str, Any]]:
+ return [
+ {**category, "plan_id": month["plan_id"], "month": month["month"]} for category in month.get("categories") or []
+ ]
def validate_credentials(api_key: str, api_version: str, schema_name: str | None) -> tuple[bool, str | None]:
@@ -104,15 +101,11 @@ def ynab_source(
# Wildcard selectors have no matches when every category group is empty.
"data_selector_required": endpoint != "categories",
},
- child_params_extra=(
- {"month": {"type": "resolve", "resource": "plans", "field": "month"}}
- if endpoint_config.per_month
- else None
- ),
- parent_data_map=_plan_months if endpoint_config.per_month else None,
resume_hook=save_checkpoint,
initial_paginator_state={"completed": resume.completed} if resume else None,
)
+ if endpoint_config.categories_by_month:
+ resource = cast(Resource, resource).add_map(_month_category_rows)
else:
config: RESTAPIConfig = {
"client": client_config,