AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
feat(backend): send the onboarding role to MailerLite and PostHog, and keep Iran and Russia out of MailerLite (#15249)
Nicholas Tindle75e702072766
diff --git a/autogpt_platform/backend/backend/api/features/billing/subscriptions/routes.py b/autogpt_platform/backend/backend/api/features/billing/subscriptions/routes.pyindex 1430637aae..5fc5485df7 100644--- a/autogpt_platform/backend/backend/api/features/billing/subscriptions/routes.py+++ b/autogpt_platform/backend/backend/api/features/billing/subscriptions/routes.py@@ -841,10 +841,14 @@ async def stripe_webhook(request: Request):# `customer.subscription.created` fires at signup too; listening to# both would double-send.if event_type == "checkout.session.completed":- await _notify_checkout_completed(data_object)# The billing address is the strongest country signal, and- # it only exists once checkout completes. Never raises.+ # it only exists once checkout completes. First, so that an+ # Iranian or Russian one is on record before the trial notice+ # queues its MailerLite change. It raises only when such a+ # country could not be recorded, so Stripe retries the event+ # instead of the notice going out without it.await checkout_audience.record_checkout_completed(data_object)+ await _notify_checkout_completed(data_object)if event_type in ("customer.subscription.created",diff --git a/autogpt_platform/backend/backend/api/features/onboarding/profile_test.py b/autogpt_platform/backend/backend/api/features/onboarding/profile_test.pyindex a7b9686982..d60d27d96d 100644--- a/autogpt_platform/backend/backend/api/features/onboarding/profile_test.py+++ b/autogpt_platform/backend/backend/api/features/onboarding/profile_test.py@@ -1,15 +1,18 @@-from unittest.mock import AsyncMock+from unittest.mock import AsyncMock, MagicMockimport fastapiimport fastapi.testclientimport pytestfrom backend.api.features.onboarding.routes import router+from backend.data.onboarding_role import OnboardingRoleapp = fastapi.FastAPI()app.include_router(router)client = fastapi.testclient.TestClient(app)+ROUTES = "backend.api.features.onboarding.routes"+@pytest.fixture(autouse=True)def setup_app_auth(mock_jwt_user):@@ -20,37 +23,69 @@ def setup_app_auth(mock_jwt_user):app.dependency_overrides.clear()-def test_onboarding_profile_success(mocker):- mock_extract = mocker.patch(- "backend.api.features.onboarding.routes.extract_business_understanding",- new_callable=AsyncMock,- )- mock_upsert = mocker.patch(- "backend.api.features.onboarding.routes.upsert_business_understanding",- new_callable=AsyncMock,- )-+def profile(mocker):from backend.data.understanding import BusinessUnderstandingInput- mock_extract.return_value = BusinessUnderstandingInput.model_construct(- user_name="John",- user_role="Founder/CEO",- pain_points=["Finding leads"],- suggested_prompts={"Learn": ["How do I automate lead gen?"]},+ mocks = {+ name: mocker.patch(f"{ROUTES}.{name}", new_callable=AsyncMock)+ for name in (+ "extract_business_understanding",+ "upsert_business_understanding",+ "save_onboarding_role",+ "queue_onboarding_role",+ )+ }+ mocks["set_onboarding_role"] = mocker.patch(+ f"{ROUTES}.set_onboarding_role", new_callable=MagicMock+ )+ # The extraction's own idea of the role, which AutoPilot-side rewrites+ # look like; the kept pick comes from the request.+ mocks["extract_business_understanding"].return_value = (+ BusinessUnderstandingInput.model_construct(+ user_name="John",+ user_role="decision maker",+ pain_points=["Finding leads"],+ suggested_prompts={"Learn": ["How do I automate lead gen?"]},+ ))- mock_upsert.return_value = AsyncMock()+ return mocks- response = client.post(++def _submit(user_role: str):+ return client.post("/onboarding/profile",json={"user_name": "John",- "user_role": "Founder/CEO",+ "user_role": user_role,"pain_points": ["Finding leads", "Email & outreach"],},)+++def test_onboarding_profile_success(profile):+ response = _submit("Founder/CEO")assert response.status_code == 200- mock_extract.assert_awaited_once()- mock_upsert.assert_awaited_once()+ profile["extract_business_understanding"].assert_awaited_once()+ profile["upsert_business_understanding"].assert_awaited_once()+++ "user_role, role",+ [+ ("Founder/CEO", OnboardingRole(choice="Founder/CEO")),+ ("Dentist", OnboardingRole(choice="Other", other="Dentist")),+ ],+)+def test_the_pick_is_kept_and_sent_to_mailerlite_and_posthog(+ profile, test_user_id, user_role, role+):+ assert _submit(user_role).status_code == 200+ profile["save_onboarding_role"].assert_awaited_once_with(test_user_id, role)+ profile["queue_onboarding_role"].assert_awaited_once_with(test_user_id, role)+ profile["set_onboarding_role"].assert_called_once_with(+ user_id=test_user_id, role=role+ )def test_onboarding_profile_missing_fields():diff --git a/autogpt_platform/backend/backend/api/features/onboarding/routes.py b/autogpt_platform/backend/backend/api/features/onboarding/routes.pyindex c39c61703e..df529ce2bf 100644--- a/autogpt_platform/backend/backend/api/features/onboarding/routes.py+++ b/autogpt_platform/backend/backend/api/features/onboarding/routes.py@@ -17,11 +17,14 @@ from backend.data.onboarding import (reset_user_onboarding,update_user_onboarding,)+from backend.data.onboarding_audience import queue_onboarding_role+from backend.data.onboarding_role import OnboardingRole, save_onboarding_rolefrom backend.data.tally import extract_business_understandingfrom backend.data.understanding import (BusinessUnderstandingInput,upsert_business_understanding,)+from backend.util.product_analytics import set_onboarding_role# Tags stay per-route: /onboarding/completed publishes ["onboarding", "public"]# while the other six publish ["onboarding"].@@ -142,4 +145,10 @@ async def submit_onboarding_profile(await upsert_business_understanding(user_id, understanding_input)+ # Kept apart from the understanding, whose copy AutoPilot rewrites.+ role = OnboardingRole.from_answer(data.user_role)+ await save_onboarding_role(user_id, role)+ await queue_onboarding_role(user_id, role)+ set_onboarding_role(user_id=user_id, role=role)+return {"status": "ok"}diff --git a/autogpt_platform/backend/backend/cli/mailerlite_backfill.py b/autogpt_platform/backend/backend/cli/mailerlite_backfill.pyindex ae61a25a25..a685fe9616 100644--- a/autogpt_platform/backend/backend/cli/mailerlite_backfill.py+++ b/autogpt_platform/backend/backend/cli/mailerlite_backfill.py@@ -14,6 +14,7 @@ if TYPE_CHECKING:from backend.notifications.mailerlite_field_backfill import FieldPlan, Person_ACCOUNT_PAGE = 5000+_SKIPPED_SINCE_PLAN = "opted out or seen in Iran or Russia since the plan (skipped)"@click.command(name="mailerlite-backfill")@@ -44,7 +45,8 @@ def mailerlite_backfill_command(customer alone does not mean they opened checkout (the billing portalmakes one too), so new people come only from mailerlite-checkout-backfill.Accounts without a Stripe customer are never read. Accounts that opted out- of marketing are never written, removals included.+ of marketing, or that a signal places in Iran or Russia, are never written,+ removals included.Dry run by default: prints counts and one pseudonymised line per customer,and writes nothing. Idempotent, so a partial or repeated --apply is safe,@@ -70,7 +72,9 @@ def mailerlite_checkout_backfill_command(apply: bool, yes: bool):checkout_opened_date (their first session), email_type, signin_method,country and country_code (the Stripe billing address, else the browser'stimezone), country_source and exclude_de_at, plus their status and dates.- Openers who opted out of marketing are counted and never written.+ Openers who opted out of marketing, or whom a signal places in Iran or+ Russia, are counted and never written. Their onboarding role is+ onboarding-role-backfill's.Visits one person every two seconds, re-reading each just before thewrite so nothing the live checkout event wrote meanwhile is overwritten.@@ -123,6 +127,7 @@ async def _run(*, apply: bool, yes: bool, fields_only: bool, groups_only: bool = False) -> None:from backend.data.db import connect, disconnect+ from backend.data.user import is_marketing_opted_outfrom backend.notifications import mailerlite, mailerlite_backfillfrom backend.notifications import mailerlite_field_backfill as field_backfillfrom backend.notifications.mailerlite import MailerLiteNotConfigured@@ -137,9 +142,10 @@ async def _run(except MailerLiteNotConfigured as e:raise click.ClickException(str(e))subscriptions = await _stripe_subscriptions()+ billing_countries = await stripe_billing_countries()await connect()try:- people = await _people(subscriptions)+ people = await _people(subscriptions, billing_countries)finally:await disconnect()@@ -164,17 +170,30 @@ async def _run(f"\nWrite {due} group changes and {len(fields.changes)} field updates?",abort=True,)- if due and audience is not None:- result = await mailerlite_backfill.apply(changes, audience)- for decision in CHANGES:+ # Connected again for the writes: each is checked against the account as+ # it is now, since the plan is older (`consent.KeptOut`).+ await connect()+ try:+ if due and audience is not None:+ result = await mailerlite_backfill.apply(+ changes, audience, kept_out=is_marketing_opted_out+ )+ for decision in CHANGES:+ click.echo(+ f"{decision.value}: {result.succeeded[decision]} ok, "+ f"{result.failed[decision]} failed"+ )+ click.echo(f"{_SKIPPED_SINCE_PLAN}: {result.skipped}")+ if fields.changes:+ await mailerlite.ensure_fields()+ ok, failed, skipped = await field_backfill.apply(+ fields.changes, _progress, kept_out=is_marketing_opted_out+ )click.echo(- f"{decision.value}: {result.succeeded[decision]} ok, "- f"{result.failed[decision]} failed"+ f"fields: {ok} ok, {failed} failed, {skipped} {_SKIPPED_SINCE_PLAN}")- if fields.changes:- await mailerlite.ensure_fields()- ok, failed = await field_backfill.apply(fields.changes, _progress)- click.echo(f"fields: {ok} ok, {failed} failed")+ finally:+ await disconnect()async def _stripe_subscriptions() -> "dict[str, list[Subscription]]":@@ -194,6 +213,58 @@ async def _stripe_subscriptions() -> "dict[str, list[Subscription]]":return subscriptions+async def stripe_billing_countries(+ sessions: list[dict] | None = None,+) -> dict[str, str]:+ """Every Stripe customer's billing country, by customer ID: the+ Customer's address, else one a Checkout Session collected. Checkouts+ from before they set `customer_update` never copied the address onto+ the Customer. An Iranian or Russian country from any of them wins, so+ the exclusion sees it (`consent.py`). Pass `sessions` when they are+ already listed."""+ import stripe++ from backend.data.stripe_client import stripe_call, stripe_list_items+ from backend.notifications.audience_enrichment import billing_country+ from backend.util.settings import Settings++ stripe.api_key = Settings().secrets.stripe_api_key+ countries: dict[str, str] = {}+ page = await stripe_call(stripe.Customer.list_async, limit=100)+ async for customer in stripe_list_items(page):+ country = (customer.get("address") or {}).get("country")+ _keep_country(countries, customer.id, country)+ for session in sessions if sessions is not None else await _checkout_sessions():+ customer = session.get("customer")+ if isinstance(customer, str):+ _keep_country(countries, customer, billing_country(session))+ return countries+++async def _checkout_sessions() -> list[dict]:+ """Every Stripe Checkout Session."""+ import stripe++ from backend.data.stripe_client import stripe_call, stripe_list_items++ page = await stripe_call(stripe.checkout.Session.list_async, limit=100)+ return [session async for session in stripe_list_items(page)]+++def _keep_country(+ countries: dict[str, str], customer_id: str, country: str | None+) -> None:+ """Hold `country` for the customer unless it already has one, or the one+ it has is not excluded and this one is."""+ from backend.notifications.audience_enrichment import excluded_country++ if not country:+ return+ held = countries.get(customer_id)+ if held is None or (excluded_country((country,)) and not excluded_country((held,))):+ countries[customer_id] = str(country)++def _subscription(sub) -> "Subscription":from backend.notifications.mailerlite_backfill import Subscription@@ -236,12 +307,16 @@ def _refresher(converted: set[str]):return refresh-async def _people(subscriptions: "dict[str, list[Subscription]]") -> "list[Person]":+async def _people(+ subscriptions: "dict[str, list[Subscription]]",+ billing_countries: dict[str, str],+) -> "list[Person]":"""Every account with a Stripe customer, paged, with its Stripe- subscriptions. Accounts without one never reached checkout, so they are- not read at all. Having one is not proof of a checkout either: callers- that create MailerLite subscribers must also check for a Checkout Session.- The account's email is used, as the live handlers do, never Stripe's."""+ subscriptions and billing country. Accounts without one never reached+ checkout, so they are not read at all. Having one is not proof of a+ checkout either: callers that create MailerLite subscribers must also+ check for a Checkout Session. The account's email is used, as the live+ handlers do, never Stripe's."""import prisma.modelsfrom backend.notifications.mailerlite_field_backfill import Person@@ -276,6 +351,8 @@ async def _people(subscriptions: "dict[str, list[Subscription]]") -> "list[Persostripe_customer_id=user.stripeCustomerId,timezone=user.timezone,marketing_opt_out_at=user.marketingOptOutAt,+ billing_country=billing_countries.get(user.stripeCustomerId or ""),+ excluded_country=user.marketingExcludedCountry,))if len(page) < _ACCOUNT_PAGE:@@ -291,6 +368,9 @@ def _customer(person: "Person") -> "Customer":email=person.email,subscriptions=person.subscriptions,marketing_opt_out_at=person.marketing_opt_out_at,+ timezone=person.timezone,+ billing_country=person.billing_country,+ excluded_country=person.excluded_country,)@@ -322,6 +402,7 @@ def _report_fields(plan: "FieldPlan", accounts: int) -> None:click.echo(f" {status.value}: {count}")click.echo(f" invalid email (skipped): {plan.invalid}")click.echo(f" opted out of marketing (skipped): {plan.opted_out}")+ click.echo(f" placed in Iran or Russia (skipped): {plan.excluded_country}")click.echo(f"{len(plan.changes)} existing MailerLite subscribers to update ""(nobody is created here; see mailerlite-checkout-backfill)"@@ -339,7 +420,7 @@ async def _run_checkout(*, apply: bool, yes: bool) -> None:import stripefrom backend.data.db import connect, disconnect- from backend.data.stripe_client import stripe_call, stripe_list_items+ from backend.data.user import is_marketing_opted_outfrom backend.notifications import checkout_backfill, mailerlitefrom backend.notifications import mailerlite_field_backfill as field_backfillfrom backend.notifications.mailerlite import MailerLiteNotConfigured@@ -360,25 +441,20 @@ async def _run_checkout(*, apply: bool, yes: bool) -> None:raise click.ClickException(str(e))stripe.api_key = settings.secrets.stripe_api_key+ # Listed once, for the first opens and the billing countries alike.+ sessions = await _checkout_sessions()first_open: dict[str, int] = {}- page = await stripe_call(stripe.checkout.Session.list_async, limit=100)- async for session in stripe_list_items(page):+ for session in sessions:customer = session.get("customer")if isinstance(customer, str):- first_open[customer] = min(- first_open.get(customer, session.created), session.created- )- billing_countries: dict[str, str] = {}- page = await stripe_call(stripe.Customer.list_async, limit=100)- async for customer in stripe_list_items(page):- country = (customer.get("address") or {}).get("country")- if country:- billing_countries[customer.id] = str(country)+ created = int(session["created"])+ first_open[customer] = min(first_open.get(customer, created), created)+ billing_countries = await stripe_billing_countries(sessions)subscriptions = await _stripe_subscriptions()await connect()try:- people = await _people(subscriptions)+ people = await _people(subscriptions, billing_countries)providers = await _signin_providers()finally:await disconnect()@@ -388,7 +464,7 @@ async def _run_checkout(*, apply: bool, yes: bool) -> None:person=person,opened_at=first_open[person.stripe_customer_id],signin_providers=providers.get(person.user_id, []),- stripe_country=billing_countries.get(person.stripe_customer_id),+ stripe_country=person.billing_country,)for person in peopleif person.stripe_customer_id in first_open@@ -413,9 +489,19 @@ async def _run_checkout(*, apply: bool, yes: bool) -> None:)await mailerlite.ensure_fields()converted = {s.id for p in people for s in p.subscriptions if s.converted}- ok, failed, skipped = await checkout_backfill.apply(- plan.changes, group_id, _checkout_progress, refresh=_refresher(converted)- )+ # Connected again for the writes: each opener is checked against the+ # account as it is now, since the plan is older (`consent.KeptOut`).+ await connect()+ try:+ ok, failed, skipped = await checkout_backfill.apply(+ plan.changes,+ group_id,+ _checkout_progress,+ refresh=_refresher(converted),+ kept_out=is_marketing_opted_out,+ )+ finally:+ await disconnect()_finish_checkout(ok, failed, skipped)@@ -424,7 +510,8 @@ def _finish_checkout(ok: int, failed: int, skipped: int) -> None:on past each failure, but a Job or script must not read a partial run asdone. A rerun retries only what is left."""click.echo(- f"checkout openers: {ok} ok, {failed} failed, {skipped} already up to date"+ f"checkout openers: {ok} ok, {failed} failed, {skipped} already up to "+ "date or no longer to be written")if failed:raise click.ClickException(@@ -469,6 +556,7 @@ def _report_checkout(click.echo(" Accounts without a Stripe customer: never read or written")click.echo(f" invalid email (skipped): {plan.invalid}")click.echo(f" opted out of marketing (skipped): {plan.opted_out}")+ click.echo(f" placed in Iran or Russia (skipped): {plan.excluded_country}")click.echo(f"\nCountry source: {counts(plan.country_sources)}")click.echo(f"Countries: {counts(plan.countries)}")click.echo(f"exclude_de_at=yes: {plan.exclude_de_at}")diff --git a/autogpt_platform/backend/backend/cli/mailerlite_backfill_test.py b/autogpt_platform/backend/backend/cli/mailerlite_backfill_test.pyindex 0b19561888..c5f592032a 100644--- a/autogpt_platform/backend/backend/cli/mailerlite_backfill_test.py+++ b/autogpt_platform/backend/backend/cli/mailerlite_backfill_test.py@@ -9,6 +9,7 @@ from unittest.mock import AsyncMock, MagicMock, patchimport clickimport pytest+import stripefrom click.testing import CliRunnerfrom backend.cli import mailerlite_backfill as cli@@ -60,7 +61,9 @@ def test_the_command_exits_non_zero_when_anyone_failed(monkeypatch):assert "1 checkout openers were not written" in result.output-def _account(user_id: str, opted_out_at: datetime | None) -> SimpleNamespace:+def _account(+ user_id: str, opted_out_at: datetime | None, excluded_country: str | None = None+) -> SimpleNamespace:"""A `User` row as `_people` reads it."""return SimpleNamespace(id=user_id,@@ -69,14 +72,15 @@ def _account(user_id: str, opted_out_at: datetime | None) -> SimpleNamespace:stripeCustomerId=f"cus_{user_id}",timezone=None,marketingOptOutAt=opted_out_at,+ marketingExcludedCountry=excluded_country,)@pytest.mark.asyncio-async def test_each_account_carries_its_opt_out_into_the_backfills():+async def test_each_account_carries_its_consent_and_countries_into_the_backfills():users = MagicMock(find_many=AsyncMock(- return_value=[_account("out", OPTED_OUT), _account("in", None)]+ return_value=[_account("out", OPTED_OUT, "IR"), _account("in", None)]))trials = MagicMock(find_many=AsyncMock(return_value=[]))@@ -85,11 +89,16 @@ async def test_each_account_carries_its_opt_out_into_the_backfills():patch("prisma.models.SubscriptionTrial.prisma", return_value=trials),):out, kept = await cli._people(- {"cus_out": [Subscription(status="active")], "cus_in": []}+ {"cus_out": [Subscription(status="active")], "cus_in": []},+ {"cus_out": "RU"},)assert out.marketing_opt_out_at == OPTED_OUTassert kept.marketing_opt_out_at is None- assert cli._customer(out).marketing_opt_out_at == OPTED_OUT+ assert (out.billing_country, out.excluded_country) == ("RU", "IR")+ assert (kept.billing_country, kept.excluded_country) == (None, None)+ customer = cli._customer(out)+ assert customer.marketing_opt_out_at == OPTED_OUT+ assert (customer.billing_country, customer.excluded_country) == ("RU", "IR")assert cli._customer(kept).marketing_opt_out_at is None@@ -124,3 +133,100 @@ def test_every_report_counts_the_opted_out(capsys):assert "skip_opted_out: 1" in outassert out.count("opted out of marketing (skipped): 1") == 2assert "[email protected]" not in out+++def test_every_report_counts_anyone_placed_in_iran_or_russia(capsys):+ person = Person(+ user_id="ru",+ email="[email protected]",+ created_at=CREATED,+ subscriptions=[Subscription(status="active")],+ stripe_customer_id="cus_ru",+ timezone="Europe/Moscow",+ )+ customer = cli._customer(person)+ assert customer.timezone == "Europe/Moscow"+ cli._report(+ [+ mailerlite_backfill.decide(+ customer, Audience(tour={}, changelog={}, trial={}), trial_enabled=True+ )+ ],+ unmatched=0,+ )+ cli._report_fields(field_backfill.plan([person], {}, create=False), 1)+ cli._report_checkout(+ checkout_backfill.plan(+ [checkout_backfill.Opener(person=person, opened_at=1788305400)], {}, {}+ ),+ sessions_unlinked=0,+ customers_without_session=0,+ )+ out = capsys.readouterr().out+ assert "skip_excluded_country: 1" in out+ assert out.count("placed in Iran or Russia (skipped): 1") == 2+ assert "[email protected]" not in out+++ "seen, kept",+ [+ # The Customer's address first; a session only fills a gap.+ ([("cus_1", "US"), ("cus_1", "GB")], "US"),+ ([("cus_1", None), ("cus_1", "GB")], "GB"),+ # An older checkout billed to Russia, while the Customer has a+ # different address now: the exclusion must still see it.+ ([("cus_1", "US"), ("cus_1", "RU")], "RU"),+ ([("cus_1", "IR"), ("cus_1", "US")], "IR"),+ ],+)+def test_a_billing_country_from_any_source_is_kept_when_it_excludes(seen, kept):+ countries: dict[str, str] = {}+ for customer_id, country in seen:+ cli._keep_country(countries, customer_id, country)+ assert countries == {"cus_1": kept}+++async def test_checkout_sessions_fill_in_customers_without_an_address(monkeypatch):+ """Checkouts before `customer_update` never copied the billing address+ onto the Customer, so the sessions are read too."""+ customers = [+ SimpleNamespace(id="cus_a", get=lambda k: {"country": "US"}),+ SimpleNamespace(id="cus_b", get=lambda k: None),+ ]+ sessions = [+ {"customer": "cus_b", "customer_details": {"address": {"country": "RU"}}},+ {"customer": None, "customer_details": {"address": {"country": "IR"}}},+ ]++ async def listed(page):+ for item in page:+ yield item++ async def call(fn, **_):+ return customers if fn == stripe.Customer.list_async else sessions++ monkeypatch.setattr("backend.data.stripe_client.stripe_call", call)+ monkeypatch.setattr("backend.data.stripe_client.stripe_list_items", listed)+ assert await cli.stripe_billing_countries() == {"cus_a": "US", "cus_b": "RU"}+++async def test_sessions_already_listed_are_not_listed_again(monkeypatch):+ customers = [SimpleNamespace(id="cus_a", get=lambda k: None)]+ sessions = [+ {"customer": "cus_a", "customer_details": {"address": {"country": "IR"}}}+ ]++ async def listed(page):+ for item in page:+ yield item++ async def call(fn, **_):+ assert fn == stripe.Customer.list_async, "sessions were listed again"+ return customers++ monkeypatch.setattr("backend.data.stripe_client.stripe_call", call)+ monkeypatch.setattr("backend.data.stripe_client.stripe_list_items", listed)+ assert await cli.stripe_billing_countries(sessions) == {"cus_a": "IR"}diff --git a/autogpt_platform/backend/backend/cli/main.py b/autogpt_platform/backend/backend/cli/main.pyindex f902d6676d..e7c960a20e 100644--- a/autogpt_platform/backend/backend/cli/main.py+++ b/autogpt_platform/backend/backend/cli/main.py@@ -14,6 +14,7 @@ from .mailerlite_backfill import (mailerlite_checkout_backfill_command,mailerlite_fields_command,)+from .onboarding_role_backfill import onboarding_role_backfill_commandfrom .rotate_key import rotate_encryption_keyfrom .store import storefrom .test import test@@ -29,6 +30,7 @@ main.add_command(chat)main.add_command(mailerlite_backfill_command)main.add_command(mailerlite_checkout_backfill_command)main.add_command(mailerlite_fields_command)+main.add_command(onboarding_role_backfill_command)main.add_command(rotate_encryption_key)main.add_command(store)main.add_command(test)diff --git a/autogpt_platform/backend/backend/cli/onboarding_role_backfill.py b/autogpt_platform/backend/backend/cli/onboarding_role_backfill.pynew file mode 100644index 0000000000..24582cc918--- /dev/null+++ b/autogpt_platform/backend/backend/cli/onboarding_role_backfill.py@@ -0,0 +1,173 @@+import asyncio+import logging+from typing import TYPE_CHECKING++import click++if TYPE_CHECKING:+ from backend.notifications.role_backfill import RolePlan, RoleRecord++# Every account with a role on record: the pick the wizard kept, or the copy+# in its business understanding. Only the role is read from the+# understanding, which can hold whole brain dump transcripts.+_ROLE_RECORDS = """+ SELECT u."id" AS user_id, u."email", u."timezone",+ u."marketingOptOutAt" AS marketing_opt_out_at,+ u."stripeCustomerId" AS stripe_customer_id,+ u."marketingExcludedCountry" AS excluded_country,+ o."role" AS choice, o."roleOther" AS other,+ c."data"->'business'->>'user_role' AS understanding_role+ FROM {schema_prefix}"User" u+ LEFT JOIN {schema_prefix}"UserOnboarding" o ON o."userId" = u."id"+ LEFT JOIN {schema_prefix}"CoPilotUnderstanding" c ON c."userId" = u."id"+ WHERE o."role" IS NOT NULL+ OR c."data"->'business'->>'user_role' IS NOT NULL+"""++[email protected](name="onboarding-role-backfill")[email protected]("--apply", is_flag=True, help="Write the changes. Without it, dry run.")[email protected]("--yes", is_flag=True, help="With --apply, skip the confirmation.")+def onboarding_role_backfill_command(apply: bool, yes: bool):+ """Give existing accounts their onboarding role in PostHog and MailerLite.++ Reliable roles only: the pick the wizard kept, which the profile step has+ sent itself since SECRT-2852, or an exact option ID in the business+ understanding, the only value there that can only have come from the+ wizard. Any other value is Other's typed text or an AutoPilot rewrite, and+ is counted and skipped.++ PostHog gets onboarding_role and onboarding_role_other with $set_once, so+ a role it already holds stays. MailerLite gets role and role_other only on+ subscribers that have no role yet. Nobody is created there, and anyone who+ opted out of marketing or whom a signal places in Iran or Russia, their+ Stripe billing country included, is left out.++ Dry run by default, with counts only. Idempotent, so a repeated --apply+ resumes an interrupted one; run the dry run again afterwards to confirm+ nothing is left.+ """+ # Keep Prisma and client chatter out of the report.+ logging.disable(logging.INFO)+ asyncio.run(_run(apply=apply, yes=yes))+++async def _run(*, apply: bool, yes: bool) -> None:+ from backend.cli.mailerlite_backfill import stripe_billing_countries+ from backend.data.db import connect, disconnect+ from backend.data.user import is_marketing_opted_out+ from backend.notifications import mailerlite+ from backend.notifications import mailerlite_field_backfill as field_backfill+ from backend.notifications import role_backfill+ from backend.notifications.mailerlite import MailerLiteNotConfigured+ from backend.util.posthog_client import get_posthog_client+ from backend.util.settings import Settings++ if not Settings().secrets.stripe_api_key:+ raise click.ClickException("STRIPE_API_KEY is not set.")+ try:+ current = await field_backfill.read_current()+ except MailerLiteNotConfigured as e:+ raise click.ClickException(str(e))+ billing_countries = await stripe_billing_countries()+ await connect()+ try:+ records = _with_billing_countries(await _records(), billing_countries)+ finally:+ await disconnect()++ plan = role_backfill.plan(records, current)+ _report(plan)++ if not apply:+ click.echo("\nDry run: nothing was written. Re-run with --apply to write.")+ return+ if not plan.posthog and not plan.mailerlite:+ click.echo("\nNothing to write.")+ return+ if get_posthog_client() is None:+ raise click.ClickException("POSTHOG_API_KEY is not set.")+ if not yes:+ click.confirm(+ f"\nSet the role on {len(plan.posthog)} PostHog persons and "+ f"{len(plan.mailerlite)} MailerLite subscribers?",+ abort=True,+ )+ await role_backfill.send_to_posthog(plan.posthog)+ click.echo(f"PostHog: {len(plan.posthog)} sent")+ if plan.mailerlite:+ await mailerlite.ensure_fields()+ # Connected again for the writes: each subscriber is checked against+ # the account as it is now, since the plan is older.+ await connect()+ try:+ ok, failed, skipped = await role_backfill.apply(+ plan.mailerlite, _progress, kept_out=is_marketing_opted_out+ )+ finally:+ await disconnect()+ _finish(ok, failed, skipped)+++async def _records() -> "list[RoleRecord]":+ from backend.data.db import query_raw_with_schema+ from backend.notifications.role_backfill import RoleRecord++ return await query_raw_with_schema(_ROLE_RECORDS, model=RoleRecord)+++def _with_billing_countries(+ records: "list[RoleRecord]", billing_countries: dict[str, str]+) -> "list[RoleRecord]":+ return [+ record.model_copy(+ update={+ "billing_country": billing_countries.get(+ record.stripe_customer_id or ""+ )+ }+ )+ for record in records+ ]+++def _report(plan: "RolePlan") -> None:+ from backend.notifications.role_backfill import WRITE_INTERVAL_SECONDS++ roles = ", ".join(+ f"{label}={count}"+ for label, count in sorted(plan.roles.items(), key=lambda kv: -kv[1])+ )+ click.echo(f"Accounts with a role on record: {plan.accounts}")+ click.echo(f" reliable, kept by the wizard: {plan.kept}")+ click.echo(f" reliable, an exact option in the understanding: {plan.exact}")+ click.echo(f" skipped, Other's text or an AutoPilot rewrite: {plan.skipped}")+ click.echo(f"Roles: {roles}")+ click.echo(f"\nPostHog: {len(plan.posthog)} persons ($set_once)")+ click.echo(f"MailerLite: {len(plan.mailerlite)} subscribers to fill in")+ click.echo(f" already have a role (kept): {plan.already_set}")+ click.echo(f" not subscribers (never created): {plan.not_subscribers}")+ click.echo(f" opted out of marketing (skipped): {plan.opted_out}")+ click.echo(f" placed in Iran or Russia (skipped): {plan.excluded_country}")+ minutes = len(plan.mailerlite) * WRITE_INTERVAL_SECONDS / 60+ click.echo(f"About {minutes:.0f} minutes at one subscriber every two seconds")+++def _progress(done: int, total: int) -> None:+ click.echo(f"MailerLite: {done}/{total} written")+++def _finish(ok: int, failed: int, skipped: int) -> None:+ """Report the run, and fail it when anyone was not written, so a Job or+ script never reads a partial run as done. A rerun retries only what is+ left."""+ click.echo(+ f"MailerLite: {ok} ok, {failed} failed, {skipped} no longer needed or "+ "no longer to be written"+ )+ if failed:+ raise click.ClickException(+ f"{failed} MailerLite subscribers were not written (the log says "+ "why); run again to retry them"+ )+ click.echo("Run the dry run again to confirm nothing is left.")diff --git a/autogpt_platform/backend/backend/cli/onboarding_role_backfill_db_test.py b/autogpt_platform/backend/backend/cli/onboarding_role_backfill_db_test.pynew file mode 100644index 0000000000..f7deaaf54b--- /dev/null+++ b/autogpt_platform/backend/backend/cli/onboarding_role_backfill_db_test.py@@ -0,0 +1,59 @@+"""The role backfill's query against the real tables: the pick the wizard kept+and the role in the business understanding, side by side, for every account+that has either."""++import uuid++import prisma.models+import pytest++from backend.cli import onboarding_role_backfill as cli+from backend.data.onboarding_role import OnboardingRole, save_onboarding_role+from backend.util.json import SafeJson+++async def account():+ user_id = str(uuid.uuid4())+ await prisma.models.User.prisma().create(+ data={"id": user_id, "email": f"{user_id}@example.com"}+ )+ yield user_id+ await prisma.models.User.prisma().delete(where={"id": user_id})+++async def test_both_records_of_the_role_are_read(account):+ await save_onboarding_role(account, OnboardingRole(choice="Other", other="CFO"))+ await prisma.models.CoPilotUnderstanding.prisma().create(+ data={+ "userId": account,+ "data": SafeJson({"name": "Sam", "business": {"user_role": "Marketing"}}),+ }+ )++ (record,) = [r for r in await cli._records() if r.user_id == account]++ assert record.email == f"{account}@example.com"+ assert (record.choice, record.other) == ("Other", "CFO")+ assert record.understanding_role == "Marketing"+ assert record.marketing_opt_out_at is None+++async def test_an_account_with_no_role_is_not_read(account):+ assert account not in {r.user_id for r in await cli._records()}+++async def test_the_country_a_checkout_recorded_is_read(account):+ await save_onboarding_role(account, OnboardingRole(choice="Marketing"))+ await prisma.models.User.prisma().update(+ where={"id": account},+ data={"stripeCustomerId": f"cus_{account}", "marketingExcludedCountry": "RU"},+ )++ (record,) = [r for r in await cli._records() if r.user_id == account]++ assert record.stripe_customer_id == f"cus_{account}"+ assert record.excluded_country == "RU"diff --git a/autogpt_platform/backend/backend/cli/onboarding_role_backfill_test.py b/autogpt_platform/backend/backend/cli/onboarding_role_backfill_test.pynew file mode 100644index 0000000000..fe72545be3--- /dev/null+++ b/autogpt_platform/backend/backend/cli/onboarding_role_backfill_test.py@@ -0,0 +1,85 @@+"""The role backfill's dry run reports reliable against skipped accounts and+where each one would go, by count only, and a run that could not write+everyone fails as a whole after reporting."""++import click+import pytest+from click.testing import CliRunner++from backend.cli import onboarding_role_backfill as cli+from backend.notifications import role_backfill+from backend.notifications.role_backfill import RoleRecord+++# The root conftest spins a full test server for every test via an autouse+# session fixture; these never touch it.[email protected](scope="session")+def server():+ yield None++[email protected](scope="session", autouse=True)+def graph_cleanup():+ yield+++def test_the_dry_run_report_counts_reliable_and_skipped(capsys):+ records = [+ RoleRecord(user_id="1", email="[email protected]", choice="Marketing"),+ RoleRecord(user_id="2", email="[email protected]", understanding_role="Marketing"),+ RoleRecord(user_id="3", email="[email protected]", understanding_role="Dentist"),+ RoleRecord(+ user_id="4",+ email="[email protected]",+ understanding_role="Engineering",+ ),+ ]+ cli._report(+ role_backfill.plan(records, {"[email protected]": {}, "[email protected]": {"role": "x"}})+ )+ out = capsys.readouterr().out+ assert "Accounts with a role on record: 4" in out+ assert "reliable, kept by the wizard: 1" in out+ assert "reliable, an exact option in the understanding: 2" in out+ assert "skipped, Other's text or an AutoPilot rewrite: 1" in out+ assert "Roles: Marketing=2, Engineering=1" in out+ assert "PostHog: 3 persons" in out+ assert "MailerLite: 1 subscribers to fill in" in out+ assert "already have a role (kept): 1" in out+ assert "placed in Iran or Russia (skipped): 1" in out+ assert "@x.io" not in out and "firma.ru" not in out+++def test_a_complete_run_reports_and_succeeds(capsys):+ cli._finish(5, 0, 2)+ out = capsys.readouterr().out+ assert "5 ok, 0 failed, 2 no longer needed" in out+ assert "Run the dry run again" in out+++def test_the_command_exits_non_zero_when_anyone_failed(monkeypatch):+ async def run(*, apply: bool, yes: bool) -> None:+ cli._finish(4, 1, 0)++ monkeypatch.setattr(cli, "_run", run)+ result = CliRunner().invoke(+ cli.onboarding_role_backfill_command, ["--apply", "--yes"]+ )+ assert result.exit_code == 1+ assert "4 ok, 1 failed" in result.output+ assert "1 MailerLite subscribers were not written" in result.output+++def test_a_run_with_failures_reports_then_fails():+ with pytest.raises(click.ClickException, match="2 MailerLite subscribers"):+ cli._finish(1, 2, 0)+++def test_each_record_gets_its_stripe_billing_country():+ records = [+ RoleRecord(user_id="1", email="[email protected]", stripe_customer_id="cus_1"),+ RoleRecord(user_id="2", email="[email protected]", stripe_customer_id="cus_2"),+ RoleRecord(user_id="3", email="[email protected]"),+ ]+ filled = cli._with_billing_countries(records, {"cus_1": "RU", "cus_2": "US"})+ assert [r.billing_country for r in filled] == ["RU", "US", None]diff --git a/autogpt_platform/backend/backend/data/checkout_audience.py b/autogpt_platform/backend/backend/data/checkout_audience.pyindex 4dcd5380de..89661959d7 100644--- a/autogpt_platform/backend/backend/data/checkout_audience.py+++ b/autogpt_platform/backend/backend/data/checkout_audience.py@@ -9,8 +9,11 @@ strongest country it has seen (`audience_enrichment.merge_with_held`).It runs in the background and never raises. A checkout must not wait on, orfail because of, MailerLite bookkeeping; the backfill catches anyone missed.-Nothing is queued for an account that opted out of marketing: it never enters-MailerLite (`notifications/consent.py`).+Nothing is queued for an account that opted out of marketing, or that any+signal places in Iran or Russia, the IP and billing countries included: it+never enters MailerLite (`notifications/consent.py`). An Iranian or Russian+IP or billing country is also recorded on the account, since the trial and+billing events that follow a checkout don't carry it."""import asyncio@@ -21,10 +24,16 @@ import prisma.modelsfrom backend.data.db import query_raw_with_schemafrom backend.data.notifications import AudienceAction-from backend.data.user import get_user_by_id-from backend.notifications.audience_enrichment import checkout_fields+from backend.data.onboarding_role import OnboardingRole, get_onboarding_role+from backend.data.user import get_user_by_id, record_excluded_country+from backend.notifications.audience_enrichment import (+ billing_country,+ checkout_fields,+ excluded_country,+)from backend.notifications.consent import audience_change_allowedfrom backend.notifications.subscriber_fields import queue_fields+from backend.util.retry import func_retrylogger = logging.getLogger(__name__)@@ -64,8 +73,11 @@ async def queue_checkout_opened(opened_at: datetime | int | None = None,) -> None:try:+ await _record_excluded_country(user_id, (ip_country, stripe_country))user = await get_user_by_id(user_id)- if not audience_change_allowed(user, AudienceAction.CHECKOUT_OPENED):+ if not audience_change_allowed(+ user, AudienceAction.CHECKOUT_OPENED, (ip_country, stripe_country)+ ):returnfields = checkout_fields(email=user.email,@@ -75,6 +87,7 @@ async def queue_checkout_opened(timezone=user.timezone,stripe_country=stripe_country,ip_country=ip_country,+ role=await _role(user_id),)await queue_fields(user_id, user.email, fields, AudienceAction.CHECKOUT_OPENED)except Exception:@@ -84,7 +97,16 @@ async def queue_checkout_opened(async def record_checkout_completed(session: dict) -> None:"""A finished checkout carries the billing address, the strongest countrysignal, which also catches a German or Austrian buyer whose IP and- timezone did not say so. Never raises: the webhook must not fail on it."""+ timezone did not say so. An Iranian or Russian one is recorded before+ this returns, so the webhook runs it before queueing the trial notice,+ whose MailerLite change can only be stopped by that record.++ So it raises when such a country could not be recorded: the webhook fails+ and Stripe retries it, rather than queue the trial notice anyway. Any+ other failure is logged: the webhook must not fail on MailerLite+ bookkeeping."""+ country = billing_country(session)+ excluded = excluded_country((country,))try:customer = session.get("customer")customer_id = customer if isinstance(customer, str) else None@@ -96,13 +118,14 @@ async def record_checkout_completed(session: dict) -> None:if user is None:# An organization's customer, or one whose account is gone.return- address = (session.get("customer_details") or {}).get("address") or {}+ if excluded:+ await record_excluded_country(user.id, excluded)schedule_checkout_opened(- user.id,- stripe_country=address.get("country"),- opened_at=session.get("created"),+ user.id, stripe_country=country, opened_at=session.get("created"))except Exception:+ if excluded:+ raiselogger.exception("Could not record the MailerLite checkout completion")@@ -115,3 +138,25 @@ async def signin_providers(user_id: str) -> list[str]:user_id,)return [str(row["provider"]) for row in rows if row.get("provider")]+++@func_retry+async def _record_excluded_country(+ user_id: str, countries: tuple[str | None, ...]+) -> None:+ """Retried in place: this runs in the background, so a failure has no+ caller to retry it, and the trial and billing events that follow the+ checkout are only stopped by this record. A failure that outlasts the+ retries is logged as an error by `queue_checkout_opened`."""+ if country := excluded_country(countries):+ await record_excluded_country(user_id, country)+++async def _role(user_id: str) -> OnboardingRole | None:+ """The role picked in onboarding, if there is one yet. A failed read+ costs the opener their role, never their checkout event."""+ try:+ return await get_onboarding_role(user_id)+ except Exception:+ logger.warning(f"Could not read the onboarding role of {user_id}")+ return Nonediff --git a/autogpt_platform/backend/backend/data/checkout_audience_test.py b/autogpt_platform/backend/backend/data/checkout_audience_test.pyindex 39c6a81748..88b1051d6e 100644--- a/autogpt_platform/backend/backend/data/checkout_audience_test.py+++ b/autogpt_platform/backend/backend/data/checkout_audience_test.py@@ -22,6 +22,7 @@ from backend.data.notifications import (NotificationResult,SubscriberField,)+from backend.data.onboarding_role import OnboardingRolefrom backend.data.subscription_trial_checkout import TrialUnavailablefrom backend.notifications import consent, subscriber_fieldsfrom backend.notifications.consent_test import _cached_before_consent@@ -48,7 +49,15 @@ def graph_cleanup():@pytest.fixture-def queued(monkeypatch):+def recorded(monkeypatch):+ """Where an Iranian or Russian country would be recorded on the account."""+ record = AsyncMock()+ monkeypatch.setattr(checkout_audience, "record_excluded_country", record)+ return record+++def queued(monkeypatch, recorded):queue = AsyncMock(return_value=NotificationResult(success=True))monkeypatch.setattr(subscriber_fields, "queue_audience_change", queue)monkeypatch.setattr(@@ -66,23 +75,107 @@ def queued(monkeypatch):monkeypatch.setattr(checkout_audience, "signin_providers", AsyncMock(return_value=["credential"]))+ monkeypatch.setattr(+ checkout_audience,+ "get_onboarding_role",+ AsyncMock(return_value=OnboardingRole(choice="Marketing")),+ )return queue@pytest.mark.asyncio-async def test_opening_checkout_queues_the_enriched_change(queued):+async def test_opening_checkout_queues_the_enriched_change(queued, recorded):checkout_audience.schedule_checkout_opened("user-1", ip_country="US")for task in list(checkout_audience._tasks):await task+ recorded.assert_not_awaited()event = queued.await_args.args[0]assert event.action is AudienceAction.CHECKOUT_OPENEDassert event.fields[SubscriberField.COUNTRY_CODE] == "US"assert event.fields[SubscriberField.COUNTRY_SOURCE] == "ip"assert event.fields[SubscriberField.SIGNIN_METHOD] == "email"+ assert event.fields[SubscriberField.ROLE] == "Marketing"# The IP says US, but the browser sits in Vienna.assert event.fields[SubscriberField.EXCLUDE_DE_AT] == "yes"+ "role",+ [+ AsyncMock(return_value=None),+ AsyncMock(side_effect=RuntimeError("db down")),+ ],+ ids=["not-picked-yet", "unreadable"],+)+async def test_an_opener_without_a_readable_role_is_still_queued(+ queued, monkeypatch, role+):+ monkeypatch.setattr(checkout_audience, "get_onboarding_role", role)+ await checkout_audience.queue_checkout_opened("user-1", ip_country="US")+ event = queued.await_args.args[0]+ assert event.action is AudienceAction.CHECKOUT_OPENED+ assert SubscriberField.ROLE not in event.fields+ assert SubscriberField.ROLE_OTHER not in event.fields+++ "countries, code",+ [+ (dict(ip_country="RU"), "RU"),+ (dict(stripe_country="IR"), "IR"),+ (dict(ip_country="ir"), "IR"),+ (dict(ip_country="US", stripe_country="RU"), "RU"),+ ],+)+async def test_an_opener_seen_in_iran_or_russia_is_recorded_and_never_queued(+ queued, recorded, monkeypatch, caplog, countries, code+):+ """Recorded on the account, since the trial and billing events that+ follow carry no IP or billing country."""+ providers = AsyncMock(return_value=["credential"])+ monkeypatch.setattr(checkout_audience, "signin_providers", providers)+ with caplog.at_level(logging.DEBUG, logger=consent.__name__):+ await checkout_audience.queue_checkout_opened("user-1", **countries)+ recorded.assert_awaited_once_with("user-1", code)+ queued.assert_not_awaited()+ providers.assert_not_awaited()+ assert pseudonym(EMAIL) in caplog.text+ assert EMAIL not in caplog.text+++async def test_a_failed_record_of_a_russian_ip_is_retried(queued, recorded):+ """This runs in the background, so nothing else would retry it, and the+ events after the checkout carry no IP country."""+ recorded.side_effect = [RuntimeError("db blip"), None]+ await checkout_audience.queue_checkout_opened("user-1", ip_country="RU")+ assert recorded.await_count == 2+ recorded.assert_awaited_with("user-1", "RU")+ queued.assert_not_awaited()+++async def test_an_opener_whose_browser_sits_in_moscow_is_never_queued(+ queued, monkeypatch+):+ monkeypatch.setattr(+ checkout_audience,+ "get_user_by_id",+ AsyncMock(+ return_value=SimpleNamespace(+ email=EMAIL,+ created_at=CREATED,+ timezone="Europe/Moscow",+ marketing_opt_out_at=None,+ )+ ),+ )+ await checkout_audience.queue_checkout_opened("user-1", ip_country="US")+ queued.assert_not_awaited()++@pytest.mark.asyncioasync def test_a_failure_never_reaches_the_checkout(queued, monkeypatch):monkeypatch.setattr(@@ -163,6 +256,77 @@ def _user_prisma(user):return MagicMock(find_first=AsyncMock(return_value=user))+async def test_a_checkout_billed_to_russia_is_recorded_before_it_returns(+ monkeypatch,+):+ """The webhook then queues the trial notice, whose MailerLite change only+ this record can stop, so it must not wait for the background task."""+ order = []+ monkeypatch.setattr(+ checkout_audience,+ "record_excluded_country",+ AsyncMock(side_effect=lambda *args: order.append(("recorded", *args))),+ )+ monkeypatch.setattr(+ checkout_audience,+ "schedule_checkout_opened",+ MagicMock(side_effect=lambda *_, **kw: order.append(("scheduled", kw))),+ )+ with patch(+ "prisma.models.User.prisma",+ return_value=_user_prisma(SimpleNamespace(id="user-1")),+ ):+ await checkout_audience.record_checkout_completed(+ {+ "customer": "cus_1",+ "created": 1788305400,+ "customer_details": {"address": {"country": "RU"}},+ }+ )+ assert order == [+ ("recorded", "user-1", "RU"),+ ("scheduled", {"stripe_country": "RU", "opened_at": 1788305400}),+ ]+++_BILLED_TO_RUSSIA = {+ "customer": "cus_1",+ "created": 1788305400,+ "customer_details": {"address": {"country": "RU"}},+}+++ "lookup, record",+ [+ (SimpleNamespace(id="user-1"), AsyncMock(side_effect=RuntimeError("db"))),+ (RuntimeError("db down"), AsyncMock()),+ ],+ ids=["record-failed", "lookup-failed"],+)+async def test_a_russian_billing_country_that_cannot_be_recorded_fails_the_webhook(+ monkeypatch, lookup, record+):+ """Stripe then retries the event, rather than the webhook queueing a+ trial notice whose MailerLite change nothing would stop."""+ schedule = MagicMock()+ monkeypatch.setattr(checkout_audience, "record_excluded_country", record)+ monkeypatch.setattr(checkout_audience, "schedule_checkout_opened", schedule)+ users = (+ MagicMock(find_first=AsyncMock(side_effect=lookup))+ if isinstance(lookup, Exception)+ else _user_prisma(lookup)+ )+ with (+ patch("prisma.models.User.prisma", return_value=users),+ pytest.raises(RuntimeError),+ ):+ await checkout_audience.record_checkout_completed(_BILLED_TO_RUSSIA)+ schedule.assert_not_called()++@pytest.mark.asyncioasync def test_a_completed_checkout_sends_its_billing_country(monkeypatch):schedule = MagicMock()@@ -297,3 +461,8 @@ def test_the_subscription_checkout_and_the_webhook_are_wired():)webhook = _source(subscription_routes.stripe_webhook)assert "awaitcheckout_audience.record_checkout_completed(data_object)" in webhook+ # Before the trial notice is queued, so an Iranian or Russian billing+ # country is on record by the time its MailerLite change is consumed.+ assert webhook.index("record_checkout_completed(") < webhook.index(+ "_notify_checkout_completed("+ )diff --git a/autogpt_platform/backend/backend/data/notifications.py b/autogpt_platform/backend/backend/data/notifications.pyindex 2cddde1195..e268462e40 100644--- a/autogpt_platform/backend/backend/data/notifications.py+++ b/autogpt_platform/backend/backend/data/notifications.py@@ -434,6 +434,10 @@ class AudienceAction(Enum):# unsubscribed, and nobody is created (see `mailerlite.unsubscribe`). The# only change queued for an opted-out account, since it is the refusal.UNSUBSCRIBE = "unsubscribe"+ # The onboarding answers. On cloud the paywall comes first, so they are+ # given after checkout opened: they fill in someone MailerLite already has+ # and never create anyone (see `mailerlite.record_onboarding_profile`).+ ONBOARDING_PROFILE = "onboarding_profile"class SubscriberField(str, Enum):@@ -451,6 +455,10 @@ class SubscriberField(str, Enum):CHECKOUT_OPENED = "checkout_opened_date"EMAIL_TYPE = "email_type"SIGNIN_METHOD = "signin_method"+ # The role picked in onboarding, as labelled there, and what was typed+ # after picking Other (see `data/onboarding_role.py`).+ ROLE = "role"+ ROLE_OTHER = "role_other"# Built into MailerLite, so never created: the full English name.COUNTRY = "country"COUNTRY_CODE = "country_code"diff --git a/autogpt_platform/backend/backend/data/onboarding_audience.py b/autogpt_platform/backend/backend/data/onboarding_audience.pynew file mode 100644index 0000000000..43f99c3390--- /dev/null+++ b/autogpt_platform/backend/backend/data/onboarding_audience.py@@ -0,0 +1,36 @@+"""Queue the onboarding role for the account's MailerLite subscriber.++On cloud the paywall comes first, so the wizard's role is answered after+checkout opened, once the account is a checkout opener in MailerLite. It is+written onto that subscriber and never creates one: only checkout openers+belong in MailerLite (`mailerlite.record_onboarding_profile`). Someone who+answers before opening checkout gets their role with the checkout event+instead (`checkout_audience`).++Nothing is queued for an account that may not be in MailerLite at all+(`notifications/consent.py`), and nothing here ever raises: the profile is+saved whatever happens to the MailerLite bookkeeping.+"""++import logging++from backend.data.notifications import AudienceAction+from backend.data.onboarding_role import OnboardingRole+from backend.data.user import get_user_by_id+from backend.notifications.audience_enrichment import role_fields+from backend.notifications.consent import audience_change_allowed+from backend.notifications.subscriber_fields import queue_fields++logger = logging.getLogger(__name__)+++async def queue_onboarding_role(user_id: str, role: OnboardingRole) -> None:+ try:+ user = await get_user_by_id(user_id)+ if not audience_change_allowed(user, AudienceAction.ONBOARDING_PROFILE):+ return+ await queue_fields(+ user_id, user.email, role_fields(role), AudienceAction.ONBOARDING_PROFILE+ )+ except Exception:+ logger.exception(f"Could not queue the MailerLite role for {user_id}")diff --git a/autogpt_platform/backend/backend/data/onboarding_audience_test.py b/autogpt_platform/backend/backend/data/onboarding_audience_test.pynew file mode 100644index 0000000000..7032a4b2f5--- /dev/null+++ b/autogpt_platform/backend/backend/data/onboarding_audience_test.py@@ -0,0 +1,103 @@+"""The role picked in onboarding is queued for the account's MailerLite+subscriber, unless the account may not be in MailerLite at all, and none of+it can cost the profile."""++from datetime import UTC, datetime+from types import SimpleNamespace+from unittest.mock import AsyncMock++import pytest++from backend.data import onboarding_audience+from backend.data.notifications import (+ AudienceAction,+ NotificationResult,+ SubscriberField,+)+from backend.data.onboarding_role import OnboardingRole+from backend.notifications import subscriber_fields++EMAIL = "[email protected]"+OPTED_OUT = datetime(2026, 10, 2, tzinfo=UTC)+++# The root conftest spins a full test server for every test via an autouse+# session fixture. These are unit tests over mocks, so shadow it for this+# module, as backend/data/db_test.py does.[email protected](scope="session")+def server():+ yield None++[email protected](scope="session", autouse=True)+def graph_cleanup():+ yield+++def _user(**overrides) -> SimpleNamespace:+ fields = dict(email=EMAIL, timezone="America/Chicago", marketing_opt_out_at=None)+ return SimpleNamespace(**{**fields, **overrides})+++def queued(monkeypatch):+ queue = AsyncMock(return_value=NotificationResult(success=True))+ monkeypatch.setattr(subscriber_fields, "queue_audience_change", queue)+ monkeypatch.setattr(+ onboarding_audience, "get_user_by_id", AsyncMock(return_value=_user())+ )+ return queue+++ "role, fields",+ [+ (OnboardingRole(choice="HR/People"), ("HR / People", None)),+ (OnboardingRole(choice="Other", other="Dentist"), ("Other", "Dentist")),+ ],+)+async def test_the_pick_is_queued_as_labelled(queued, role, fields):+ await onboarding_audience.queue_onboarding_role("user-1", role)+ event = queued.await_args.args[0]+ assert event.action is AudienceAction.ONBOARDING_PROFILE+ assert event.email == EMAIL+ assert event.fields == {+ SubscriberField.ROLE: fields[0],+ SubscriberField.ROLE_OTHER: fields[1],+ }+++ "user",+ [+ _user(marketing_opt_out_at=OPTED_OUT),+ _user(timezone="Asia/Tehran"),+ _user(email="[email protected]"),+ ],+ ids=["opted-out", "iran", "russia"],+)+async def test_nothing_is_queued_for_an_account_mailerlite_may_not_hold(+ queued, monkeypatch, user+):+ monkeypatch.setattr(+ onboarding_audience, "get_user_by_id", AsyncMock(return_value=user)+ )+ await onboarding_audience.queue_onboarding_role(+ "user-1", OnboardingRole(choice="Marketing")+ )+ queued.assert_not_awaited()+++async def test_a_failure_never_reaches_the_profile(queued, monkeypatch):+ monkeypatch.setattr(+ onboarding_audience,+ "get_user_by_id",+ AsyncMock(side_effect=RuntimeError("db down")),+ )+ await onboarding_audience.queue_onboarding_role(+ "user-1", OnboardingRole(choice="Marketing")+ )+ queued.assert_not_awaited()diff --git a/autogpt_platform/backend/backend/data/onboarding_role.py b/autogpt_platform/backend/backend/data/onboarding_role.pynew file mode 100644index 0000000000..1e2f47f4bf--- /dev/null+++ b/autogpt_platform/backend/backend/data/onboarding_role.py@@ -0,0 +1,77 @@+"""The role picked in the onboarding wizard, kept as it was picked.++The wizard (frontend onboarding/steps/RoleStep.tsx) offers seven roles and+Other, which takes typed text. POST /onboarding/profile receives the option ID,+or for Other the typed text in its place, and the pick is kept on+`UserOnboarding`. The copy in the business understanding is no record of it:+AutoPilot's add_understanding tool, the brain dump extraction and Tally all+rewrite that field.++GTM segments on the pick, under the label people saw, in MailerLite (`role`,+`role_other`) and PostHog (`onboarding_role`, `onboarding_role_other`).+"""++from prisma.models import UserOnboarding+from pydantic import BaseModel++# Each option ID the wizard sends, and the label people saw.+ROLE_LABELS = {+ "Founder/CEO": "Founder / CEO",+ "Operations": "Operations",+ "Sales/BD": "Sales / BD",+ "Marketing": "Marketing",+ "Product/PM": "Product / PM",+ "Engineering": "Engineering",+ "HR/People": "HR / People",+}+OTHER = "Other"+OTHER_MAX_CHARS = 100+++class OnboardingRole(BaseModel):+ # One of `ROLE_LABELS`' option IDs, or Other.+ choice: str+ # What was typed after picking Other; None for every other choice.+ other: str | None = None++ @property+ def label(self) -> str:+ return ROLE_LABELS.get(self.choice, OTHER)++ @classmethod+ def from_answer(cls, answer: str) -> "OnboardingRole":+ """The wizard sends the option ID, or for Other what was typed. Typing+ an option's exact ID under Other reads as picking that option."""+ text = answer.strip()+ if text in ROLE_LABELS:+ return cls(choice=text)+ return cls(choice=OTHER, other=text[:OTHER_MAX_CHARS].strip() or None)++ @classmethod+ def from_understanding(cls, user_role: str | None) -> "OnboardingRole | None":+ """The pick, for an account the wizard kept none for, from the copy in+ its business understanding. An exact option ID can only have come+ from the wizard. Anything else is Other's typed text or a rewrite,+ and the two can't be told apart, so it gives no pick."""+ if user_role in ROLE_LABELS:+ return cls(choice=user_role)+ return None+++async def save_onboarding_role(user_id: str, role: OnboardingRole) -> None:+ await UserOnboarding.prisma().upsert(+ where={"userId": user_id},+ data={+ "create": {"userId": user_id, "role": role.choice, "roleOther": role.other},+ "update": {"role": role.choice, "roleOther": role.other},+ },+ )+++async def get_onboarding_role(user_id: str) -> OnboardingRole | None:+ """The pick the wizard kept, or None for an account that never reached+ the end of it."""+ row = await UserOnboarding.prisma().find_unique(where={"userId": user_id})+ if row is None or not row.role:+ return None+ return OnboardingRole(choice=row.role, other=row.roleOther)diff --git a/autogpt_platform/backend/backend/data/onboarding_role_test.py b/autogpt_platform/backend/backend/data/onboarding_role_test.pynew file mode 100644index 0000000000..6dbe77bd72--- /dev/null+++ b/autogpt_platform/backend/backend/data/onboarding_role_test.py@@ -0,0 +1,101 @@+"""The wizard's pick is read from what it sends, kept as picked, and labelled+as people saw it; the business understanding's copy only counts when it is an+exact option ID."""++from types import SimpleNamespace+from unittest.mock import AsyncMock, MagicMock, patch++import pytest++from backend.data import onboarding_role+from backend.data.onboarding_role import OnboardingRole+++# The root conftest spins a full test server for every test via an autouse+# session fixture. These are unit tests over mocks, so shadow it for this+# module, as backend/data/db_test.py does.[email protected](scope="session")+def server():+ yield None++[email protected](scope="session", autouse=True)+def graph_cleanup():+ yield+++ "answer, choice, other, label",+ [+ ("Founder/CEO", "Founder/CEO", None, "Founder / CEO"),+ (" Sales/BD ", "Sales/BD", None, "Sales / BD"),+ ("Marketing", "Marketing", None, "Marketing"),+ # Other sends what was typed in its place.+ (" Dentist ", "Other", "Dentist", "Other"),+ ("marketing", "Other", "marketing", "Other"),+ ("x" * 99 + " yz", "Other", "x" * 99, "Other"),+ # Passes the route's min_length but never comes from the wizard.+ (" ", "Other", None, "Other"),+ ],+)+def test_the_answer_is_an_option_or_others_text(answer, choice, other, label):+ role = OnboardingRole.from_answer(answer)+ assert (role.choice, role.other, role.label) == (choice, other, label)+++ "stored, choice",+ [+ ("Engineering", "Engineering"),+ ("HR/People", "HR/People"),+ # Other's text and an AutoPilot rewrite look the same: no pick.+ ("Dentist", None),+ ("decision maker", None),+ ("Marketing manager", None),+ (" Marketing", None),+ (None, None),+ ("", None),+ ],+)+def test_only_an_exact_option_in_the_understanding_counts(stored, choice):+ role = OnboardingRole.from_understanding(stored)+ assert (role.choice if role else None) == choice+++async def test_the_pick_is_kept_on_the_onboarding_row():+ table = MagicMock(upsert=AsyncMock())+ with patch("prisma.models.UserOnboarding.prisma", return_value=table):+ await onboarding_role.save_onboarding_role(+ "user-1", OnboardingRole(choice="Other", other="Dentist")+ )+ table.upsert.assert_awaited_once_with(+ where={"userId": "user-1"},+ data={+ "create": {"userId": "user-1", "role": "Other", "roleOther": "Dentist"},+ "update": {"role": "Other", "roleOther": "Dentist"},+ },+ )+++ "row, role",+ [+ (+ SimpleNamespace(role="Marketing", roleOther=None),+ OnboardingRole(choice="Marketing"),+ ),+ (+ SimpleNamespace(role="Other", roleOther="Dentist"),+ OnboardingRole(choice="Other", other="Dentist"),+ ),+ (SimpleNamespace(role=None, roleOther=None), None),+ (None, None),+ ],+ ids=["option", "other", "not-picked", "no-row"],+)+async def test_the_kept_pick_is_read_back(row, role):+ table = MagicMock(find_unique=AsyncMock(return_value=row))+ with patch("prisma.models.UserOnboarding.prisma", return_value=table):+ assert await onboarding_role.get_onboarding_role("user-1") == rolediff --git a/autogpt_platform/backend/backend/data/user.py b/autogpt_platform/backend/backend/data/user.pyindex d527d5c1f7..b658843579 100644--- a/autogpt_platform/backend/backend/data/user.py+++ b/autogpt_platform/backend/backend/data/user.py@@ -40,6 +40,7 @@ from backend.data.notifications import (from backend.data.org_migration import ensure_personal_orgfrom backend.data.posthog_lifecycle_sync import schedule_posthog_lifecycle_syncfrom backend.data.subscription_trial import get_subscription_trial+from backend.notifications.audience_enrichment import points_at_excluded_countryfrom backend.notifications.queue import queue_audience_changefrom backend.notifications.subscriber_fields import audience_eventfrom backend.util.cache import cached@@ -1132,17 +1133,37 @@ async def record_marketing_opt_out_by_email(email: str, source: str) -> str | Noasync def is_marketing_opted_out(user_id: str) -> bool:- """Whether the account refused marketing, read from the database rather- than the user cache, for the last check before a MailerLite write. An- account that no longer exists counts as opted out: nothing about it should- reach MailerLite either."""+ """Whether nothing about the account may reach MailerLite, read from the+ database rather than the user cache, for the last check before a+ MailerLite write: it refused marketing, or a signal places it in Iran or+ Russia, the country a checkout recorded included (`consent.py`). An+ account that no longer exists counts too.++ It is named for the opt-out it was first written for. The audience+ consumer calls it over the RPC, so a new name would fail every call made+ while a rolling deploy runs one side ahead of the other."""try:row = await PrismaUser.prisma().find_unique(where={"id": user_id})except Exception as e:raise DatabaseError(f"Failed to read the marketing opt-out for user {user_id}: {e}") from e- return row is None or row.marketingOptOutAt is not None+ if row is None or row.marketingOptOutAt is not None:+ return True+ return points_at_excluded_country(+ email=row.email,+ timezone=row.timezone,+ countries=(row.marketingExcludedCountry,),+ )+++async def record_excluded_country(user_id: str, country: str) -> None:+ """Keep the account out of MailerLite for good: a checkout saw it in a+ country MailerLite must never hold. The first country seen is kept."""+ await PrismaUser.prisma().update_many(+ where={"id": user_id, "marketingExcludedCountry": None},+ data={"marketingExcludedCountry": country},+ )def _invalidate_user_caches(user_id: str, email: str | None) -> None:@@ -1252,6 +1273,8 @@ class BillingEmailRecipient(BaseModel):# Set when the customer refused marketing: billing emails still go out,# MailerLite never hears of them (notifications/consent.py).marketing_opt_out_at: datetime | None = None+ # The browser's IANA timezone, one of the signals consent.py reads.+ timezone: str | None = Noneasync def get_billing_email_recipient(@@ -1271,6 +1294,7 @@ async def get_billing_email_recipient(name=row.name,welcome_email_sent_at=row.welcomeEmailSentAt,marketing_opt_out_at=row.marketingOptOutAt,+ timezone=row.timezone,)except Exception as e:raise DatabaseError(diff --git a/autogpt_platform/backend/backend/data/user_test.py b/autogpt_platform/backend/backend/data/user_test.pyindex e631e651b4..b7bff49666 100644--- a/autogpt_platform/backend/backend/data/user_test.py+++ b/autogpt_platform/backend/backend/data/user_test.py@@ -267,15 +267,20 @@ def _consent_row(marketing_opt_out_at: datetime | None = None,marketing_opt_out_source: str | None = None,email: str | None = "[email protected]",+ timezone: str = "Europe/London",+ excluded_country: str | None = None,) -> MagicMock:- """A Prisma User row carrying only what record_signup_consent reads."""+ """A Prisma User row carrying only what record_signup_consent and the+ MailerLite checks read."""return MagicMock(id="user-1",email=email,+ timezone=timezone,termsAcceptedAt=terms_accepted_at,termsVersion=terms_version,marketingOptOutAt=marketing_opt_out_at,marketingOptOutSource=marketing_opt_out_source,+ marketingExcludedCountry=excluded_country,)@@ -924,6 +929,12 @@ class TestIsMarketingOptedOut:pytest.param(_accepted(True), True, id="opted-out"),pytest.param(_accepted(False), False, id="opted-in"),pytest.param(None, True, id="deleted-account"),+ # Seen in Iran or Russia, by a checkout or the account itself.+ pytest.param(+ _consent_row(excluded_country="RU"), True, id="checkout-in-russia"+ ),+ pytest.param(_consent_row(timezone="Asia/Tehran"), True, id="tehran"),+ pytest.param(_consent_row(email="[email protected]"), True, id="ru-address"),],)async def test_reads_the_row_not_the_cache(@@ -952,6 +963,20 @@ class TestIsMarketingOptedOut:await is_marketing_opted_out("user-1")+async def test_the_first_excluded_country_seen_is_kept():+ with patch.object(user_module, "PrismaUser") as mock_prisma_user:+ db = mock_prisma_user.prisma.return_value+ db.update_many = AsyncMock(return_value=1)++ await user_module.record_excluded_country("user-1", "RU")++ db.update_many.assert_awaited_once_with(+ where={"id": "user-1", "marketingExcludedCountry": None},+ data={"marketingExcludedCountry": "RU"},+ )++class TestGetBillingEmailRecipient:@pytest.mark.asyncio@pytest.mark.parametrize("opted_out_at", [None, CONSENTED_AT])@@ -961,6 +986,7 @@ class TestGetBillingEmailRecipient:email="[email protected]",welcomeEmailSentAt=None,marketingOptOutAt=opted_out_at,+ timezone="Europe/London",)row.name = "User"@@ -977,6 +1003,7 @@ class TestGetBillingEmailRecipient:name="User",welcome_email_sent_at=None,marketing_opt_out_at=opted_out_at,+ timezone="Europe/London",)diff --git a/autogpt_platform/backend/backend/notifications/audience_consent_recheck_test.py b/autogpt_platform/backend/backend/notifications/audience_consent_recheck_test.pyindex 7870b11b58..55bcbbaa81 100644--- a/autogpt_platform/backend/backend/notifications/audience_consent_recheck_test.py+++ b/autogpt_platform/backend/backend/notifications/audience_consent_recheck_test.py@@ -22,6 +22,7 @@ HANDLERS = {AudienceAction.SIGNUP: "record_signup",AudienceAction.CHECKOUT_OPENED: "record_checkout_opened",AudienceAction.UNSUBSCRIBE: "unsubscribe",+ AudienceAction.ONBOARDING_PROFILE: "record_onboarding_profile",}WRITES = [action for action in AudienceAction if action != AudienceAction.UNSUBSCRIBE]diff --git a/autogpt_platform/backend/backend/notifications/audience_enrichment.py b/autogpt_platform/backend/backend/notifications/audience_enrichment.pyindex a6cce00556..44afb6a18d 100644--- a/autogpt_platform/backend/backend/notifications/audience_enrichment.py+++ b/autogpt_platform/backend/backend/notifications/audience_enrichment.py@@ -12,6 +12,11 @@ write the same values:then the browser's timezone. `country_source` says which.- `exclude_de_at`: "yes" when any signal points at Germany or Austria, whosestricter marketing rules GTM skips. Once yes it is never set back to no.+- `role` and `role_other`: the role picked in the onboarding wizard, once+ there is one (`role_fields`, `data/onboarding_role.py`).++Anyone a signal places in Iran or Russia is not segmented at all: they are+never written to MailerLite (`points_at_excluded_country`, `consent.py`).The timezone and country tables come from tzdata, which zoneinfo alreadydepends on. A zone is looked up by the name the browser reported before any@@ -26,6 +31,7 @@ from collections.abc import Iterable, Mappingfrom datetime import datetimefrom backend.data.notifications import SubscriberField, SubscriptionStatus+from backend.data.onboarding_role import OnboardingRolefrom backend.notifications.subscriber_fields import Fields, mailerlite_dateYES = "yes"@@ -33,6 +39,14 @@ NO = "no"DE_AT = frozenset({"DE", "AT"})+# Nobody a signal places in these is written to MailerLite at all.+EXCLUDED_COUNTRIES = frozenset({"IR", "RU"})+# Their country-code domains, the internationalised ones in both forms. .su is+# still Russia's.+_EXCLUDED_DOMAINS = frozenset(+ {"ir", "ایران", "xn--mgba3a4f16a", "ru", "su", "рф", "xn--p1ai"}+)+# Strongest first. A held country from a stronger source is never replaced by# a weaker one; a hand-typed one with no source ranks lowest.COUNTRY_SOURCES = ("stripe", "ip", "timezone")@@ -190,6 +204,39 @@ def email_points_at_de_at(email: str) -> bool:return email_domain(email).rsplit(".", 1)[-1] in ("de", "at")+def points_at_excluded_country(+ *, email: str, timezone: str | None, countries: Iterable[str | None] = ()+) -> bool:+ """Whether any signal places the account in Iran or Russia: a country it+ was seen in (the Stripe billing address, the visitor's IP country, the one+ a checkout recorded on the account), the browser's timezone, or the+ address's country-code domain. Any signal, as for exclude_de_at: a billing+ address elsewhere does not outweigh a Moscow timezone."""+ if excluded_country([*countries, timezone_country(timezone)]):+ return True+ return email_domain(email).rsplit(".", 1)[-1] in _EXCLUDED_DOMAINS+++def excluded_country(countries: Iterable[str | None]) -> str | None:+ """The first of `countries` that MailerLite must never hold, as its code."""+ return next(+ (code for c in countries if (code := country_code(c)) in EXCLUDED_COUNTRIES),+ None,+ )+++def billing_country(session: dict) -> str | None:+ """The billing address country a Stripe Checkout Session collected."""+ address = (session.get("customer_details") or {}).get("address") or {}+ return address.get("country")+++def role_fields(role: OnboardingRole) -> Fields:+ """The label people saw, and what they typed after picking Other, which+ is cleared for any other pick."""+ return {SubscriberField.ROLE: role.label, SubscriberField.ROLE_OTHER: role.other}++def signin_method(providers: Iterable[str]) -> str | None:"""How the account signs in, from its auth provider ids: google wins overa password, since it proves the address."""@@ -211,9 +258,11 @@ def checkout_fields(stripe_country: str | None = None,ip_country: str | None = None,status: str = SubscriptionStatus.SIGNED.value,+ role: OnboardingRole | None = None,) -> Fields:"""Every field a checkout opener gets. Merge it with what MailerLite holds- (`merge_with_held`) before writing."""+ (`merge_with_held`) before writing. Without a `role` picked yet, the role+ MailerLite holds is left alone."""tz_country = timezone_country(timezone)candidates = {"stripe": country_code(stripe_country),@@ -233,6 +282,8 @@ def checkout_fields(method = signin_method(signin_providers)if method:fields[SubscriberField.SIGNIN_METHOD] = method+ if role:+ fields.update(role_fields(role))for source in COUNTRY_SOURCES:code = candidates[source]if code:diff --git a/autogpt_platform/backend/backend/notifications/audience_enrichment_test.py b/autogpt_platform/backend/backend/notifications/audience_enrichment_test.pyindex c29d9253a0..d863b8c265 100644--- a/autogpt_platform/backend/backend/notifications/audience_enrichment_test.py+++ b/autogpt_platform/backend/backend/notifications/audience_enrichment_test.py@@ -1,12 +1,14 @@"""What GTM segments checkout openers on: country from the strongest source,-Germany and Austria caught by any signal and never released, and nothing-MailerLite holds made worse by a later or weaker write."""+Germany and Austria caught by any signal and never released, the onboarding+role, and nothing MailerLite holds made worse by a later or weaker write. Iran+and Russia are caught by any signal too, and never written at all."""from datetime import UTC, datetimeimport pytestfrom backend.data.notifications import SubscriberField+from backend.data.onboarding_role import OnboardingRolefrom backend.notifications import audience_enrichment as enrichmentCREATED = datetime(2026, 7, 14, 9, 0, tzinfo=UTC)@@ -89,6 +91,70 @@ def test_a_de_or_at_address_counts_whatever_sits_under_it(email, de_at):assert enrichment.email_points_at_de_at(email) is de_at+ "signals",+ [+ dict(timezone="Europe/Moscow"),+ dict(timezone="Asia/Vladivostok"),+ dict(timezone="Asia/Tehran"),+ # An old name a browser may still report.+ dict(timezone="Iran"),+ dict(countries=("RU",)),+ dict(countries=(None, " ir ")),+ dict(email="[email protected]"),+ dict(email="[email protected]"),+ dict(email="[email protected]"),+ dict(email="sam@почта.рф"),+ dict(email="[email protected]"),+ # A billing address elsewhere does not outweigh a Moscow timezone.+ dict(countries=("US",), timezone="Europe/Moscow"),+ ],+)+def test_any_iran_or_russia_signal_excludes(signals):+ signals = {"email": "[email protected]", "timezone": "America/Chicago", **signals}+ assert enrichment.points_at_excluded_country(**signals)+++ "signals",+ [+ dict(),+ dict(timezone="Europe/Kiev"),+ dict(timezone="Europe/Minsk"),+ dict(countries=("UA", "XX", None)),+ dict(email="[email protected]"),+ dict(email="[email protected]"),+ ],+)+def test_neighbours_and_lookalikes_are_not_excluded(signals):+ signals = {"email": "[email protected]", "timezone": "America/Chicago", **signals}+ assert not enrichment.points_at_excluded_country(**signals)+++def test_the_billing_country_comes_from_the_checkout_session():+ session = {"customer_details": {"address": {"country": "RU"}}}+ assert enrichment.billing_country(session) == "RU"+ assert enrichment.billing_country({"customer_details": None}) is None+ assert enrichment.billing_country({}) is None+++ "role, fields",+ [+ (+ OnboardingRole(choice="Sales/BD"),+ {SubscriberField.ROLE: "Sales / BD", SubscriberField.ROLE_OTHER: None},+ ),+ (+ OnboardingRole(choice="Other", other="Dentist"),+ {SubscriberField.ROLE: "Other", SubscriberField.ROLE_OTHER: "Dentist"},+ ),+ ],+)+def test_the_role_is_written_as_labelled_with_others_text(role, fields):+ assert enrichment.role_fields(role) == fields++@pytest.mark.parametrize("providers, method",[@@ -143,6 +209,18 @@ def test_country_comes_from_the_strongest_source(sources, country, source):assert fields[SubscriberField.COUNTRY_SOURCE] == source+def test_a_picked_role_rides_on_the_checkout():+ fields = _fields(role=OnboardingRole(choice="HR/People"))+ assert fields[SubscriberField.ROLE] == "HR / People"+ assert fields[SubscriberField.ROLE_OTHER] is None+++def test_without_a_pick_the_role_mailerlite_holds_is_left_alone():+ fields = _fields()+ assert SubscriberField.ROLE not in fields+ assert SubscriberField.ROLE_OTHER not in fields++def test_without_any_country_signal_the_country_is_left_alone():fields = _fields(timezone="UTC", signin_providers=[])for field in (diff --git a/autogpt_platform/backend/backend/notifications/checkout_audience_test.py b/autogpt_platform/backend/backend/notifications/checkout_audience_test.pyindex 9a6c19b7bc..523baa2e67 100644--- a/autogpt_platform/backend/backend/notifications/checkout_audience_test.py+++ b/autogpt_platform/backend/backend/notifications/checkout_audience_test.py@@ -1,6 +1,7 @@"""The notification service puts a checkout opener in the checkout openersgroup with the fields GTM segments on, and a later or weaker event never-makes MailerLite's copy worse."""+makes MailerLite's copy worse. The role they answer afterwards fills in that+subscriber and never creates one."""from datetime import UTC, datetimefrom types import SimpleNamespace@@ -9,8 +10,9 @@ from unittest.mock import AsyncMock, MagicMock, patchimport pytestfrom backend.data.notifications import AudienceAction+from backend.data.onboarding_role import OnboardingRolefrom backend.notifications import mailerlite, subscriber_fields-from backend.notifications.audience_enrichment import checkout_fields+from backend.notifications.audience_enrichment import checkout_fields, role_fieldsfrom backend.notifications.notifications import NotificationManagerEMAIL = "[email protected]"@@ -175,6 +177,56 @@ async def test_the_billing_country_upgrades_a_timezone_guess(mailerlite_configurassert ml.subscribers[EMAIL]["exclude_de_at"] == "yes"+async def test_a_checkout_opener_gets_the_role_they_picked(mailerlite_configured):+ ml = _FakeMailerLite()+ with patch.object(mailerlite, "_client", return_value=ml):+ await _consume(_event(role=OnboardingRole(choice="Other", other="Dentist")))+ assert ml.subscribers[EMAIL]["role"] == "Other"+ assert ml.subscribers[EMAIL]["role_other"] == "Dentist"+++# ── the onboarding profile ─────────────────────────────────────────────────+++def _profile(role: OnboardingRole):+ return subscriber_fields.audience_event(+ AudienceAction.ONBOARDING_PROFILE, EMAIL, "user-1", role_fields(role)+ )+++async def test_the_role_picked_after_checkout_fills_in_the_opener(+ mailerlite_configured,+):+ """On cloud the paywall comes first: the opener is already a subscriber+ when they pick, and only the role is written. A switch away from Other+ clears its text."""+ held = {"subscription_status": "in_trial", "role": "Other", "role_other": "x"}+ ml = _FakeMailerLite({EMAIL: held})+ with patch.object(mailerlite, "_client", return_value=ml):+ await _consume(_profile(OnboardingRole(choice="Marketing")))+ assert ml.subscribers[EMAIL] == {+ "subscription_status": "in_trial",+ "role": "Marketing",+ "role_other": None,+ }+ assert ml.posts == [+ {"email": EMAIL, "fields": {"role": "Marketing", "role_other": None}}+ ]+++async def test_the_role_never_creates_a_subscriber(mailerlite_configured):+ """Only checkout openers belong in MailerLite; anyone who picks first+ gets their role with the checkout event."""+ ml = _FakeMailerLite()+ with patch.object(mailerlite, "_client", return_value=ml):+ await _consume(_profile(OnboardingRole(choice="Marketing")))+ assert ml.posts == []+ assert EMAIL not in ml.subscribers++@pytest.mark.asyncioasync def test_every_audience_action_has_a_handler(monkeypatch):"""An action the consumer cannot route would raise on every delivery."""@@ -190,6 +242,7 @@ async def test_every_audience_action_has_a_handler(monkeypatch):"record_signup","record_checkout_opened","unsubscribe",+ "record_onboarding_profile",):handlers[name] = AsyncMock()monkeypatch.setattr(mailerlite, name, handlers[name])diff --git a/autogpt_platform/backend/backend/notifications/checkout_backfill.py b/autogpt_platform/backend/backend/notifications/checkout_backfill.pyindex 66a1e919d4..90ad6585a9 100644--- a/autogpt_platform/backend/backend/notifications/checkout_backfill.py+++ b/autogpt_platform/backend/backend/notifications/checkout_backfill.py@@ -4,8 +4,8 @@ fields GTM segments on (`audience_enrichment`).Only accounts with at least one Stripe Checkout Session are openers. Nobodyelse is written, so the accounts that never reached checkout stay out of-MailerLite. An opener who opted out of marketing stays out too (`consent.py`):-they are counted, never planned.+MailerLite. An opener who opted out of marketing, or whom a signal places in+Iran or Russia, stays out too (`consent.py`): they are counted, never planned.Each person is written with one subscriber upsert, one at a time and paced,never a /batch: MailerLite processes an upsert-only batch as an import, and@@ -29,7 +29,7 @@ from pydantic import BaseModelfrom backend.data.notifications import SubscriberFieldfrom backend.notifications.audience_enrichment import checkout_fields, merge_with_held-from backend.notifications.consent import marketing_allowed+from backend.notifications.consent import KeptOut, marketing_allowedfrom backend.notifications.mailerlite import (API_BASE,_client,@@ -86,6 +86,8 @@ class OpenerPlan(BaseModel):invalid: int# Openers who refused marketing, left out of the plan and the tallies.opted_out: int+ # Openers a signal places in Iran or Russia, left out the same way.+ excluded_country: int# What the openers end up with, for the report.country_sources: dict[str, int]countries: dict[str, int]@@ -114,7 +116,7 @@ def plan(openers: list[Opener], current: Current, members: Mapping[str, str]) -> OpenerPlan:changes: list[OpenerChange] = []- invalid = opted_out = 0+ invalid = opted_out = excluded_country = 0sources: Counter[str] = Counter()countries: Counter[str] = Counter()email_types: Counter[str] = Counter()@@ -125,6 +127,9 @@ def plan(opted_out += 1continueemail = opener.person.email+ if opener.person.placed_in_excluded_country(opener.stripe_country):+ excluded_country += 1+ continueif not _valid(email):invalid += 1continue@@ -150,6 +155,7 @@ def plan(openers=len(openers),invalid=invalid,opted_out=opted_out,+ excluded_country=excluded_country,country_sources=dict(sources),countries=dict(countries),email_types=dict(email_types),@@ -180,15 +186,18 @@ async def apply(on_progress: Callable[[int, int], None] | None = None,*,refresh: Callable[[str], Awaitable[list[Subscription]]] | None = None,+ kept_out: KeptOut | None = None,) -> tuple[int, int, int]:- """Visit each planned opener: read them again from MailerLite and, with- `refresh`, their subscriptions again from Stripe, and write what they- still need as one subscriber upsert into the group. The status the plan- took from Stripe hours earlier is never written over a newer one.+ """Visit each planned opener: with `kept_out`, check they may still enter+ MailerLite, since they may have opted out or been seen in Iran or Russia+ since the plan; read them again from MailerLite and, with `refresh`,+ their subscriptions again from Stripe; and write what they still need as+ one subscriber upsert into the group. The status the plan took from+ Stripe hours earlier is never written over a newer one.Returns (succeeded, failed, skipped); a failure is logged withMailerLite's reason and left for the next run, and someone who needs- nothing any more is skipped."""+ nothing any more, or may no longer be written, is skipped."""succeeded = failed = skipped = 0for index, change in enumerate(changes):if index:@@ -198,6 +207,9 @@ async def apply(email = change.opener.person.emailopener = change.openertry:+ if kept_out and await kept_out(opener.person.user_id):+ skipped += 1+ continueif refresh and opener.person.stripe_customer_id:person = opener.person.model_copy(update={diff --git a/autogpt_platform/backend/backend/notifications/checkout_backfill_test.py b/autogpt_platform/backend/backend/notifications/checkout_backfill_test.pyindex 574b748785..d3beb41033 100644--- a/autogpt_platform/backend/backend/notifications/checkout_backfill_test.py+++ b/autogpt_platform/backend/backend/notifications/checkout_backfill_test.py@@ -26,6 +26,7 @@ def _opener(email="[email protected]", subscriptions=None, **kwargs):stripe_customer_id="cus_1",timezone=kwargs.pop("timezone", "Asia/Kolkata"),marketing_opt_out_at=kwargs.pop("opted_out_at", None),+ excluded_country=kwargs.pop("excluded_country", None),),opened_at=OPENED,signin_providers=["google"],@@ -248,7 +249,7 @@ async def test_a_failed_write_is_counted_and_the_run_goes_on(monkeypatch, caplogACTIVE = [Subscription(id="sub_1", status="active", start_date=OPENED)]-async def _apply_with(monkeypatch, plan, *, held, refresh):+async def _apply_with(monkeypatch, plan, *, held, refresh, kept_out=None):monkeypatch.setattr(checkout_backfill.asyncio, "sleep", AsyncMock())client = MagicMock(post=AsyncMock(return_value=MagicMock(status=200)))with (@@ -258,7 +259,7 @@ async def _apply_with(monkeypatch, plan, *, held, refresh):),):result = await checkout_backfill.apply(- plan.changes, "grp_checkout", refresh=refresh+ plan.changes, "grp_checkout", refresh=refresh, kept_out=kept_out)return result, client@@ -342,3 +343,53 @@ async def test_an_opted_out_opener_is_never_written(monkeypatch):assert result == (1, 0, 0)written = [c.kwargs["json"]["email"] for c in client.post.await_args_list]assert written == ["[email protected]"]+++ "excluded",+ [+ _opener(email="[email protected]", stripe_country="RU", timezone="Europe/London"),+ _opener(email="[email protected]", timezone="Asia/Tehran"),+ _opener(email="[email protected]"),+ _opener(email="[email protected]", excluded_country="IR"),+ ],+ ids=["billing", "timezone", "email", "recorded-by-a-checkout"],+)+def test_an_opener_placed_in_iran_or_russia_is_counted_but_never_planned(excluded):+ email = excluded.person.email+ plan = checkout_backfill.plan(+ [excluded, _opener()], current={email: {}}, members={}+ )+ assert [c.opener.person.email for c in plan.changes] == ["[email protected]"]+ assert plan.openers == 2+ assert plan.excluded_country == 1+ assert plan.opted_out == 0+ assert plan.countries == {"IN": 1}+++async def test_an_opener_who_may_no_longer_be_written_is_skipped(monkeypatch):+ """The plan is old by the time a paced run reaches someone: an opt-out or+ an Iranian or Russian checkout since then is read right before the write."""+ plan = checkout_backfill.plan([_opener()], current={}, members={})+ kept_out = AsyncMock(return_value=True)+ result, client = await _apply_with(+ monkeypatch, plan, held=None, refresh=None, kept_out=kept_out+ )+ assert result == (0, 0, 1)+ kept_out.assert_awaited_once_with("user-1")+ client.post.assert_not_awaited()+++async def test_an_unreadable_account_is_retried_later_not_written(monkeypatch):+ plan = checkout_backfill.plan([_opener()], current={}, members={})+ result, client = await _apply_with(+ monkeypatch,+ plan,+ held=None,+ refresh=None,+ kept_out=AsyncMock(side_effect=RuntimeError("db down")),+ )+ assert result == (0, 1, 0)+ client.post.assert_not_awaited()diff --git a/autogpt_platform/backend/backend/notifications/consent.py b/autogpt_platform/backend/backend/notifications/consent.pyindex 8991df1efa..bb873343ad 100644--- a/autogpt_platform/backend/backend/notifications/consent.py+++ b/autogpt_platform/backend/backend/notifications/consent.py@@ -2,18 +2,27 @@Someone who refused marketing (on the signup page; later, by unsubscribing froman email or in settings) is never written to MailerLite: no subscriber, no-group, no fields, and no "opted out" placeholder either. Every MailerLite write-upserts the subscriber, a group removal or a field update included (see-`mailerlite.py`), so every path that queues an audience change asks-`marketing_allowed` first, and the backfills leave such accounts out of their-plan. Billing and account emails are service mail and are not affected.+group, no fields, and no "opted out" placeholder either. Nor is anyone a signal+places in Iran or Russia (`audience_enrichment.points_at_excluded_country`).+Every MailerLite write upserts the subscriber, a group removal or a field update+included (see `mailerlite.py`), so every path that queues an audience change asks+`audience_change_allowed` first, and the backfills leave such accounts out of+their plan. Billing and account emails are service mail and are not affected.++An IP or billing country is only seen at checkout, so a checkout that sees an+Iranian or Russian one records it on the account (`User.marketingExcludedCountry`).+The audience consumer re-reads all of it right before every write+(`data.user.is_marketing_opted_out`), which also stops what a later trial or+billing event queues without that country."""import logging+from collections.abc import Awaitable, Callable, Iterablefrom datetime import datetimefrom typing import Protocolfrom backend.data.notifications import AudienceAction+from backend.notifications.audience_enrichment import points_at_excluded_countryfrom backend.notifications.mailerlite import pseudonymlogger = logging.getLogger(__name__)@@ -21,6 +30,11 @@ logger = logging.getLogger(__name__)# Never a stored value, so a missing attribute cannot pass for an unset one._MISSING = object()+# Whether an account, by id, must stay out of MailerLite now, read fresh from+# the database (`data.user.is_marketing_opted_out`). The backfills call it+# right before each write, since a run lasts hours and their plan is older.+KeptOut = Callable[[str], Awaitable[bool]]+class MarketingConsent(Protocol):@property@@ -36,6 +50,9 @@ class MarketingContact(MarketingConsent, Protocol):@propertydef email(self) -> str: ...+ @property+ def timezone(self) -> str | None: ...+def marketing_allowed(user: MarketingConsent) -> bool:# Shared-cache entries can outlive a rolling deploy. A user pickled by the@@ -46,12 +63,23 @@ def marketing_allowed(user: MarketingConsent) -> bool:return getattr(user, "marketing_opt_out_at", _MISSING) is None-def audience_change_allowed(user: MarketingContact, action: AudienceAction) -> bool:- """Whether `action` may be queued for this account. A refusal is logged."""- if marketing_allowed(user):- return True- log_opted_out_skip(user.email, action.value)- return False+def audience_change_allowed(+ user: MarketingContact,+ action: AudienceAction,+ countries: Iterable[str | None] = (),+) -> bool:+ """Whether `action` may be queued for this account. `countries` are the+ ones the caller saw it in, such as the visitor's IP country or a Stripe+ billing address. A refusal is logged."""+ if not marketing_allowed(user):+ log_opted_out_skip(user.email, action.value)+ return False+ if points_at_excluded_country(+ email=user.email, timezone=user.timezone, countries=countries+ ):+ log_excluded_skip(user.email, action.value)+ return False+ return Truedef log_opted_out_skip(email: str, what: str) -> None:@@ -59,3 +87,22 @@ def log_opted_out_skip(email: str, what: str) -> None:logger.debug("Skipping MailerLite %s for %s: opted out of marketing", what, pseudonym(email))+++def log_excluded_skip(email: str, what: str) -> None:+ """Like an opt-out: at debug level and by pseudonym."""+ logger.debug(+ "Skipping MailerLite %s for %s: placed in an excluded country",+ what,+ pseudonym(email),+ )+++def log_kept_out_skip(email: str, what: str) -> None:+ """For the consumer's last check, which can't tell which rule applied."""+ logger.debug(+ "Skipping MailerLite %s for %s: opted out of marketing or placed in an "+ "excluded country",+ what,+ pseudonym(email),+ )diff --git a/autogpt_platform/backend/backend/notifications/consent_test.py b/autogpt_platform/backend/backend/notifications/consent_test.pyindex 3fc0b9638a..bd0f6707a1 100644--- a/autogpt_platform/backend/backend/notifications/consent_test.py+++ b/autogpt_platform/backend/backend/notifications/consent_test.py@@ -1,5 +1,6 @@-"""Someone who refused marketing never enters MailerLite, and the skip is-logged at debug level by pseudonym, never by address."""+"""Someone who refused marketing never enters MailerLite, nor does anyone a+signal places in Iran or Russia, and the skip is logged at debug level by+pseudonym, never by address."""import loggingimport pickle@@ -17,9 +18,14 @@ EMAIL = "[email protected]"OPTED_OUT = datetime(2026, 10, 2, 12, 0, tzinfo=UTC)-def _user(opted_out_at: datetime | None) -> BillingEmailRecipient:+def _user(+ opted_out_at: datetime | None, email: str = EMAIL, timezone: str | None = None+) -> BillingEmailRecipient:return BillingEmailRecipient(- id="user-1", email=EMAIL, marketing_opt_out_at=opted_out_at+ id="user-1",+ email=email,+ marketing_opt_out_at=opted_out_at,+ timezone=timezone,)@@ -77,4 +83,33 @@ def test_a_refused_change_is_logged_by_pseudonym_at_debug(caplog, action):def test_the_skip_never_reaches_an_info_log(caplog):with caplog.at_level(logging.INFO, logger=consent.__name__):consent.log_opted_out_skip(EMAIL, "checkout_opened")+ consent.log_excluded_skip(EMAIL, "checkout_opened")assert caplog.records == []+++ "user, countries",+ [+ (_user(None, timezone="Europe/Moscow"), ()),+ (_user(None, email="[email protected]"), ()),+ # Only the caller saw it: the visitor's IP, or a billing address.+ (_user(None, timezone="America/Chicago"), (None, "IR")),+ ],+ ids=["timezone", "email", "seen-by-the-caller"],+)[email protected]("action", list(AudienceAction))+def test_iran_or_russia_is_refused_and_logged_by_pseudonym(+ caplog, user, countries, action+):+ with caplog.at_level(logging.DEBUG, logger=consent.__name__):+ assert not consent.audience_change_allowed(user, action, countries)+ (record,) = caplog.records+ assert record.levelno == logging.DEBUG+ assert pseudonym(user.email) in record.getMessage()+ assert action.value in record.getMessage()+ assert user.email not in caplog.text+++def test_a_country_elsewhere_is_allowed():+ user = _user(None, timezone="Europe/Kiev")+ assert consent.audience_change_allowed(user, AudienceAction.ADD_TRIAL, ("UA",))diff --git a/autogpt_platform/backend/backend/notifications/lifecycle.py b/autogpt_platform/backend/backend/notifications/lifecycle.pyindex 4e7ba14620..ddbc50aee4 100644--- a/autogpt_platform/backend/backend/notifications/lifecycle.py+++ b/autogpt_platform/backend/backend/notifications/lifecycle.py@@ -9,7 +9,8 @@ Trial enrollment and conversion notices are handled separately in `trial.py`.A customer who opted out of marketing still gets every one of these emails;only the MailerLite audience changes that ride along are skipped for them-(`consent.py`).+(`consent.py`). So does one a signal places in Iran or Russia, the checkout's+billing address included."""import logging@@ -32,6 +33,7 @@ from backend.data.notifications import (from backend.data.stripe_client import stripe_callfrom backend.data.user import BillingEmailRecipientfrom backend.notifications import subscriber_fields+from backend.notifications.audience_enrichment import billing_countryfrom backend.notifications.consent import audience_change_allowedfrom backend.notifications.dedupe import claim_once, release_claimfrom backend.notifications.lifecycle_plan import (@@ -116,9 +118,10 @@ async def on_checkout_completed(session: dict, subscription: dict) -> None:if user is None:returnfields = subscriber_fields.subscribed(subscription.get("start_date"))+ countries = (billing_country(session),)if user.welcome_email_sent_at is not None:- if not audience_change_allowed(user, AudienceAction.ADD_CHANGELOG):+ if not audience_change_allowed(user, AudienceAction.ADD_CHANGELOG, countries):returnevent = audience_event(AudienceAction.ADD_CHANGELOG, user.email, user.id, fields@@ -158,7 +161,7 @@ async def on_checkout_completed(session: dict, subscription: dict) -> None:await _release_welcome(user)raise- if not audience_change_allowed(user, AudienceAction.ENROLL_TOUR):+ if not audience_change_allowed(user, AudienceAction.ENROLL_TOUR, countries):return# Must not propagate: the welcome is already out and the claim is durable,# so a Stripe retry would take the returning-customer branch and enrol themdiff --git a/autogpt_platform/backend/backend/notifications/lifecycle_test.py b/autogpt_platform/backend/backend/notifications/lifecycle_test.pyindex 92e5651400..0be7d6852b 100644--- a/autogpt_platform/backend/backend/notifications/lifecycle_test.py+++ b/autogpt_platform/backend/backend/notifications/lifecycle_test.py@@ -33,12 +33,13 @@ INVOICE_PLAN_PATCH = "backend.notifications.lifecycle.plan_from_invoice"class _User:"""Shaped like `BillingEmailRecipient`, which is what the RPC returns."""- def __init__(self, welcome_sent_at=None, opted_out_at=None):+ def __init__(self, welcome_sent_at=None, opted_out_at=None, timezone=None):self.id = "user-1"self.email = "[email protected]"self.name = "Sam Carter"self.welcome_email_sent_at = welcome_sent_atself.marketing_opt_out_at = opted_out_at+ self.timezone = timezonedef _subscription(**over) -> dict:@@ -773,3 +774,43 @@ async def test_an_opted_out_churn_emails_but_leaves_mailerlite_alone():queued = calls["notify"].await_args.args[0]assert queued.type is NotificationType.SUBSCRIPTION_ENDEDcalls["audience"].assert_not_called()+++# ── placed in Iran or Russia ───────────────────────────────────────────────+#+# Every billing email still goes out; MailerLite never hears of them.++[email protected]("welcomed", [False, True], ids=["first", "returning"])+async def test_a_checkout_billed_to_russia_is_welcomed_but_not_enrolled(welcomed):+ user = _User(+ welcome_sent_at=datetime(2026, 1, 1, tzinfo=timezone.utc) if welcomed else None+ )+ session = {+ "customer": CUSTOMER,+ "customer_details": {"address": {"country": "RU"}},+ }+ with patch(+ "backend.notifications.lifecycle._claim_welcome",+ AsyncMock(return_value=True),+ ):+ calls = await _run(+ lambda: lifecycle.on_checkout_completed(session, _subscription()),+ user,+ )+ if not welcomed:+ queued = calls["notify"].await_args.args[0]+ assert queued.type is NotificationType.SUBSCRIPTION_WELCOME+ calls["audience"].assert_not_called()+++async def test_a_churn_in_tehran_emails_but_leaves_mailerlite_alone():+ calls = await _run(+ lambda: lifecycle.on_subscription_deleted(_subscription()),+ _User(timezone="Asia/Tehran"),+ )+ queued = calls["notify"].await_args.args[0]+ assert queued.type is NotificationType.SUBSCRIPTION_ENDED+ calls["audience"].assert_not_called()diff --git a/autogpt_platform/backend/backend/notifications/mailerlite.py b/autogpt_platform/backend/backend/notifications/mailerlite.pyindex ba3cced75f..5267e40364 100644--- a/autogpt_platform/backend/backend/notifications/mailerlite.py+++ b/autogpt_platform/backend/backend/notifications/mailerlite.py@@ -19,13 +19,13 @@ transition:checkout joins it, and nobody else enters MailerLite through us: a signupalone does not create a subscriber. GTM segments it for outreach.-Someone who opted out of marketing enters none of these. Every write below-upserts the subscriber, a removal or a field update included, so nothing is-queued for them at all (`consent.py`), the backfills leave them out, and the-consumer re-reads the opt-out right before each write, which drops a change-queued just before the refusal. The one exception is `unsubscribe`, which-carries the refusal itself to someone MailerLite already has, and never-creates a subscriber.+Someone who opted out of marketing, or whom a signal places in Iran or Russia,+enters none of these. Every write below upserts the subscriber, a removal or a+field update included, so nothing is queued for them at all (`consent.py`) and+the backfills leave them out. The consumer also re-reads the opt-out right+before each write, which drops a change queued just before the refusal. The+one exception is `unsubscribe`, which carries the refusal itself to someone+MailerLite already has, and never creates a subscriber.Subscriber fields (`SubscriberField`) are the backend's alone: every writecomes from here, and MailerLite automations only read them.@@ -81,6 +81,8 @@ FIELD_TYPES: dict[SubscriberField, str] = {SubscriberField.CHECKOUT_OPENED: "date",SubscriberField.EMAIL_TYPE: "text",SubscriberField.SIGNIN_METHOD: "text",+ SubscriberField.ROLE: "text",+ SubscriberField.ROLE_OTHER: "text",SubscriberField.COUNTRY_CODE: "text",SubscriberField.COUNTRY_SOURCE: "text",SubscriberField.EXCLUDE_DE_AT: "text",@@ -193,6 +195,26 @@ async def record_signup(email: str, fields: Fields | None = None) -> None:await update_fields(email, fields)+async def record_onboarding_profile(email: str, fields: Fields | None = None) -> None:+ """The onboarding answers. On cloud the paywall comes first, so they are+ given once the person is a checkout opener: they fill in someone MailerLite+ already has and never create a subscriber, since only checkout openers+ belong in MailerLite. Anyone who answers before opening checkout gets their+ role with the checkout event instead (`checkout_audience`), and+ onboarding-role-backfill fills in anyone missed."""+ if not fields:+ return+ _require_token()+ if await _find_subscriber(email) is None:+ logger.info(+ "No MailerLite subscriber for %s; onboarding profile not written",+ pseudonym(email),+ )+ return+ await update_fields(email, fields)+ logger.info(f"Wrote the onboarding profile of {pseudonym(email)} to MailerLite")++async def record_checkout_opened(email: str, fields: Fields | None = None) -> None:"""Someone opened Stripe checkout: into the checkout openers group, withthe fields GTM segments on. Whatever MailerLite already holds that isdiff --git a/autogpt_platform/backend/backend/notifications/mailerlite_backfill.py b/autogpt_platform/backend/backend/notifications/mailerlite_backfill.pyindex 7139bf0428..5c76facea9 100644--- a/autogpt_platform/backend/backend/notifications/mailerlite_backfill.py+++ b/autogpt_platform/backend/backend/notifications/mailerlite_backfill.py@@ -10,8 +10,9 @@ the tour group is left alone: they are either mid-tour, and must not get thechangelog yet, or they finished it and MailerLite's automation has alreadymoved them across.-A customer who opted out of marketing is left out entirely, removals-included: they never enter MailerLite (`consent.py`).+A customer who opted out of marketing, or whom a signal places in Iran or+Russia, is left out entirely, removals included: they never enter MailerLite+(`consent.py`).Idempotent by construction: current membership is read first, so a second runfinds nothing to do and a failed call is simply picked up by the next run.@@ -27,7 +28,8 @@ from urllib.parse import urlencodefrom pydantic import BaseModel-from backend.notifications.consent import marketing_allowed+from backend.notifications.audience_enrichment import points_at_excluded_country+from backend.notifications.consent import KeptOut, marketing_allowedfrom backend.notifications.mailerlite import (API_BASE,MailerLiteError,@@ -72,6 +74,7 @@ class Decision(str, Enum):SKIP_NO_TRIAL_GROUP = "skip_no_trial_group"SKIP_UNSETTLED = "skip_unsettled"SKIP_OPTED_OUT = "skip_opted_out"+ SKIP_EXCLUDED_COUNTRY = "skip_excluded_country"ALREADY_CORRECT = "already_correct"@@ -109,6 +112,11 @@ class Customer(BaseModel):subscriptions: list[Subscription]# Set when they refused marketing: they must never enter MailerLite.marketing_opt_out_at: datetime | None = None+ # The browser's IANA timezone, the Stripe billing address country, and+ # the country a checkout recorded: any may place them in Iran or Russia.+ timezone: str | None = None+ billing_country: str | None = None+ excluded_country: str | None = Noneclass Audience(BaseModel):@@ -129,6 +137,8 @@ class PlannedChange(BaseModel):class ApplyResult(BaseModel):succeeded: dict[Decision, int]failed: dict[Decision, int]+ # Opted out or seen in Iran or Russia since the plan: not written.+ skipped: int = 0class BatchAnswer(BaseModel):@@ -169,6 +179,16 @@ def decide(return PlannedChange(customer=customer, standing=standing, decisions=[Decision.SKIP_OPTED_OUT])+ if points_at_excluded_country(+ email=customer.email,+ timezone=customer.timezone,+ countries=(customer.billing_country, customer.excluded_country),+ ):+ return PlannedChange(+ customer=customer,+ standing=standing,+ decisions=[Decision.SKIP_EXCLUDED_COUNTRY],+ )email = customer.email.strip().lower()in_tour = email in audience.tourin_changelog = email in audience.changelog@@ -214,7 +234,16 @@ async def read_audience() -> Audience:)-async def apply(changes: list[PlannedChange], audience: Audience) -> ApplyResult:+async def apply(+ changes: list[PlannedChange],+ audience: Audience,+ *,+ kept_out: KeptOut | None = None,+) -> ApplyResult:+ """With `kept_out`, each batch first drops anyone who may no longer be+ written: they opted out or were seen in Iran or Russia since the plan.+ Someone whose account can't be read then is counted as failed and left+ for the next run; the run goes on."""result = ApplyResult(succeeded={d: 0 for d in CHANGES}, failed={d: 0 for d in CHANGES})@@ -227,6 +256,10 @@ async def apply(changes: list[PlannedChange], audience: Audience) -> ApplyResultfor index, (decision, chunk) in enumerate(batches):if index:await asyncio.sleep(_interval_before(decision))+ if kept_out:+ chunk = await _writable(chunk, decision, result, kept_out)+ if not chunk:+ continueanswers = await _send_batch([_call_for(decision, c.customer.email, audience) for c in chunk])@@ -235,6 +268,32 @@ async def apply(changes: list[PlannedChange], audience: Audience) -> ApplyResultreturn result+async def _writable(+ chunk: list[PlannedChange],+ decision: Decision,+ result: ApplyResult,+ kept_out: KeptOut,+) -> list[PlannedChange]:+ """The batch without anyone `kept_out` rejects (counted as skipped) or+ whose account it can't read (counted as failed)."""+ allowed: list[PlannedChange] = []+ for change in chunk:+ try:+ if await kept_out(change.customer.user_id):+ result.skipped += 1+ continue+ except Exception:+ result.failed[decision] += 1+ logger.warning(+ f"Re-reading the account of "+ f"{_refusal(change.customer.email, BatchAnswer(code=0))} failed; "+ "the next run retries it"+ )+ continue+ allowed.append(change)+ return allowed++def _interval_before(decision: Decision) -> float:if decision in (Decision.ADD_CHANGELOG, Decision.ADD_TRIAL):return UPSERT_BATCH_INTERVAL_SECONDSdiff --git a/autogpt_platform/backend/backend/notifications/mailerlite_backfill_test.py b/autogpt_platform/backend/backend/notifications/mailerlite_backfill_test.pyindex 2864b973ba..1f76de0d62 100644--- a/autogpt_platform/backend/backend/notifications/mailerlite_backfill_test.py+++ b/autogpt_platform/backend/backend/notifications/mailerlite_backfill_test.py@@ -468,3 +468,88 @@ async def test_apply_never_writes_an_opted_out_customer(]assert sum(result.succeeded.values()) == 1assert sum(result.failed.values()) == 0+++# ── placed in Iran or Russia ───────────────────────────────────────────────+++ "customer",+ [+ _customer("[email protected]", "active").model_copy(update={"timezone": "Asia/Tehran"}),+ _customer("[email protected]", "active"),+ _customer("[email protected]", "canceled").model_copy(+ update={"timezone": "Europe/Moscow"}+ ),+ _customer("[email protected]", "active").model_copy(update={"billing_country": "RU"}),+ _customer("[email protected]", "trialing").model_copy(update={"excluded_country": "IR"}),+ ],+ ids=[+ "paying-in-tehran",+ "russian-address",+ "churned-in-moscow",+ "billed-to-russia",+ "recorded-by-a-checkout",+ ],+)+def test_a_customer_placed_in_iran_or_russia_is_only_ever_skipped(customer):+ audience = _audience(changelog=[customer.email])+ change = mailerlite_backfill.decide(customer, audience, trial_enabled=True)+ assert change.decisions == [Decision.SKIP_EXCLUDED_COUNTRY]+ assert Decision.SKIP_EXCLUDED_COUNTRY not in mailerlite_backfill.CHANGES+++async def test_apply_drops_anyone_who_may_no_longer_be_written(+ configured, no_sleep, monkeypatch+):+ """The plan is old by the last batch: someone who opted out or was seen+ in Iran or Russia since then is checked out right before the write."""+ client = MagicMock()+ client.post = AsyncMock(+ side_effect=lambda url, **kw: _batch_ok(kw["json"]["requests"])+ )+ monkeypatch.setattr(mailerlite_backfill, "_client", lambda: client)+ audience = _audience()+ changes = mailerlite_backfill.plan(+ [_customer("[email protected]", "active"), _customer("[email protected]", "active")], audience+ )++ async def kept_out(user_id: str) -> bool:+ return user_id == "[email protected]"++ result = await mailerlite_backfill.apply(changes, audience, kept_out=kept_out)++ (batch,) = [c.kwargs["json"]["requests"] for c in client.post.await_args_list]+ assert [r["body"]["email"] for r in batch] == ["[email protected]"]+ assert result.succeeded[Decision.ADD_CHANGELOG] == 1+ assert result.skipped == 1+++async def test_an_account_that_cannot_be_read_fails_alone_and_the_run_goes_on(+ configured, no_sleep, monkeypatch, caplog+):+ client = MagicMock()+ client.post = AsyncMock(+ side_effect=lambda url, **kw: _batch_ok(kw["json"]["requests"])+ )+ monkeypatch.setattr(mailerlite_backfill, "_client", lambda: client)+ audience = _audience()+ changes = mailerlite_backfill.plan(+ [_customer("[email protected]", "active"), _customer("[email protected]", "active")], audience+ )++ async def kept_out(user_id: str) -> bool:+ if user_id == "[email protected]":+ raise RuntimeError("db down")+ return False++ result = await mailerlite_backfill.apply(changes, audience, kept_out=kept_out)++ (batch,) = [c.kwargs["json"]["requests"] for c in client.post.await_args_list]+ assert [r["body"]["email"] for r in batch] == ["[email protected]"]+ assert result.succeeded[Decision.ADD_CHANGELOG] == 1+ assert result.failed[Decision.ADD_CHANGELOG] == 1+ assert result.skipped == 0+ assert "[email protected]" not in caplog.textdiff --git a/autogpt_platform/backend/backend/notifications/mailerlite_field_backfill.py b/autogpt_platform/backend/backend/notifications/mailerlite_field_backfill.pyindex a760e8391c..25bb803af4 100644--- a/autogpt_platform/backend/backend/notifications/mailerlite_field_backfill.py+++ b/autogpt_platform/backend/backend/notifications/mailerlite_field_backfill.py@@ -5,8 +5,8 @@ events, so this works out each account's fields from scratch: the signup datefrom our database, the rest from Stripe, by the rules in `subscriber_fields.py`.Only accounts with a Stripe customer are given (see `cli/mailerlite_backfill`):MailerLite holds checkout openers, not every signup. An account that opted out-of marketing is never written, since a field write creates the subscriber-(`consent.py`).+of marketing, or that a signal places in Iran or Russia, is never written,+since a field write creates the subscriber (`consent.py`).Resumable by construction: the fields MailerLite already holds are read firstand only the difference is written, so an interrupted or repeated run picks up@@ -22,7 +22,8 @@ from urllib.parse import urlencodefrom pydantic import BaseModel, EmailStr, TypeAdapter, ValidationErrorfrom backend.data.notifications import SubscriberField, SubscriptionStatus-from backend.notifications.consent import marketing_allowed+from backend.notifications.audience_enrichment import points_at_excluded_country+from backend.notifications.consent import KeptOut, marketing_allowedfrom backend.notifications.mailerlite import (API_BASE,MailerLiteError,@@ -36,6 +37,7 @@ from backend.notifications.mailerlite_backfill import (MEMBER_STATUSES,PAGE_SIZE,UPSERT_BATCH_INTERVAL_SECONDS,+ BatchAnswer,Subscription,_refusal,_send_batch,@@ -64,6 +66,19 @@ class Person(BaseModel):timezone: str | None = None# Set when they refused marketing: they must never enter MailerLite.marketing_opt_out_at: datetime | None = None+ # The Stripe customer's billing address country, when it has one.+ billing_country: str | None = None+ # The Iranian or Russian country a checkout recorded (`consent.py`).+ excluded_country: str | None = None++ def placed_in_excluded_country(self, *countries: str | None) -> bool:+ """Whether any signal, `countries` included, places them in Iran or+ Russia, so they must never enter MailerLite (`consent.py`)."""+ return points_at_excluded_country(+ email=self.email,+ timezone=self.timezone,+ countries=(self.billing_country, self.excluded_country, *countries),+ )class FieldChange(BaseModel):@@ -82,6 +97,8 @@ class FieldPlan(BaseModel):invalid: int# People who refused marketing, left out of the plan and the statuses.opted_out: int+ # People a signal places in Iran or Russia, left out the same way.+ excluded_country: intdef standing(subscriptions: list[Subscription]) -> tuple[SubscriptionStatus, Fields]:@@ -142,11 +159,15 @@ def plan(people: list[Person], current: Current, *, create: bool = True) -> Fielchanges=[],invalid=0,opted_out=0,+ excluded_country=0,)for person in people:if not marketing_allowed(person):result.opted_out += 1continue+ if person.placed_in_excluded_country():+ result.excluded_country += 1+ continueif not _valid(person.email):result.invalid += 1continue@@ -197,16 +218,25 @@ async def read_current() -> Current:async def apply(changes: list[FieldChange],on_progress: Callable[[int, int], None] | None = None,-) -> tuple[int, int]:+ *,+ kept_out: KeptOut | None = None,+) -> tuple[int, int, int]:"""Write each change as a subscriber upsert, fifty to a batch, paced for- MailerLite's import limit. Returns (succeeded, failed); a failure is left- for the next run."""- succeeded = failed = 0+ MailerLite's import limit. With `kept_out`, each batch first drops anyone+ who may no longer be written: they opted out or were seen in Iran or+ Russia since the plan. Returns (succeeded, failed, skipped); a failure,+ an account that can't be read then included, is left for the next run."""+ succeeded = failed = skipped = 0for start in range(0, len(changes), BATCH_SIZE):if start:await asyncio.sleep(UPSERT_BATCH_INTERVAL_SECONDS)chunk = changes[start : start + BATCH_SIZE]- answers = await _send_batch([_upsert(c) for c in chunk])+ done = start + len(chunk)+ if kept_out:+ chunk, dropped, unreadable = await _writable(chunk, kept_out)+ skipped += dropped+ failed += unreadable+ answers = await _send_batch([_upsert(c) for c in chunk]) if chunk else []for change, answer in zip(chunk, answers):if answer.code in (200, 201, 202, 204):succeeded += 1@@ -217,8 +247,32 @@ async def apply("the next run retries it")if on_progress:- on_progress(start + len(chunk), len(changes))- return succeeded, failed+ on_progress(done, len(changes))+ return succeeded, failed, skipped+++async def _writable(+ chunk: list[FieldChange], kept_out: KeptOut+) -> tuple[list[FieldChange], int, int]:+ """The batch without anyone `kept_out` rejects or whose account it can't+ read, and how many of each were dropped."""+ allowed: list[FieldChange] = []+ rejected = unreadable = 0+ for change in chunk:+ try:+ if await kept_out(change.person.user_id):+ rejected += 1+ continue+ except Exception:+ unreadable += 1+ logger.warning(+ f"Re-reading the account of "+ f"{_refusal(change.person.email, BatchAnswer(code=0))} failed; "+ "the next run retries it"+ )+ continue+ allowed.append(change)+ return allowed, rejected, unreadabledef _upsert(change: FieldChange) -> dict:diff --git a/autogpt_platform/backend/backend/notifications/mailerlite_field_backfill_test.py b/autogpt_platform/backend/backend/notifications/mailerlite_field_backfill_test.pyindex 6db7e292db..d346d78847 100644--- a/autogpt_platform/backend/backend/notifications/mailerlite_field_backfill_test.py+++ b/autogpt_platform/backend/backend/notifications/mailerlite_field_backfill_test.py@@ -182,7 +182,7 @@ async def test_apply_batches_upserts_at_the_import_pace(configured, monkeypatch)changes = backfill.plan([_person(f"p{i}@x.io") for i in range(120)], {}).changesprogress = []- ok, failed = await backfill.apply(changes, lambda d, t: progress.append(d))+ ok, failed, skipped = await backfill.apply(changes, lambda d, t: progress.append(d))batches = [c.kwargs["json"]["requests"] for c in client.post.await_args_list]assert [len(b) for b in batches] == [50, 50, 20]@@ -197,10 +197,33 @@ async def test_apply_batches_upserts_at_the_import_pace(configured, monkeypatch)assert [c.args[0] for c in configured.await_args_list] == [mailerlite_backfill.UPSERT_BATCH_INTERVAL_SECONDS] * 2- assert (ok, failed) == (120, 0)+ assert (ok, failed, skipped) == (120, 0, 0)assert progress == [50, 100, 120]+async def test_apply_drops_anyone_who_may_no_longer_be_written(configured, monkeypatch):+ """The plan is hours old by its last batch: someone who opted out or was+ seen in Iran or Russia since then is checked out right before the write."""+ client = MagicMock()+ client.post = AsyncMock(+ side_effect=lambda url, **kw: _response(+ 200,+ {"responses": [{"code": 200} for _ in kw["json"]["requests"]]},+ )+ )+ monkeypatch.setattr(mailerlite_backfill, "_client", lambda: client)+ people = [_person("[email protected]"), _person("[email protected]")]+ changes = backfill.plan(people, {}).changes++ async def kept_out(user_id: str) -> bool:+ return user_id == "[email protected]"++ assert await backfill.apply(changes, kept_out=kept_out) == (1, 0, 1)+ (batch,) = [c.kwargs["json"]["requests"] for c in client.post.await_args_list]+ assert [r["body"]["email"] for r in batch] == ["[email protected]"]++@pytest.mark.asyncioasync def test_apply_counts_a_refused_upsert_without_logging_the_address(configured, monkeypatch, caplog@@ -216,7 +239,7 @@ async def test_apply_counts_a_refused_upsert_without_logging_the_address(monkeypatch.setattr(mailerlite_backfill, "_client", lambda: client)changes = backfill.plan([_person("[email protected]")], {}).changes- assert await backfill.apply(changes) == (0, 1)+ assert await backfill.apply(changes) == (0, 1, 0)assert "[email protected]" not in caplog.textassert " at .io with 422 " in caplog.textassert "The given data was invalid." in caplog.text@@ -313,3 +336,59 @@ def test_an_opted_out_person_is_counted_and_never_written(create):assert plan.opted_out == 1assert plan.invalid == 0assert plan.statuses[S.SUBSCRIBED] == 1+++ "excluded",+ [+ _person("[email protected]", _paid("active")).model_copy(+ update={"timezone": "Europe/Moscow"}+ ),+ _person("[email protected]", _paid("active")),+ _person("[email protected]", _paid("active")).model_copy(+ update={"billing_country": "RU"}+ ),+ _person("[email protected]", _paid("active")).model_copy(+ update={"excluded_country": "IR"}+ ),+ ],+ ids=["timezone", "email", "billing", "recorded-by-a-checkout"],+)+def test_a_person_placed_in_iran_or_russia_is_counted_and_never_written(excluded):+ plan = backfill.plan(+ [excluded, _person("[email protected]", _paid("active"))],+ {+ excluded.email: {"subscription_status": "signed"},+ "[email protected]": {"subscription_status": "signed"},+ },+ create=False,+ )+ assert [c.person.email for c in plan.changes] == ["[email protected]"]+ assert plan.excluded_country == 1+ assert plan.opted_out == 0+ assert plan.statuses[S.SUBSCRIBED] == 1+++async def test_an_account_that_cannot_be_read_fails_alone_and_the_run_goes_on(+ configured, monkeypatch, caplog+):+ client = MagicMock()+ client.post = AsyncMock(+ side_effect=lambda url, **kw: _response(+ 200,+ {"responses": [{"code": 200} for _ in kw["json"]["requests"]]},+ )+ )+ monkeypatch.setattr(mailerlite_backfill, "_client", lambda: client)+ changes = backfill.plan([_person("[email protected]"), _person("[email protected]")], {}).changes++ async def kept_out(user_id: str) -> bool:+ if user_id == "[email protected]":+ raise RuntimeError("db down")+ return False++ assert await backfill.apply(changes, kept_out=kept_out) == (1, 1, 0)+ (batch,) = [c.kwargs["json"]["requests"] for c in client.post.await_args_list]+ assert [r["body"]["email"] for r in batch] == ["[email protected]"]+ assert "[email protected]" not in caplog.textdiff --git a/autogpt_platform/backend/backend/notifications/notifications.py b/autogpt_platform/backend/backend/notifications/notifications.pyindex aaecd3854a..01bc5a526a 100644--- a/autogpt_platform/backend/backend/notifications/notifications.py+++ b/autogpt_platform/backend/backend/notifications/notifications.py@@ -36,7 +36,7 @@ from backend.data.user import (generate_unsubscribe_link,)from backend.notifications import briefing_runner, mailerlite-from backend.notifications.consent import log_opted_out_skip+from backend.notifications.consent import log_kept_out_skipfrom backend.notifications.dedupe import claim_daily_sendfrom backend.notifications.email import EmailSenderfrom backend.notifications.preferences import SERVICE_MESSAGES, wants_notification@@ -75,11 +75,13 @@ def _utc_today() -> date:return datetime.now(tz=timezone.utc).date()-async def _opted_out_since_queued(event: AudienceEventModel) -> bool:- """Re-read the opt-out right before the write. A change queued just before- the account refused marketing must not reach MailerLite: every write but the- unsubscribe upserts the subscriber. The unsubscribe is the refusal itself,- so it always goes through."""+async def _kept_out_since_queued(event: AudienceEventModel) -> bool:+ """Re-read, right before the write, whether the account may be in+ MailerLite at all. A change queued just before the account refused+ marketing must not reach MailerLite: every write but the unsubscribe+ upserts the subscriber. Nor may one queued without the country a checkout+ recorded, such as a trial start, for an account seen in Iran or Russia.+ The unsubscribe is the refusal itself, so it always goes through."""if event.action is AudienceAction.UNSUBSCRIBE:return Falsereturn await get_database_manager_async_client(@@ -405,8 +407,8 @@ class NotificationManager(AppService):_mailerlite_off_logged = Truereturn True- if await _opted_out_since_queued(event):- log_opted_out_skip(event.email, event.action.value)+ if await _kept_out_since_queued(event):+ log_kept_out_skip(event.email, event.action.value)return Truehandler = {@@ -419,6 +421,7 @@ class NotificationManager(AppService):AudienceAction.SIGNUP: mailerlite.record_signup,AudienceAction.CHECKOUT_OPENED: mailerlite.record_checkout_opened,AudienceAction.UNSUBSCRIBE: mailerlite.unsubscribe,+ AudienceAction.ONBOARDING_PROFILE: mailerlite.record_onboarding_profile,}[event.action]try:await handler(event.email, event.fields or None)diff --git a/autogpt_platform/backend/backend/notifications/role_backfill.py b/autogpt_platform/backend/backend/notifications/role_backfill.pynew file mode 100644index 0000000000..e4668b18d4--- /dev/null+++ b/autogpt_platform/backend/backend/notifications/role_backfill.py@@ -0,0 +1,245 @@+"""Give accounts from before SECRT-2852 the onboarding role the live code sends.++Since SECRT-2852 the profile step sends the role to MailerLite and PostHog+from the pick the wizard keeps (`data/onboarding_role.py`). Accounts that got+there earlier have no kept pick, only the copy in their business+understanding, which AutoPilot rewrites. An exact option ID there can only+have come from the wizard, so it is reliable; anything else is skipped rather+than guessed (`OnboardingRole.from_understanding`).++Both sides only fill gaps, so an older pick never replaces a newer one:+PostHog gets a `$set_once`, and a MailerLite subscriber is written only while+it has no role. Nobody is created in MailerLite, since only checkout openers+belong there, and nobody is written there who opted out of marketing or whom+a signal places in Iran or Russia (`consent.py`). PostHog is analytics, not+marketing, so it gets every reliable pick.++Each MailerLite write is one update by subscriber ID, one at a time and paced+like `checkout_backfill`, after checking the account may still be written and+reading the subscriber again: the live code keeps writing while a run lasts,+and a role it wrote meanwhile stays.+"""++import asyncio+import logging+from collections import Counter+from collections.abc import Callable+from datetime import datetime++from pydantic import BaseModel++from backend.data.notifications import SubscriberField+from backend.data.onboarding_role import OnboardingRole+from backend.notifications.audience_enrichment import (+ points_at_excluded_country,+ role_fields,+)+from backend.notifications.consent import KeptOut, marketing_allowed+from backend.notifications.mailerlite import (+ API_BASE,+ _client,+ _find_subscriber,+ _headers,+ _payload,+)+from backend.notifications.mailerlite_backfill import BatchAnswer, _refusal+from backend.notifications.mailerlite_field_backfill import Current+from backend.util.posthog_client import get_posthog_client+from backend.util.product_analytics import set_onboarding_role++logger = logging.getLogger(__name__)++# One person every two seconds, a read and a write each: half MailerLite's+# 120 calls a minute, so the live consumer keeps the rest.+WRITE_INTERVAL_SECONDS = 2.0+# The PostHog client drops events once its in-memory queue is full.+POSTHOG_FLUSH_EVERY = 500+++class RoleRecord(BaseModel):+ """An account with a role on record, and what MailerLite's rules need."""++ user_id: str+ email: str+ timezone: str | None = None+ marketing_opt_out_at: datetime | None = None+ stripe_customer_id: str | None = None+ # The Stripe billing address country, and the Iranian or Russian country a+ # checkout recorded: either may place them in Iran or Russia.+ billing_country: str | None = None+ excluded_country: str | None = None+ # The pick the wizard kept, if it kept one.+ choice: str | None = None+ other: str | None = None+ # The copy in the business understanding.+ understanding_role: str | None = None++ @property+ def kept(self) -> bool:+ return bool(self.choice)++ def reliable_role(self) -> OnboardingRole | None:+ if self.choice:+ return OnboardingRole(choice=self.choice, other=self.other)+ return OnboardingRole.from_understanding(self.understanding_role)+++class RoleAssignment(BaseModel):+ user_id: str+ email: str+ role: OnboardingRole+++class RolePlan(BaseModel):+ accounts: int+ # Reliable: the pick the wizard kept, or an exact option ID.+ kept: int+ exact: int+ # Neither: Other's text or a rewrite, left unset.+ skipped: int+ # Reliable picks by label.+ roles: dict[str, int]+ posthog: list[RoleAssignment]+ # MailerLite subscribers without a role.+ mailerlite: list[RoleAssignment]+ # MailerLite subscribers that already have one, left alone.+ already_set: int+ # Not MailerLite subscribers, never created here.+ not_subscribers: int+ opted_out: int+ excluded_country: int+++def plan(records: list[RoleRecord], current: Current) -> RolePlan:+ result = RolePlan(+ accounts=len(records),+ kept=0,+ exact=0,+ skipped=0,+ roles={},+ posthog=[],+ mailerlite=[],+ already_set=0,+ not_subscribers=0,+ opted_out=0,+ excluded_country=0,+ )+ roles: Counter[str] = Counter()+ for record in records:+ role = record.reliable_role()+ if role is None:+ result.skipped += 1+ continue+ if record.kept:+ result.kept += 1+ else:+ result.exact += 1+ roles[role.label] += 1+ assignment = RoleAssignment(+ user_id=record.user_id, email=record.email, role=role+ )+ result.posthog.append(assignment)+ _plan_mailerlite(result, record, assignment, current)+ result.roles = dict(roles)+ return result+++def _plan_mailerlite(+ result: RolePlan, record: RoleRecord, assignment: RoleAssignment, current: Current+) -> None:+ if not marketing_allowed(record):+ result.opted_out += 1+ return+ if points_at_excluded_country(+ email=record.email,+ timezone=record.timezone,+ countries=(record.billing_country, record.excluded_country),+ ):+ result.excluded_country += 1+ return+ held = current.get(record.email.strip().lower())+ if held is None:+ result.not_subscribers += 1+ elif held.get(SubscriberField.ROLE.value):+ result.already_set += 1+ else:+ result.mailerlite.append(assignment)+++async def send_to_posthog(assignments: list[RoleAssignment]) -> None:+ """A `$set_once` per person, flushed as it goes."""+ client = get_posthog_client()+ if client is None:+ raise RuntimeError("PostHog is not configured")+ for index, assignment in enumerate(assignments, start=1):+ set_onboarding_role(+ user_id=assignment.user_id, role=assignment.role, keep_existing=True+ )+ if index % POSTHOG_FLUSH_EVERY == 0:+ await asyncio.to_thread(client.flush)+ await asyncio.to_thread(client.flush)+++async def apply(+ changes: list[RoleAssignment],+ on_progress: Callable[[int, int], None] | None = None,+ *,+ kept_out: KeptOut | None = None,+) -> tuple[int, int, int]:+ """Returns (succeeded, failed, skipped). A failure is logged with+ MailerLite's reason and left for the next run. Someone who has gone, has+ a role by now, or with `kept_out` may no longer be written (opted out or+ seen in Iran or Russia since the plan) is skipped."""+ succeeded = failed = skipped = 0+ for index, change in enumerate(changes):+ if index:+ await asyncio.sleep(WRITE_INTERVAL_SECONDS)+ if on_progress and index and index % 100 == 0:+ on_progress(index, len(changes))+ outcome = await _write(change, kept_out)+ if outcome is None:+ skipped += 1+ elif outcome:+ succeeded += 1+ else:+ failed += 1+ if on_progress:+ on_progress(len(changes), len(changes))+ return succeeded, failed, skipped+++async def _write(change: RoleAssignment, kept_out: KeptOut | None) -> bool | None:+ """True once written, False on a failure, None when there is nothing to+ write any more."""+ try:+ if kept_out and await kept_out(change.user_id):+ return None+ subscriber = await _find_subscriber(change.email)+ if subscriber is None or not subscriber.get("id"):+ return None+ if (subscriber.get("fields") or {}).get(SubscriberField.ROLE.value):+ return None+ response = await _client().put(+ f"{API_BASE}/subscribers/{subscriber['id']}",+ headers=_headers(),+ json={"fields": _payload(role_fields(change.role))},+ )+ except Exception:+ logger.warning(+ "Role write failed for %s; the next run retries it",+ _refusal(change.email, BatchAnswer(code=0)),+ )+ return False+ if response.status == 404:+ return None+ if response.status in (200, 201, 202, 204):+ return True+ try:+ body = response.json()+ except Exception:+ body = None+ logger.warning(+ "Role write failed for %s; the next run retries it",+ _refusal(change.email, BatchAnswer(code=response.status, body=body)),+ )+ return Falsediff --git a/autogpt_platform/backend/backend/notifications/role_backfill_test.py b/autogpt_platform/backend/backend/notifications/role_backfill_test.pynew file mode 100644index 0000000000..3a3e9e32ff--- /dev/null+++ b/autogpt_platform/backend/backend/notifications/role_backfill_test.py@@ -0,0 +1,195 @@+"""The role backfill takes only reliable picks, fills gaps without replacing a+newer pick on either side, never creates a MailerLite subscriber, and keeps+anyone MailerLite may not hold out of it."""++import logging+from datetime import UTC, datetime+from unittest.mock import AsyncMock, MagicMock++import pytest++from backend.data.onboarding_role import OnboardingRole+from backend.notifications import role_backfill+from backend.notifications.mailerlite import pseudonym+from backend.notifications.role_backfill import RoleAssignment, RoleRecord++OPTED_OUT = datetime(2026, 10, 2, tzinfo=UTC)+++def _record(email: str, **fields) -> RoleRecord:+ return RoleRecord(user_id=f"u-{email}", email=email, **fields)+++def test_only_reliable_picks_are_planned():+ plan = role_backfill.plan(+ [+ _record("[email protected]", choice="Other", other="Dentist"),+ _record("[email protected]", understanding_role="Sales/BD"),+ # The kept pick wins over whatever the understanding says now.+ _record("[email protected]", choice="Marketing", understanding_role="CMO"),+ _record("[email protected]", understanding_role="Dentist"),+ _record("[email protected]", understanding_role="decision maker"),+ ],+ current={},+ )+ assert (plan.accounts, plan.kept, plan.exact, plan.skipped) == (5, 2, 1, 2)+ assert plan.roles == {"Other": 1, "Sales / BD": 1, "Marketing": 1}+ assert {a.email: a.role for a in plan.posthog} == {+ "[email protected]": OnboardingRole(choice="Other", other="Dentist"),+ "[email protected]": OnboardingRole(choice="Sales/BD"),+ "[email protected]": OnboardingRole(choice="Marketing"),+ }+++def test_mailerlite_only_fills_in_its_own_subscribers_without_a_role():+ plan = role_backfill.plan(+ [+ _record("[email protected]", understanding_role="Marketing"),+ _record("[email protected]", understanding_role="Marketing"),+ _record("[email protected]", understanding_role="Marketing"),+ ],+ current={+ "[email protected]": {"role": None, "subscription_status": "signed"},+ "[email protected]": {"role": "Engineering"},+ },+ )+ assert [a.email for a in plan.mailerlite] == ["[email protected]"]+ assert plan.already_set == 1+ assert plan.not_subscribers == 1+ assert len(plan.posthog) == 3+++def test_anyone_mailerlite_may_not_hold_still_reaches_posthog():+ """Marketing consent and the Iran and Russia rule are MailerLite's; the+ role is analytics too."""+ records = [+ _record(+ "[email protected]", understanding_role="Marketing", marketing_opt_out_at=OPTED_OUT+ ),+ _record(+ "[email protected]", understanding_role="Marketing", timezone="Europe/Moscow"+ ),+ _record("[email protected]", understanding_role="Marketing"),+ _record("[email protected]", understanding_role="Marketing", billing_country="RU"),+ _record("[email protected]", understanding_role="Marketing", excluded_country="IR"),+ ]+ current = {r.email: {} for r in records}+ plan = role_backfill.plan(records, current)+ assert plan.mailerlite == []+ assert (plan.opted_out, plan.excluded_country) == (1, 4)+ assert len(plan.posthog) == 5+++def _assignment(email: str = "[email protected]") -> RoleAssignment:+ return RoleAssignment(+ user_id="u-1", email=email, role=OnboardingRole(choice="Other", other="CFO")+ )+++def _response(status: int, body: dict | None = None) -> MagicMock:+ response = MagicMock(status=status)+ response.json.return_value = body or {}+ return response+++def mailerlite(monkeypatch):+ monkeypatch.setattr(role_backfill.asyncio, "sleep", AsyncMock())+ client = MagicMock(put=AsyncMock(return_value=_response(200)))+ monkeypatch.setattr(role_backfill, "_client", lambda: client)+ monkeypatch.setattr(role_backfill, "_headers", lambda: {})+ lookup = AsyncMock(return_value={"id": "ml_1", "fields": {"role": None}})+ monkeypatch.setattr(role_backfill, "_find_subscriber", lookup)+ return client, lookup+++async def test_each_subscriber_is_updated_by_id_never_upserted(mailerlite):+ client, lookup = mailerlite+ assert await role_backfill.apply([_assignment()]) == (1, 0, 0)+ lookup.assert_awaited_once_with("[email protected]")+ url = client.put.await_args.args[0]+ assert url.endswith("/subscribers/ml_1")+ assert client.put.await_args.kwargs["json"] == {+ "fields": {"role": "Other", "role_other": "CFO"}+ }+++ "subscriber",+ [None, {"id": "ml_1", "fields": {"role": "Engineering"}}],+ ids=["gone", "picked-meanwhile"],+)+async def test_someone_who_needs_nothing_any_more_is_skipped(mailerlite, subscriber):+ """The live code keeps writing while a run lasts: a role it wrote+ meanwhile is the newer pick."""+ client, lookup = mailerlite+ lookup.return_value = subscriber+ assert await role_backfill.apply([_assignment()]) == (0, 0, 1)+ client.put.assert_not_awaited()+++async def test_a_subscriber_deleted_before_the_write_is_skipped(mailerlite):+ client, _ = mailerlite+ client.put.return_value = _response(404)+ assert await role_backfill.apply([_assignment()]) == (0, 0, 1)+++async def test_a_failure_is_counted_by_pseudonym_and_the_run_goes_on(+ mailerlite, caplog+):+ client, _ = mailerlite+ client.put.side_effect = [+ _response(422, {"message": "nope"}),+ RuntimeError("network"),+ _response(200),+ ]+ changes = [_assignment(f"{n}@x.io") for n in ("a", "b", "c")]+ with caplog.at_level(logging.WARNING, logger=role_backfill.__name__):+ assert await role_backfill.apply(changes) == (1, 2, 0)+ assert pseudonym("[email protected]") in caplog.text+ assert "[email protected]" not in caplog.text+++async def test_posthog_gets_a_set_once_per_person_and_a_flush(monkeypatch):+ client = MagicMock()+ monkeypatch.setattr(role_backfill, "get_posthog_client", lambda: client)+ sent = MagicMock()+ monkeypatch.setattr(role_backfill, "set_onboarding_role", sent)+ monkeypatch.setattr(role_backfill, "POSTHOG_FLUSH_EVERY", 2)++ await role_backfill.send_to_posthog([_assignment() for _ in range(3)])++ assert sent.call_count == 3+ assert all(call.kwargs["keep_existing"] for call in sent.call_args_list)+ assert client.flush.call_count == 2+++async def test_posthog_must_be_configured(monkeypatch):+ monkeypatch.setattr(role_backfill, "get_posthog_client", lambda: None)+ with pytest.raises(RuntimeError, match="PostHog is not configured"):+ await role_backfill.send_to_posthog([_assignment()])+++ "kept_out, outcome",+ [+ (AsyncMock(return_value=True), (0, 0, 1)),+ (AsyncMock(side_effect=RuntimeError("db down")), (0, 1, 0)),+ ],+ ids=["kept-out-since-the-plan", "unreadable"],+)+async def test_an_account_checked_out_before_the_write_is_never_written(+ mailerlite, kept_out, outcome+):+ client, lookup = mailerlite+ assert await role_backfill.apply([_assignment()], kept_out=kept_out) == outcome+ kept_out.assert_awaited_once_with("u-1")+ lookup.assert_not_awaited()+ client.put.assert_not_awaited()diff --git a/autogpt_platform/backend/backend/notifications/trial_audience.py b/autogpt_platform/backend/backend/notifications/trial_audience.pyindex 3a007306ef..18880233e8 100644--- a/autogpt_platform/backend/backend/notifications/trial_audience.py+++ b/autogpt_platform/backend/backend/notifications/trial_audience.py@@ -12,8 +12,9 @@ The subscriber's status and dates (`subscriber_fields.py`) ride on the sameevent. The notification service writes only the fields while the trial groupis not configured.-Nothing is queued for a customer who opted out of marketing (`consent.py`);-their trial notices and claims are unaffected.+Nothing is queued for a customer who opted out of marketing, or whom a signal+places in Iran or Russia (`consent.py`); their trial notices and claims are+unaffected."""import loggingdiff --git a/autogpt_platform/backend/backend/notifications/trial_audience_test.py b/autogpt_platform/backend/backend/notifications/trial_audience_test.pyindex 71cc2df3f2..c518faa9be 100644--- a/autogpt_platform/backend/backend/notifications/trial_audience_test.py+++ b/autogpt_platform/backend/backend/notifications/trial_audience_test.py@@ -112,11 +112,16 @@ async def _notify(claim_welcome=None,opted_out_at=None,user=None,+ timezone="America/Chicago",):audience = audience or AsyncMock(return_value=NotificationResult(success=True))notice = AsyncMock(return_value=NotificationResult(success=True))user = user or SimpleNamespace(- id="user-1", name="Sam", email=email, marketing_opt_out_at=opted_out_at+ id="user-1",+ name="Sam",+ email=email,+ marketing_opt_out_at=opted_out_at,+ timezone=timezone,)users = MagicMock(get_user_by_id=AsyncMock(return_value=user),@@ -258,6 +263,16 @@ async def test_an_opted_out_trialist_gets_the_notice_but_no_group_change(trial,release.assert_not_awaited()[email protected]("kind", ["started", "canceled", "converted"])+async def test_a_trialist_in_tehran_gets_the_notice_but_no_group_change(trial, kind):+ trial, raw = _state(trial, kind)+ got, notice, release = await _notify(trial, raw, kind, timezone="Asia/Tehran")+ assert got == []+ notice.assert_awaited_once()+ release.assert_not_awaited()++@pytest.mark.asyncio@pytest.mark.parametrize("welcomed", [False, True])async def test_an_opted_out_conversion_still_takes_the_welcome_claim(trial, welcomed):diff --git a/autogpt_platform/backend/backend/notifications/trial_test.py b/autogpt_platform/backend/backend/notifications/trial_test.pyindex 9456d133d0..4cbb1c6bde 100644--- a/autogpt_platform/backend/backend/notifications/trial_test.py+++ b/autogpt_platform/backend/backend/notifications/trial_test.py@@ -146,6 +146,7 @@ async def test_notice_uses_shared_notification_queue(trial, outcome):name="Sam",email="[email protected]",marketing_opt_out_at=None,+ timezone="America/Chicago",))),diff --git a/autogpt_platform/backend/backend/util/product_analytics.py b/autogpt_platform/backend/backend/util/product_analytics.pyindex 5c9da8f88a..c41cfa20dd 100644--- a/autogpt_platform/backend/backend/util/product_analytics.py+++ b/autogpt_platform/backend/backend/util/product_analytics.py@@ -54,6 +54,7 @@ from datetime import datetimefrom typing import TYPE_CHECKING, Any, Literalfrom backend.data.experiments import list_assignments+from backend.data.onboarding_role import OnboardingRolefrom backend.util import posthog_clientfrom backend.util.posthog_events import PostHogEvent@@ -349,6 +350,21 @@ def track_onboarding_completed(*, user_id: str) -> None:track(user_id, PostHogEvent.ONBOARDING_COMPLETED)+def set_onboarding_role(+ *, user_id: str, role: OnboardingRole, keep_existing: bool = False+) -> None:+ """The role picked in onboarding as person properties, so funnels and+ retention can be split by it. Other's text is cleared for any other pick.+ `keep_existing` is for the backfill: a role PostHog already holds is the+ newer pick, so it is left alone."""+ person = {"onboarding_role": role.label, "onboarding_role_other": role.other}+ posthog_client.capture(+ user_id,+ PostHogEvent.SET_PERSON_PROPERTIES,+ {"$set_once" if keep_existing else "$set": person},+ )++async def track_checkout_started(*,user_id: str,diff --git a/autogpt_platform/backend/backend/util/product_analytics_test.py b/autogpt_platform/backend/backend/util/product_analytics_test.pyindex eeada29c44..3e8946bfd4 100644--- a/autogpt_platform/backend/backend/util/product_analytics_test.py+++ b/autogpt_platform/backend/backend/util/product_analytics_test.py@@ -7,6 +7,7 @@ from unittest.mock import Mockimport pytestfrom backend.data.experiments import ExperimentAssignment+from backend.data.onboarding_role import OnboardingRolefrom backend.util import posthog_client, product_analyticsfrom backend.util.posthog_events import PostHogEvent@@ -571,3 +572,41 @@ def test_safe_error_detail_drops_a_word_cut_by_the_scan_limit() -> None:product_analytics.safe_error_detail(f"{url} {token}")== "https://example.com/token?[redacted]")+++ "role, person",+ [+ (+ OnboardingRole(choice="Sales/BD"),+ {"onboarding_role": "Sales / BD", "onboarding_role_other": None},+ ),+ (+ OnboardingRole(choice="Other", other="Dentist"),+ {"onboarding_role": "Other", "onboarding_role_other": "Dentist"},+ ),+ ],+)+def test_the_onboarding_role_is_set_on_the_person(+ capture: Mock, role: OnboardingRole, person: dict+) -> None:+ product_analytics.set_onboarding_role(user_id="user-1", role=role)++ event, properties = _only_call(capture)+ assert capture.call_args.kwargs["distinct_id"] == "user-1"+ assert event == "$set"+ assert properties["$set"] == person+ assert "$set_once" not in properties+++def test_the_backfilled_role_never_replaces_one_posthog_holds(capture: Mock) -> None:+ product_analytics.set_onboarding_role(+ user_id="user-1", role=OnboardingRole(choice="Marketing"), keep_existing=True+ )++ _, properties = _only_call(capture)+ assert properties["$set_once"] == {+ "onboarding_role": "Marketing",+ "onboarding_role_other": None,+ }+ assert "$set" not in propertiesdiff --git a/autogpt_platform/backend/migrations/20261008120000_onboarding_role/migration.sql b/autogpt_platform/backend/migrations/20261008120000_onboarding_role/migration.sqlnew file mode 100644index 0000000000..609871b3e8--- /dev/null+++ b/autogpt_platform/backend/migrations/20261008120000_onboarding_role/migration.sql@@ -0,0 +1,7 @@+-- The role picked in the onboarding wizard (SECRT-2852), kept apart from the+-- AutoPilot business understanding, which rewrites its own copy.++-- Nullable with no default: a catalog-only change, no table rewrite.+-- AlterTable+ALTER TABLE "UserOnboarding" ADD COLUMN "role" TEXT,+ADD COLUMN "roleOther" TEXT;diff --git a/autogpt_platform/backend/migrations/20261008130000_marketing_excluded_country/migration.sql b/autogpt_platform/backend/migrations/20261008130000_marketing_excluded_country/migration.sqlnew file mode 100644index 0000000000..53556647f0--- /dev/null+++ b/autogpt_platform/backend/migrations/20261008130000_marketing_excluded_country/migration.sql@@ -0,0 +1,7 @@+-- The first Iranian or Russian country a checkout saw for an account, which+-- keeps it out of MailerLite for good. Every later MailerLite write is checked+-- against it, since a trial or billing event no longer carries the IP country.++-- Nullable with no default: a catalog-only change, no table rewrite.+-- AlterTable+ALTER TABLE "User" ADD COLUMN "marketingExcludedCountry" TEXT;diff --git a/autogpt_platform/backend/schema.prisma b/autogpt_platform/backend/schema.prismaindex 6a21eef230..7d2ddd88bd 100644--- a/autogpt_platform/backend/schema.prisma+++ b/autogpt_platform/backend/schema.prisma@@ -136,6 +136,10 @@ model User {termsVersion String?marketingOptOutAt DateTime?marketingOptOutSource String? // "signup" | "email_unsubscribe" | "settings"+ // The first Iranian or Russian country a checkout saw for the account (its+ // IP or billing country). Like an opt-out, it keeps the account out of+ // MailerLite for good (backend/notifications/consent.py).+ marketingExcludedCountry String?// Default AutoPilot connection for chats the caller didn't route explicitly// (bot links, schedules, a fresh /copilot session). NULL means "automatic" —@@ -334,6 +338,13 @@ model UserOnboarding {lastRunAt DateTime?consecutiveRunDays Int @default(0)+ // The role picked in the wizard: one of RoleStep's option IDs, or "Other"+ // with the typed text in roleOther. Written only by POST /onboarding/profile:+ // AutoPilot rewrites the copy in CoPilotUnderstanding, so that is no record+ // of the pick (backend/data/onboarding_role.py).+ role String?+ roleOther String?+userId String @uniqueUser User @relation(fields: [userId], references: [id], onDelete: Cascade)}diff --git a/docs/engineering/tracking-plan.md b/docs/engineering/tracking-plan.mdindex 08bbfbb6cb..dfca9566ed 100644--- a/docs/engineering/tracking-plan.md+++ b/docs/engineering/tracking-plan.md@@ -326,6 +326,23 @@ Dates are ISO-8601 UTC strings of the lifecycle moment, not of the sync. Adate that doesn't apply to the current status is `$unset`, never sent asnull.+The role picked in the onboarding wizard is set with a `$set` of its own when+the profile is submitted at the Preparing step (`POST /api/onboarding/profile`,+`product_analytics.set_onboarding_role`), so funnels and retention can be split+by role. Someone who left the wizard before Preparing has neither property.++| Property | Value |+| --- | --- |+| `onboarding_role` | The pick, as labelled in the wizard: `Founder / CEO`, `Operations`, `Sales / BD`, `Marketing`, `Product / PM`, `Engineering`, `HR / People` or `Other`. |+| `onboarding_role_other` | What was typed after picking `Other` (trimmed, at most 100 characters); null for any other pick. |++The pick is kept on `UserOnboarding`, not read back from the AutoPilot+business understanding, whose copy AutoPilot rewrites. `poetry run cli+onboarding-role-backfill` sets both for earlier accounts with `$set_once`, from+exact picks only: any other stored value may be `Other`'s text or a rewrite,+so it is left unset. MailerLite gets the same values as `role` and+`role_other`.+## RemovedNo longer sent (SECRT-2722, SECRT-2723). The names stay reserved: the pin tests fail if