Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(workflows): run the c.io import in the background and show its progress (#124695)

Harley Alexander3c792dcf1f39
diff --git a/frontend/snapshots.yml b/frontend/snapshots.yml
index 46312c45fd11..304dff80714c 100644
--- a/frontend/snapshots.yml
+++ b/frontend/snapshots.yml
@@ -8378,6 +8378,14 @@ snapshots:
hash: v1.k794b7964.ea5155a38680b44a133382c65e04c769f83c65fc39b9e6755ffd32886c3050c1.kPCkKfbt0yvdjTV8VJn_CxXAYIoek_yy07zyItt9Itc
products-workflows-customerioimportmodal--both-steps-not-completed--light:
hash: v1.k794b7964.1df6ca04972f8b0d0b0c517ce16d6924fe27254f359cc0c8e16e816cea95d56c.GQootOFxJjyqS8jojYyVBjB5IY92XYB_MYa0bDIA9s8
+ products-workflows-customerioimportmodal--import-running--dark:
+ hash: v1.k794b7964.57dd6aca001fbf101255b448c27de4688d18c095cfca3c042ab44ccc6bcc3f9f.inc_-_1Gkkm4ByDHH97dG8v-iH2-AyTNW5T9ZqVFLuA
+ products-workflows-customerioimportmodal--import-running--light:
+ hash: v1.k794b7964.9213d0fd3adaa8e7da81fa39786fe293e535dc103850bd9f35d5c1ab5dd47e14.ymzYjelfakfjTVTYmn4ev6e0Ck8ur-Ha4SjiPNnxck8
+ products-workflows-customerioimportmodal--import-step-failed--dark:
+ hash: v1.k794b7964.2ddd7fdaac8c5d05c03c4ca70a9c1e6e3dbb2f9aa2be9966795d776f205c3319.ROrRNxbO6pDmia5A5H_oxodvc71spgu3ch2gfrCoOs8
+ products-workflows-customerioimportmodal--import-step-failed--light:
+ hash: v1.k794b7964.e6f80c0774e1d41294546fa74a08ddef80f8366688f6366273a8e26f19a4159d.MUXxZf-aSFCOIy1uIP3QdcFC7egvFpGYcqnUPBPduGE
products-workflows-customerioimportmodal--step-1-completed--dark:
hash: v1.k794b7964.c83846d9a55e8de54910a1334c52aa8dd46108c77d73826d694e3a8b113b75f6.x7ONobAIrfQGWsfDxcoRtXBqw0UA-6GqPlyhg17HRiY
products-workflows-customerioimportmodal--step-1-completed--light:
diff --git a/frontend/src/lib/constants.tsx b/frontend/src/lib/constants.tsx
index 2950d71b4495..8cb40bb6acbe 100644
--- a/frontend/src/lib/constants.tsx
+++ b/frontend/src/lib/constants.tsx
@@ -618,6 +618,7 @@ export const FEATURE_FLAGS = {
WORKFLOWS_AI_FIRST_NEW: 'workflows-ai-first-new', // owner: @mayteio #team-workflows
WORKFLOWS_BROADCAST_AUDIENCE_SUGGESTIONS: 'workflows-broadcast-audience-suggestions', // owner: @dmarchuk #team-workflows, AI audience ideas on the Broadcasts tab
WORKFLOWS_COHORT_CONDITIONS: 'workflows-cohort-conditions', // owner: @meikelmosby #team-workflows
+ WORKFLOWS_CUSTOMERIO_BACKGROUND_IMPORT: 'workflows-customerio-background-import', // owner: @mayteio #team-workflows, runs the Customer.io import in the background with progress
WORKFLOWS_EMAIL_DOMAIN_AGENT_SETUP: 'workflows-email-domain-agent-setup', // owner: @silthus #team-workflows
WORKFLOWS_EMAIL_DOMAIN_WIZARD: 'workflows-email-domain-wizard', // owner: @silthus #team-workflows
WORKFLOWS_EMAIL_SENDER_ROTATION: 'workflows-email-sender-rotation', // owner: @arthurdedeus #team-workflows
diff --git a/products/messaging/backend/services/customerio_import_service.py b/products/messaging/backend/services/customerio_import_service.py
index 8c22f7bd0cc5..ff3b592bdd27 100644
--- a/products/messaging/backend/services/customerio_import_service.py
+++ b/products/messaging/backend/services/customerio_import_service.py
@@ -590,6 +590,9 @@ class CustomerIOImportService:
Returns how many recipients were not already opted out of every category.
"""
+ # Customer.io can return several profiles with one email. A repeated address would add two rows for one
+ # recipient and fail the whole batch on the unique (team, identifier) constraint.
+ emails = list(dict.fromkeys(emails))
changed = 0
with transaction.atomic():
# Fetch existing preferences
@@ -629,6 +632,8 @@ class CustomerIOImportService:
# Bulk operations
if to_create:
+ # No ignore_conflicts: a row inserted between the read and this write must fail the batch, so a retry
+ # merges the opt-out into it instead of silently skipping the recipient.
MessageRecipientPreference.objects.bulk_create(to_create, batch_size=500)
if to_update:
diff --git a/products/messaging/backend/tests/test_provider_imports.py b/products/messaging/backend/tests/test_provider_imports.py
index be224aaff244..36044bf61c7c 100644
--- a/products/messaging/backend/tests/test_provider_imports.py
+++ b/products/messaging/backend/tests/test_provider_imports.py
@@ -9,6 +9,10 @@ from django.utils import timezone
from parameterized import parameterized
+from products.messaging.backend.models.message_preferences import (
+ ALL_MESSAGE_PREFERENCE_CATEGORY_ID,
+ MessageRecipientPreference,
+)
from products.messaging.backend.models.message_suppression import MessageSuppression
from products.messaging.backend.models.provider_import_run import ProviderImportRun
from products.messaging.backend.provider_imports import runs
@@ -126,6 +130,60 @@ class TestProviderImportRuns(BaseTest):
assert run.progress["suppressions"]["checkpoint"] == {"suppression_type": "bounces", "offset": 2}
assert self._suppressed() == {"[email protected]", "[email protected]"}
+ def test_opt_outs_step_completes_when_customerio_repeats_an_email(
+ self, mock_client_class: MagicMock, _start_workflow: MagicMock
+ ) -> None:
+ mock_client_class.return_value.get_globally_unsubscribed_customers.return_value = {
+ "identifiers": [
+ {"id": "1", "email": "[email protected]"},
+ {"id": "2", "email": "[email protected]"},
+ {"id": "3", "email": "[email protected]"},
+ ],
+ "next": "",
+ }
+ run = self._start(["opt_outs"])
+
+ runs.run_step(self.team.id, run.id, "opt_outs", attempt=1)
+
+ run.refresh_from_db()
+ assert run.progress["opt_outs"]["status"] == "completed"
+ assert run.progress["opt_outs"]["imported"] == 2
+ assert set(
+ MessageRecipientPreference.objects.filter(team_id=self.team.id).values_list("identifier", flat=True)
+
+ def test_opt_outs_merge_into_a_preference_saved_while_the_import_runs(
+ self, mock_client_class: MagicMock, _start_workflow: MagicMock
+ ) -> None:
+ mock_client_class.return_value.get_globally_unsubscribed_customers.return_value = {
+ "identifiers": [{"id": "1", "email": "[email protected]"}],
+ "next": "",
+ }
+ MessageRecipientPreference.objects.create(
+ team_id=self.team.id, identifier="[email protected]", preferences={"category-1": "OPTED_OUT"}
+ )
+ run = self._start(["opt_outs"])
+ read_existing = MessageRecipientPreference.objects.filter
+ reads = 0
+
+ def miss_the_first_read(*args: object, **kwargs: object) -> object:
+ nonlocal reads
+ reads += 1
+ return MessageRecipientPreference.objects.none() if reads == 1 else read_existing(*args, **kwargs)
+
+ with patch.object(MessageRecipientPreference.objects, "filter", side_effect=miss_the_first_read):
+ with self.assertRaises(IntegrityError):
+ runs.run_step(self.team.id, run.id, "opt_outs", attempt=1)
+ runs.run_step(self.team.id, run.id, "opt_outs", attempt=2)
+
+ run.refresh_from_db()
+ assert run.progress["opt_outs"]["status"] == "completed"
+ preference = MessageRecipientPreference.objects.get(team_id=self.team.id, identifier="[email protected]")
+ assert preference.preferences == {
+ "category-1": "OPTED_OUT",
+ ALL_MESSAGE_PREFERENCE_CATEGORY_ID: "OPTED_OUT",
+ }
+
def test_category_warnings_never_carry_raw_exception_text(
self, mock_client_class: MagicMock, _start_workflow: MagicMock
) -> None:
diff --git a/products/workflows/frontend/OptOuts/CustomerIOImportModal.stories.tsx b/products/workflows/frontend/OptOuts/CustomerIOImportModal.stories.tsx
index da3e7a33a7dd..37094f44660e 100644
--- a/products/workflows/frontend/OptOuts/CustomerIOImportModal.stories.tsx
+++ b/products/workflows/frontend/OptOuts/CustomerIOImportModal.stories.tsx
@@ -2,18 +2,55 @@ import { Meta, StoryFn } from '@storybook/react'
import { useActions } from 'kea'
import { useEffect } from 'react'
+import { FEATURE_FLAGS } from 'lib/constants'
+
import { useStorybookMocks } from '~/mocks/browser'
+import type { ProviderImportRunApi } from 'products/messaging/frontend/generated/api.schemas'
+
import { customerIOImportLogic } from './customerIOImportLogic'
import type { OptOutSyncConfigResponse } from './customerIOImportLogic'
import { CustomerIOImportModal } from './CustomerIOImportModal'
const syncConfigEndpoint = '/api/environments/:team_id/messaging_categories/optout_sync_config/'
+const importRunsEndpoint = '/api/projects/:team_id/messaging_provider_imports/'
+const importProvidersEndpoint = '/api/projects/:team_id/messaging_provider_imports/providers/'
+
+const importProviders = [
+ {
+ key: 'customerio',
+ label: 'Customer.io',
+ steps: [
+ {
+ key: 'categories',
+ label: 'Subscription topics',
+ description: 'Creates a message category for each Customer.io subscription topic.',
+ },
+ {
+ key: 'suppressions',
+ label: 'Suppressed addresses',
+ description:
+ "Adds the addresses that Customer.io's email provider bounced, marked as invalid, or got a spam report for to the suppression list.",
+ },
+ {
+ key: 'opt_outs',
+ label: 'Global unsubscribes',
+ description:
+ 'Opts people who unsubscribed from all Customer.io messages out of all marketing messages.',
+ },
+ ],
+ },
+]
-function mockSyncConfig(config: OptOutSyncConfigResponse): Record<string, any> {
+function mockSyncConfig(
+ config: OptOutSyncConfigResponse,
+ importRuns: ProviderImportRunApi[] = []
+): Record<string, any> {
return {
get: {
[syncConfigEndpoint]: config,
+ [importRunsEndpoint]: importRuns,
+ [importProvidersEndpoint]: importProviders,
},
}
}
@@ -82,10 +119,11 @@ export default meta
type StoryProps = {
syncConfig: OptOutSyncConfigResponse
+ importRuns?: ProviderImportRunApi[]
}
-const Template: StoryFn<StoryProps> = ({ syncConfig }) => {
- useStorybookMocks(mockSyncConfig(syncConfig))
+const Template: StoryFn<StoryProps> = ({ syncConfig, importRuns }) => {
+ useStorybookMocks(mockSyncConfig(syncConfig, importRuns))
const { openImportModal } = useActions(customerIOImportLogic)
useEffect(() => {
@@ -181,3 +219,93 @@ Step4TrackDisabled.parameters = { testOptions: { waitForSelector: '.LemonCollaps
export const AllStepsCompleted: StoryFn<StoryProps> = Template.bind({})
AllStepsCompleted.args = { syncConfig: step4EnabledConfig }
AllStepsCompleted.parameters = { testOptions: { waitForSelector: '.LemonCollapse' } }
+
+const storedKeyConfig: OptOutSyncConfigResponse = { ...emptySyncConfig, app_integration_id: 1 }
+
+const importRun = (
+ status: ProviderImportRunApi['status'],
+ steps: ProviderImportRunApi['steps']
+): ProviderImportRunApi => ({
+ id: 'run-1',
+ provider: 'customerio',
+ status,
+ steps,
+ created_at: '2026-04-13T10:00:00Z',
+ started_at: '2026-04-13T10:00:02Z',
+ finished_at: status === 'completed' || status === 'failed' ? '2026-04-13T10:06:00Z' : null,
+})
+
+export const ImportRunning: StoryFn<StoryProps> = Template.bind({})
+ImportRunning.args = {
+ syncConfig: storedKeyConfig,
+ importRuns: [
+ importRun('running', [
+ {
+ key: 'categories',
+ label: 'Subscription topics',
+ status: 'completed',
+ processed: 6,
+ imported: 6,
+ warnings: [],
+ last_error: null,
+ },
+ {
+ key: 'suppressions',
+ label: 'Suppressed addresses',
+ status: 'running',
+ processed: 12000,
+ imported: 11874,
+ warnings: [],
+ last_error: 'Customer.io returned HTTP 503.',
+ },
+ {
+ key: 'opt_outs',
+ label: 'Global unsubscribes',
+ status: 'pending',
+ processed: 0,
+ imported: 0,
+ warnings: [],
+ last_error: null,
+ },
+ ]),
+ ],
+}
+ImportRunning.parameters = {
+ featureFlags: [FEATURE_FLAGS.WORKFLOWS_CUSTOMERIO_BACKGROUND_IMPORT],
+ testOptions: {
+ waitForLoadersToDisappear: false,
+ waitForSelector: '[data-attr="customerio-import-run-progress"]',
+ },
+}
+
+export const ImportStepFailed: StoryFn<StoryProps> = Template.bind({})
+ImportStepFailed.args = {
+ syncConfig: storedKeyConfig,
+ importRuns: [
+ importRun('failed', [
+ {
+ key: 'suppressions',
+ label: 'Suppressed addresses',
+ status: 'completed',
+ processed: 2505,
+ imported: 2505,
+ warnings: [],
+ last_error: null,
+ },
+ {
+ key: 'opt_outs',
+ label: 'Global unsubscribes',
+ status: 'failed',
+ processed: 1000,
+ imported: 1000,
+ warnings: [],
+ last_error: 'Customer.io returned HTTP 503.',
+ },
+ ]),
+ ],
+}
+ImportStepFailed.parameters = {
+ featureFlags: [FEATURE_FLAGS.WORKFLOWS_CUSTOMERIO_BACKGROUND_IMPORT],
+ mockDate: '2026-04-13 10:30:00',
+ testOptions: { waitForSelector: '[data-attr="customerio-import-run-progress"]' },
+}
diff --git a/products/workflows/frontend/OptOuts/CustomerIOImportModal.tsx b/products/workflows/frontend/OptOuts/CustomerIOImportModal.tsx
index dca3a8781333..05cb4fffac7d 100644
--- a/products/workflows/frontend/OptOuts/CustomerIOImportModal.tsx
+++ b/products/workflows/frontend/OptOuts/CustomerIOImportModal.tsx
@@ -5,6 +5,7 @@ import { IconCopy, IconTrash } from '@posthog/icons'
import {
LemonBanner,
LemonButton,
+ LemonCheckbox,
LemonCollapse,
LemonInput,
LemonModal,
@@ -21,7 +22,10 @@ import { LemonFileInput } from 'lib/lemon-ui/LemonFileInput'
import { copyToClipboard } from 'lib/utils/copyToClipboard'
import { organizationLogic } from 'scenes/organizationLogic'
+import type { ProviderImportRunApi } from 'products/messaging/frontend/generated/api.schemas'
+
import { CSVImportProgress, customerIOImportLogic } from './customerIOImportLogic'
+import { CustomerIOImportRunProgress } from './CustomerIOImportRunProgress'
function StepBadge({ status }: { status: 'completed' | 'failed' | false }): JSX.Element | null {
if (status === 'completed') {
@@ -33,7 +37,214 @@ function StepBadge({ status }: { status: 'completed' | 'failed' | false }): JSX.
return null
}
-function Step1Content(): JSX.Element {
+function ImportStepPicker(): JSX.Element {
+ const { importProviders, importProvidersLoading, importStepOptions, deselectedImportSteps } =
+ useValues(customerIOImportLogic)
+ const { toggleImportStep, loadImportProviders } = useActions(customerIOImportLogic)
+
+ if (importProviders === null && importProvidersLoading) {
+ return <Spinner />
+ }
+ if (importProviders === null) {
+ return (
+ <LemonBanner type="error" action={{ children: 'Try again', onClick: loadImportProviders }}>
+ Couldn't load what you can import.
+ </LemonBanner>
+ )
+ }
+
+ return (
+ <div className="space-y-2">
+ <label className="LemonLabel">What to import</label>
+ {importStepOptions.map((step) => (
+ <div key={step.key}>
+ <LemonCheckbox
+ checked={!deselectedImportSteps.includes(step.key)}
+ onChange={() => toggleImportStep(step.key)}
+ label={step.label}
+ data-attr={`customerio-import-step-${step.key}`}
+ />
+ <div className="text-xs text-secondary ml-6">{step.description}</div>
+ </div>
+ ))}
+ </div>
+ )
+}
+
+function LatestImportRunResult({ run }: { run: ProviderImportRunApi }): JSX.Element {
+ return (
+ <div className="space-y-3">
+ {run.status === 'completed' ? (
+ <LemonBanner type="success">
+ <span className="font-semibold">Import finished</span>
+ </LemonBanner>
+ ) : (
+ <LemonBanner type="error">
+ Some steps failed. Run the import again to retry them. Data that already imported isn't duplicated.
+ </LemonBanner>
+ )}
+ <CustomerIOImportRunProgress run={run} />
+ {run.finished_at && (
+ <div className="flex items-center justify-between text-sm">
+ <span>Finished</span>
+ <TZLabel time={run.finished_at} />
+ </div>
+ )}
+ </div>
+ )
+}
+
+function LegacyImportResult(): JSX.Element | null {
+ const { syncConfig } = useValues(customerIOImportLogic)
+ const result = syncConfig?.app_import_result
+
+ if (result?.status === 'failed') {
+ return <LemonBanner type="error">Last import failed: {result.error || 'Unknown error'}</LemonBanner>
+ }
+ if (result?.status !== 'completed') {
+ return null
+ }
+ return (
+ <div className="space-y-2 text-sm">
+ <LemonBanner type="success">
+ <span className="font-semibold">API import complete</span>
+ </LemonBanner>
+ <div className="flex items-center justify-between">
+ <span>Last imported:</span>
+ <TZLabel time={result.imported_at} />
+ </div>
+ <div className="flex items-center justify-between">
+ <span>Categories imported:</span>
+ <LemonTag>{result.categories_created ?? 0}</LemonTag>
+ </div>
+ <div className="flex items-center justify-between">
+ <span>Globally unsubscribed users:</span>
+ <LemonTag>{(result.globally_unsubscribed_count ?? 0).toLocaleString()}</LemonTag>
+ </div>
+ <div className="flex items-center justify-between">
+ <span>Suppressed addresses:</span>
+ <LemonTag>{(result.suppressions_imported ?? 0).toLocaleString()}</LemonTag>
+ </div>
+ </div>
+ )
+}
+
+function BackgroundImportContent(): JSX.Element {
+ const {
+ importForm,
+ importError,
+ syncConfig,
+ isRemovingAppConfig,
+ importRuns,
+ importRunsLoading,
+ latestImportRun,
+ isImportRunActive,
+ selectedImportSteps,
+ startedImportRunLoading,
+ } = useValues(customerIOImportLogic)
+ const { submitImportForm, startImport, removeAppConfig } = useActions(customerIOImportLogic)
+
+ const hasStoredKey = syncConfig?.app_integration_id != null
+ const noStepsReason = selectedImportSteps.length === 0 ? 'Select at least one thing to import' : undefined
+
+ if (importRuns === null && importRunsLoading) {
+ return (
+ <div className="flex justify-center py-8">
+ <Spinner className="text-3xl" />
+ </div>
+ )
+ }
+
+ if (latestImportRun && isImportRunActive) {
+ return (
+ <div className="space-y-4">
+ <LemonBanner type="info">
+ The import runs in the background. You can close this window and come back to check on it.
+ </LemonBanner>
+ <CustomerIOImportRunProgress run={latestImportRun} />
+ </div>
+ )
+ }
+
+ return (
+ <div className="space-y-4">
+ {latestImportRun ? <LatestImportRunResult run={latestImportRun} /> : <LegacyImportResult />}
+
+ <ImportStepPicker />
+
+ {hasStoredKey ? (
+ <>
+ <div className="space-y-2">
+ <label className="LemonLabel">Customer.io App API Key</label>
+ <LemonInput
+ value="••••••••••••••••"
+ disabledReason="Can't be changed"
+ suffix={
+ <LemonButton
+ size="xsmall"
+ type="tertiary"
+ status="danger"
+ tooltip="Remove stored API key"
+ onClick={removeAppConfig}
+ loading={isRemovingAppConfig}
+ icon={<IconTrash className="text-danger" />}
+ />
+ }
+ />
+ </div>
+ <p className="text-sm text-muted">Safe to rerun, existing categories and users will be updated.</p>
+ </>
+ ) : (
+ <Form logic={customerIOImportLogic} formKey="importForm" enableFormOnSubmit>
+ <LemonField name="app_api_key" label="Customer.io App API Key">
+ <LemonInput
+ placeholder="Enter your App API key"
+ type="password"
+ data-attr="customerio-api-key"
+ autoComplete="off"
+ />
+ </LemonField>
+ <div className="text-xs text-muted-alt mt-2">
+ You can generate an App API key in Customer.io under Settings → Account Settings → API
+ Credentials.
+ </div>
+ </Form>
+ )}
+
+ {importError && (
+ <LemonBanner type="error" className="text-sm">
+ {importError}
+ </LemonBanner>
+ )}
+
+ <div className="flex justify-end">
+ {hasStoredKey ? (
+ <LemonButton
+ type="primary"
+ onClick={() => startImport({ apiKey: null })}
+ loading={startedImportRunLoading}
+ disabledReason={noStepsReason}
+ data-attr="customerio-import-run-again"
+ >
+ {latestImportRun ? 'Run again' : 'Start import'}
+ </LemonButton>
+ ) : (
+ <LemonButton
+ type="primary"
+ onClick={submitImportForm}
+ loading={startedImportRunLoading}
+ disabledReason={!importForm.app_api_key ? 'Enter your API key' : noStepsReason}
+ data-attr="customerio-import-start"
+ >
+ Start import
+ </LemonButton>
+ )}
+ </div>
+ </div>
+ )
+}
+
+function InRequestImportContent(): JSX.Element {
const { isImporting, importProgress, importError, importForm, syncConfig, isRemovingAppConfig } =
useValues(customerIOImportLogic)
const { submitImportForm, rerunImport, removeAppConfig } = useActions(customerIOImportLogic)
@@ -184,6 +395,11 @@ function Step1Content(): JSX.Element {
)
}
+function Step1Content(): JSX.Element {
+ const { isBackgroundImportEnabled } = useValues(customerIOImportLogic)
+ return isBackgroundImportEnabled ? <BackgroundImportContent /> : <InRequestImportContent />
+}
+
function Step2Content(): JSX.Element {
const { csvFile, csvProgress, isUploadingCSV, syncConfig } = useValues(customerIOImportLogic)
const { setCSVFile, uploadCSV } = useActions(customerIOImportLogic)
diff --git a/products/workflows/frontend/OptOuts/CustomerIOImportRunProgress.tsx b/products/workflows/frontend/OptOuts/CustomerIOImportRunProgress.tsx
new file mode 100644
index 000000000000..f6ad973dfdb5
--- /dev/null
+++ b/products/workflows/frontend/OptOuts/CustomerIOImportRunProgress.tsx
@@ -0,0 +1,50 @@
+import { IconCheckCircle, IconCircleDashed, IconWarning } from '@posthog/icons'
+import { Spinner } from '@posthog/lemon-ui'
+
+import type {
+ ImportStepProgressApi,
+ ImportStepStatusEnumApi,
+ ProviderImportRunApi,
+} from 'products/messaging/frontend/generated/api.schemas'
+
+const STEP_ICONS: Record<ImportStepStatusEnumApi, JSX.Element> = {
+ pending: <IconCircleDashed className="text-muted text-base shrink-0" />,
+ running: <Spinner className="text-base shrink-0" />,
+ completed: <IconCheckCircle className="text-success text-base shrink-0" />,
+ failed: <IconWarning className="text-danger text-base shrink-0" />,
+}
+
+function stepCounts(step: ImportStepProgressApi): string {
+ if (step.status === 'pending') {
+ return 'Not started'
+ }
+ return `${step.processed.toLocaleString()} read · ${step.imported.toLocaleString()} new or changed`
+}
+
+export function CustomerIOImportRunProgress({ run }: { run: ProviderImportRunApi }): JSX.Element {
+ return (
+ <ul className="flex flex-col gap-3 m-0 p-0 list-none" data-attr="customerio-import-run-progress">
+ {run.steps.map((step) => (
+ <li key={step.key} className="flex flex-col gap-1">
+ <div className="flex flex-wrap items-center justify-between gap-x-4 gap-y-1">
+ <span className="flex items-center gap-2 font-medium">
+ {STEP_ICONS[step.status]}
+ <span>{step.label}</span>
+ </span>
+ <span className="text-xs text-secondary" translate="no">
+ {stepCounts(step)}
+ </span>
+ </div>
+ {step.last_error && (
+ <div className={step.status === 'failed' ? 'text-xs text-danger' : 'text-xs text-warning'}>
+ {step.status === 'failed' ? step.last_error : `Retrying after an error: ${step.last_error}`}
+ </div>
+ )}
+ {step.warnings.length > 0 && (
+ <div className="text-xs text-secondary">{step.warnings.join(' ')}</div>
+ )}
+ </li>
+ ))}
+ </ul>
+ )
+}
diff --git a/products/workflows/frontend/OptOuts/customerIOImportLogic.test.ts b/products/workflows/frontend/OptOuts/customerIOImportLogic.test.ts
new file mode 100644
index 000000000000..a862f03ac019
--- /dev/null
+++ b/products/workflows/frontend/OptOuts/customerIOImportLogic.test.ts
@@ -0,0 +1,138 @@
+import { expectLogic } from 'kea-test-utils'
+
+import { FEATURE_FLAGS } from 'lib/constants'
+import { featureFlagLogic } from 'lib/logic/featureFlagLogic'
+
+import { useMocks } from '~/mocks/jest'
+import { initKeaTests } from '~/test/init'
+
+import type { ProviderImportRunApi } from 'products/messaging/frontend/generated/api.schemas'
+
+import { customerIOImportLogic } from './customerIOImportLogic'
+
+const STEP_OPTIONS = [
+ { key: 'categories', label: 'Subscription topics', description: '' },
+ { key: 'suppressions', label: 'Suppressed addresses', description: '' },
+ { key: 'opt_outs', label: 'Global unsubscribes', description: '' },
+]
+
+const importRun = (status: ProviderImportRunApi['status']): ProviderImportRunApi => ({
+ id: 'run-1',
+ provider: 'customerio',
+ status,
+ steps: [
+ {
+ key: 'suppressions',
+ label: 'Suppressed addresses',
+ status: status === 'completed' ? 'completed' : 'running',
+ processed: 1000,
+ imported: 1000,
+ warnings: [],
+ last_error: null,
+ },
+ ],
+ created_at: '2026-01-01T00:00:00Z',
+ started_at: '2026-01-01T00:00:01Z',
+ finished_at: status === 'completed' ? '2026-01-01T00:05:00Z' : null,
+})
+
+describe('customerIOImportLogic', () => {
+ let logic: ReturnType<typeof customerIOImportLogic.build>
+ let runResponses: [number, ProviderImportRunApi[] | { detail: string }][]
+ let listCalls: number
+ let startBodies: Record<string, unknown>[]
+ let inRequestImports: number
+
+ beforeEach(() => {
+ runResponses = [
+ [200, []],
+ [500, { detail: 'boom' }],
+ [200, [importRun('completed')]],
+ ]
+ listCalls = 0
+ startBodies = []
+ inRequestImports = 0
+ useMocks({
+ get: {
+ '/api/environments/:team_id/messaging_categories/optout_sync_config/': () => [
+ 200,
+ { app_integration_id: null },
+ ],
+ '/api/projects/:team_id/messaging_provider_imports/providers/': () => [
+ 200,
+ [{ key: 'customerio', label: 'Customer.io', steps: STEP_OPTIONS }],
+ ],
+ '/api/projects/:team_id/messaging_provider_imports/': () => {
+ listCalls += 1
+ return runResponses[Math.min(listCalls - 1, runResponses.length - 1)]
+ },
+ },
+ post: {
+ '/api/:scope/:team_id/messaging_categories/import_from_customerio/': () => {
+ inRequestImports += 1
+ return [200, { status: 'completed', topics_found: 0, errors: [] }]
+ },
+ '/api/projects/:team_id/messaging_provider_imports/': async ({ request }) => {
+ startBodies.push((await request.json()) as Record<string, unknown>)
+ return [201, importRun('queued')]
+ },
+ },
+ })
+ initKeaTests()
+ logic = customerIOImportLogic()
+ logic.mount()
+ })
+
+ afterEach(() => {
+ logic.unmount()
+ jest.useRealTimers()
+ })
+
+ it('starts the selected steps and polls the run until it finishes, through a failed refresh', async () => {
+ featureFlagLogic.mount()
+ featureFlagLogic.actions.setFeatureFlags([FEATURE_FLAGS.WORKFLOWS_CUSTOMERIO_BACKGROUND_IMPORT], {
+ [FEATURE_FLAGS.WORKFLOWS_CUSTOMERIO_BACKGROUND_IMPORT]: true,
+ })
+ await expectLogic(logic, () => logic.actions.openImportModal()).toFinishAllListeners()
+ jest.useFakeTimers()
+ logic.actions.toggleImportStep('categories')
+
+ await expectLogic(logic, () => logic.actions.startImport({ apiKey: 'app-key' })).toDispatchActions([
+ 'startImportSuccess',
+ 'loadImportRunsSuccess',
+ ])
+
+ expect(startBodies).toEqual([
+ { provider: 'customerio', steps: ['suppressions', 'opt_outs'], api_key: 'app-key' },
+ ])
+ expect(logic.values.isImportRunActive).toBe(true)
+
+ await jest.advanceTimersByTimeAsync(2000)
+ await expectLogic(logic).toDispatchActions(['loadImportRunsFailure'])
+ expect(logic.values.isImportRunActive).toBe(true)
+
+ await jest.advanceTimersByTimeAsync(2000)
+ await expectLogic(logic).toDispatchActions(['loadImportRunsSuccess'])
+
+ expect(logic.values.latestImportRun?.status).toBe('completed')
+ expect(logic.values.stepCompletion.step1).toBe('completed')
+
+ const callsWhenFinished = listCalls
+ await jest.advanceTimersByTimeAsync(10000)
+
+ expect(listCalls).toBe(callsWhenFinished)
+ })
+
+ it('keeps the in-request import while the flag is off', async () => {
+ featureFlagLogic.mount()
+ featureFlagLogic.actions.setFeatureFlags([], {})
+ await expectLogic(logic, () => logic.actions.openImportModal()).toFinishAllListeners()
+ logic.actions.setImportFormValue('app_api_key', 'app-key')
+
+ await expectLogic(logic, () => logic.actions.submitImportForm()).toFinishAllListeners()
+
+ expect(inRequestImports).toBe(1)
+ expect(startBodies).toEqual([])
+ expect(listCalls).toBe(0)
+ })
+})
diff --git a/products/workflows/frontend/OptOuts/customerIOImportLogic.ts b/products/workflows/frontend/OptOuts/customerIOImportLogic.ts
index 2489fc8f2be8..6dec9daaf1a8 100644
--- a/products/workflows/frontend/OptOuts/customerIOImportLogic.ts
+++ b/products/workflows/frontend/OptOuts/customerIOImportLogic.ts
@@ -4,13 +4,31 @@ import type { DeepPartial, DeepPartialMap, FieldName, ValidationErrorType } from
import { loaders } from 'kea-loaders'
import { ApiRequest, getCookie } from 'lib/api'
+import { FEATURE_FLAGS } from 'lib/constants'
import { lemonToast } from 'lib/lemon-ui/LemonToast'
+import { featureFlagLogic } from 'lib/logic/featureFlagLogic'
+import type { FeatureFlagsSet } from 'lib/logic/featureFlagLogic'
import { preflightLogic } from 'scenes/PreflightCheck/preflightLogic'
import { teamLogic } from 'scenes/teamLogic'
+import {
+ messagingProviderImportsCreate,
+ messagingProviderImportsList,
+ messagingProviderImportsProvidersList,
+} from 'products/messaging/frontend/generated/api'
+import type {
+ ImportProviderOptionApi,
+ ImportStepOptionApi,
+ ProviderImportRunApi,
+} from 'products/messaging/frontend/generated/api.schemas'
+
import type { PreflightStatus } from '../../../../frontend/src/types'
import { optOutCategoriesLogic } from './optOutCategoriesLogic'
+const CUSTOMERIO_PROVIDER = 'customerio'
+const IMPORT_RUN_POLL_INTERVAL_MS = 2000
+const ACTIVE_IMPORT_RUN_STATUSES: ProviderImportRunApi['status'][] = ['queued', 'running']
+
export interface ImportFormValues {
app_api_key: string
}
@@ -74,10 +92,12 @@ export interface OptOutSyncConfigResponse {
// Generated by kea-typegen. Update if you're an agent, ignore if you're human.
export interface customerIOImportLogicValues {
+ featureFlags: FeatureFlagsSet // featureFlagLogic
preflight: PreflightStatus | null // preflightLogic
currentTeamIdStrict: number | string // teamLogic
csvFile: File | null
csvProgress: CSVImportProgress | null
+ deselectedImportSteps: string[]
hasTrackCredentials: boolean
importError: string | null
importForm: ImportFormValues
@@ -90,11 +110,16 @@ export interface customerIOImportLogicValues {
importFormTouches: Record<string, boolean>
importFormValidationErrors: DeepPartialMap<ImportFormValues, ValidationErrorType>
importProgress: ImportProgress | null
- isImportComplete: boolean
- isImportFailed: boolean
+ importProviders: ImportProviderOptionApi[] | null
+ importProvidersLoading: boolean
+ importRuns: ProviderImportRunApi[] | null
+ importRunsLoading: boolean
+ importStepOptions: ImportStepOptionApi[]
+ isBackgroundImportEnabled: boolean
isImportFormSubmitting: boolean
isImportFormValid: boolean
isImportModalOpen: boolean
+ isImportRunActive: boolean
isImporting: boolean
isRemovingAppConfig: boolean
isRemovingTrackConfig: boolean
@@ -102,7 +127,11 @@ export interface customerIOImportLogicValues {
isSavingTrack: boolean
isSavingWebhook: boolean
isUploadingCSV: boolean
+ latestImportRun: ProviderImportRunApi | null
+ selectedImportSteps: string[]
showImportFormErrors: boolean
+ startedImportRun: ProviderImportRunApi | null
+ startedImportRunLoading: boolean
stepCompletion: {
step1: 'completed' | 'failed' | false
step2: 'completed' | 'failed' | false
@@ -116,6 +145,7 @@ export interface customerIOImportLogicValues {
trackError: string | null
trackRegion: string
trackSiteId: string
+ trackedImportRunId: string | null
webhookError: string | null
webhookSigningSecret: string
webhookUrl: string
@@ -126,6 +156,36 @@ export interface customerIOImportLogicActions {
closeImportModal: () => {
value: true
}
+ loadImportProviders: () => any
+ loadImportProvidersFailure: (
+ error: string,
+ errorObject?: any
+ ) => {
+ error: string
+ errorObject?: any
+ }
+ loadImportProvidersSuccess: (
+ importProviders: ImportProviderOptionApi[],
+ payload?: any
+ ) => {
+ importProviders: ImportProviderOptionApi[]
+ payload?: any
+ }
+ loadImportRuns: () => any
+ loadImportRunsFailure: (
+ error: string,
+ errorObject?: any
+ ) => {
+ error: string
+ errorObject?: any
+ }
+ loadImportRunsSuccess: (
+ importRuns: ProviderImportRunApi[],
+ payload?: any
+ ) => {
+ importRuns: ProviderImportRunApi[]
+ payload?: any
+ }
loadSyncConfig: () => any
loadSyncConfigFailure: (
error: string,
@@ -214,12 +274,36 @@ export interface customerIOImportLogicActions {
setTrackSiteId: (siteId: string) => {
siteId: string
}
+ setTrackedImportRunId: (runId: string | null) => {
+ runId: string | null
+ }
setWebhookError: (error: string | null) => {
error: string | null
}
setWebhookSigningSecret: (secret: string) => {
secret: string
}
+ startImport: ({ apiKey }: { apiKey: string | null }) => {
+ apiKey: string | null
+ }
+ startImportFailure: (
+ error: string,
+ errorObject?: any
+ ) => {
+ error: string
+ errorObject?: any
+ }
+ startImportSuccess: (
+ startedImportRun: ProviderImportRunApi,
+ payload?: {
+ apiKey: string | null
+ }
+ ) => {
+ startedImportRun: ProviderImportRunApi
+ payload?: {
+ apiKey: string | null
+ }
+ }
submitImportForm: () => {
value: boolean
}
@@ -236,6 +320,9 @@ export interface customerIOImportLogicActions {
submitImportFormSuccess: (importForm: ImportFormValues) => {
importForm: ImportFormValues
}
+ toggleImportStep: (step: string) => {
+ step: string
+ }
toggleTrackSync: (enabled: boolean) => {
enabled: boolean
}
@@ -253,12 +340,17 @@ export interface customerIOImportLogicActions {
// Generated by kea-typegen. Update if you're an agent, ignore if you're human.
export interface customerIOImportLogicMeta {
__keaTypeGenInternalSelectorTypes: {
+ isBackgroundImportEnabled: (featureFlags: FeatureFlagsSet) => boolean
isImporting: (isImportFormSubmitting: boolean, importProgress: ImportProgress | null) => boolean
- isImportComplete: (importProgress: ImportProgress | null) => boolean
- isImportFailed: (importProgress: ImportProgress | null) => boolean
+ latestImportRun: (importRuns: ProviderImportRunApi[] | null) => ProviderImportRunApi | null
+ isImportRunActive: (latestImportRun: ProviderImportRunApi | null) => boolean
+ importStepOptions: (importProviders: ImportProviderOptionApi[] | null) => ImportStepOptionApi[]
+ selectedImportSteps: (importStepOptions: ImportStepOptionApi[], deselectedImportSteps: string[]) => string[]
webhookUrl: (currentTeamIdStrict: number | string) => string
stepCompletion: (
syncConfig: OptOutSyncConfigResponse | null,
+ isBackgroundImportEnabled: boolean,
+ latestImportRun: ProviderImportRunApi | null,
importProgress: ImportProgress | null,
csvProgress: CSVImportProgress | null
) => {
@@ -282,7 +374,7 @@ export type customerIOImportLogicType = MakeLogicType<
export const customerIOImportLogic = kea<customerIOImportLogicType>([
path(['products', 'workflows', 'customerIOImportLogic']),
connect(() => ({
- values: [teamLogic, ['currentTeamIdStrict'], preflightLogic, ['preflight']],
+ values: [teamLogic, ['currentTeamIdStrict'], preflightLogic, ['preflight'], featureFlagLogic, ['featureFlags']],
})),
actions({
openImportModal: true,
@@ -291,6 +383,8 @@ export const customerIOImportLogic = kea<customerIOImportLogicType>([
setImportProgress: (importProgress: ImportProgress) => ({ importProgress }),
setImportError: (error: string | null) => ({ error }),
rerunImport: true,
+ toggleImportStep: (step: string) => ({ step }),
+ setTrackedImportRunId: (runId: string | null) => ({ runId }),
removeAppConfig: true,
setCSVFile: (file: File | null) => ({ file }),
uploadCSV: true,
@@ -311,7 +405,7 @@ export const customerIOImportLogic = kea<customerIOImportLogicType>([
setIsSavingTrack: (isSaving: boolean) => ({ isSaving }),
setTrackError: (error: string | null) => ({ error }),
}),
- loaders({
+ loaders(({ values }) => ({
syncConfig: [
null as OptOutSyncConfigResponse | null,
{
@@ -320,7 +414,34 @@ export const customerIOImportLogic = kea<customerIOImportLogicType>([
},
},
],
- }),
+ importRuns: [
+ null as ProviderImportRunApi[] | null,
+ {
+ loadImportRuns: async () => {
+ const runs = await messagingProviderImportsList(String(values.currentTeamIdStrict))
+ return runs.filter((run) => run.provider === CUSTOMERIO_PROVIDER)
+ },
+ },
+ ],
+ importProviders: [
+ null as ImportProviderOptionApi[] | null,
+ {
+ loadImportProviders: async () =>
+ await messagingProviderImportsProvidersList(String(values.currentTeamIdStrict)),
+ },
+ ],
+ startedImportRun: [
+ null as ProviderImportRunApi | null,
+ {
+ startImport: async ({ apiKey }: { apiKey: string | null }) =>
+ await messagingProviderImportsCreate(String(values.currentTeamIdStrict), {
+ provider: CUSTOMERIO_PROVIDER,
+ steps: values.selectedImportSteps,
+ ...(apiKey ? { api_key: apiKey } : {}),
+ }),
+ },
+ ],
+ })),
reducers({
isImportModalOpen: [
false,
@@ -347,9 +468,26 @@ export const customerIOImportLogic = kea<customerIOImportLogicType>([
return 'Import failed. Please check your API key and try again.'
},
submitImportFormSuccess: () => null,
+ startImport: () => null,
+ startImportFailure: (_, { error, errorObject }) =>
+ errorObject?.detail ?? error ?? "Couldn't start the import. Check your API key and try again.",
resetImport: () => null,
},
],
+ deselectedImportSteps: [
+ [] as string[],
+ {
+ toggleImportStep: (deselected, { step }) =>
+ deselected.includes(step) ? deselected.filter((key) => key !== step) : [...deselected, step],
+ },
+ ],
+ trackedImportRunId: [
+ null as string | null,
+ {
+ setTrackedImportRunId: (_, { runId }) => runId,
+ startImportSuccess: (_, { startedImportRun }) => startedImportRun.id,
+ },
+ ],
csvFile: [
null as File | null,
{
@@ -456,12 +594,16 @@ export const customerIOImportLogic = kea<customerIOImportLogicType>([
},
],
}),
- forms(({ actions }) => ({
+ forms(({ actions, values }) => ({
importForm: {
defaults: {
app_api_key: '',
} as ImportFormValues,
submit: async ({ app_api_key }) => {
+ if (values.isBackgroundImportEnabled) {
+ actions.startImport({ apiKey: app_api_key })
+ return
+ }
const response = await new ApiRequest()
.messagingCategoriesImportFromCustomerIO()
.create({ data: { app_api_key } })
@@ -474,18 +616,34 @@ export const customerIOImportLogic = kea<customerIOImportLogicType>([
},
})),
selectors({
+ isBackgroundImportEnabled: [
+ (s) => [s.featureFlags],
+ (featureFlags: FeatureFlagsSet): boolean =>
+ !!featureFlags[FEATURE_FLAGS.WORKFLOWS_CUSTOMERIO_BACKGROUND_IMPORT],
+ ],
isImporting: [
(s) => [s.isImportFormSubmitting, s.importProgress],
(isImportFormSubmitting: boolean, importProgress: ImportProgress | null) =>
isImportFormSubmitting || importProgress?.status === 'importing',
],
- isImportComplete: [
- (s) => [s.importProgress],
- (importProgress: ImportProgress | null) => importProgress?.status === 'completed',
+ latestImportRun: [
+ (s) => [s.importRuns],
+ (importRuns: ProviderImportRunApi[] | null): ProviderImportRunApi | null => importRuns?.[0] ?? null,
+ ],
+ isImportRunActive: [
+ (s) => [s.latestImportRun],
+ (latestImportRun: ProviderImportRunApi | null): boolean =>
+ !!latestImportRun && ACTIVE_IMPORT_RUN_STATUSES.includes(latestImportRun.status),
],
- isImportFailed: [
- (s) => [s.importProgress],
- (importProgress: ImportProgress | null) => importProgress?.status === 'failed',
+ importStepOptions: [
+ (s) => [s.importProviders],
+ (importProviders: ImportProviderOptionApi[] | null): ImportStepOptionApi[] =>
+ importProviders?.find((provider) => provider.key === CUSTOMERIO_PROVIDER)?.steps ?? [],
+ ],
+ selectedImportSteps: [
+ (s) => [s.importStepOptions, s.deselectedImportSteps],
+ (importStepOptions: ImportStepOptionApi[], deselectedImportSteps: string[]): string[] =>
+ importStepOptions.map((step) => step.key).filter((key) => !deselectedImportSteps.includes(key)),
],
webhookUrl: [
(s) => [s.currentTeamIdStrict],
@@ -495,9 +653,11 @@ export const customerIOImportLogic = kea<customerIOImportLogicType>([
},
],
stepCompletion: [
- (s) => [s.syncConfig, s.importProgress, s.csvProgress],
+ (s) => [s.syncConfig, s.isBackgroundImportEnabled, s.latestImportRun, s.importProgress, s.csvProgress],
(
syncConfig: OptOutSyncConfigResponse | null,
+ isBackgroundImportEnabled: boolean,
+ latestImportRun: ProviderImportRunApi | null,
importProgress: ImportProgress | null,
csvProgress: CSVImportProgress | null
): {
@@ -518,7 +678,10 @@ export const customerIOImportLogic = kea<customerIOImportLogicType>([
}
return {
- step1: resolveStatus(importProgress?.status, syncConfig?.app_import_result?.status),
+ step1: resolveStatus(
+ isBackgroundImportEnabled ? latestImportRun?.status : importProgress?.status,
+ syncConfig?.app_import_result?.status
+ ),
step2: resolveStatus(csvProgress?.status, syncConfig?.csv_import_result?.status),
step3: syncConfig?.webhook_enabled ? 'completed' : false,
step4: syncConfig?.track_enabled ? 'completed' : false,
@@ -534,11 +697,61 @@ export const customerIOImportLogic = kea<customerIOImportLogicType>([
(syncConfig: OptOutSyncConfigResponse | null): boolean => syncConfig?.has_track_credentials ?? false,
],
}),
- listeners(({ actions, values }) => ({
+ listeners(({ actions, values, cache }) => ({
openImportModal: () => {
actions.loadSyncConfig()
+ if (values.isBackgroundImportEnabled) {
+ actions.loadImportRuns()
+ actions.loadImportProviders()
+ }
actions.setTrackRegion(values.preflight?.region ?? 'US')
},
+ startImportSuccess: ({ startedImportRun }) => {
+ // Show the new run at once, so a failed first refresh still shows progress and keeps polling.
+ actions.loadImportRunsSuccess([
+ startedImportRun,
+ ...(values.importRuns ?? []).filter((run) => run.id !== startedImportRun.id),
+ ])
+ actions.resetImportForm()
+ // The first start stores the API key, which the stored-key view reads from the sync config.
+ actions.loadSyncConfig()
+ },
+ startImportFailure: ({ errorObject }) => {
+ // A 409 means an import is already running, so show its progress instead of only an error.
+ if (errorObject?.status === 409) {
+ actions.loadImportRuns()
+ }
+ },
+ loadImportRunsSuccess: ({ importRuns }) => {
+ const latest = importRuns?.[0] ?? null
+ if (latest && ACTIVE_IMPORT_RUN_STATUSES.includes(latest.status)) {
+ actions.setTrackedImportRunId(latest.id)
+ cache.disposables.add(() => {
+ const timeout = setTimeout(() => actions.loadImportRuns(), IMPORT_RUN_POLL_INTERVAL_MS)
+ return () => clearTimeout(timeout)
+ }, 'importRunPoll')
+ return
+ }
+ cache.disposables.dispose('importRunPoll')
+ if (latest && latest.id === values.trackedImportRunId) {
+ actions.setTrackedImportRunId(null)
+ if (latest.status === 'completed') {
+ lemonToast.success('Customer.io import finished.')
+ } else {
+ lemonToast.error('Some Customer.io import steps failed. Open the import to see which ones.')
+ }
+ optOutCategoriesLogic.findMounted()?.actions.loadCategories()
+ }
+ },
+ loadImportRunsFailure: () => {
+ // Keep polling through a failed request while an import is running, so one network error doesn't freeze the progress.
+ if (values.trackedImportRunId) {
+ cache.disposables.add(() => {
+ const timeout = setTimeout(() => actions.loadImportRuns(), IMPORT_RUN_POLL_INTERVAL_MS)
+ return () => clearTimeout(timeout)
+ }, 'importRunPoll')
+ }
+ },
removeAppConfig: async () => {
try {
await new ApiRequest().messagingCategoriesRemoveCustomerIOAppConfig().delete()