Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
feat(import/onboarding): support auto-import in onboarding (#5122)
teo10bd79c2a1ec
diff --git a/.sqlx/query-0a41e84a695b39ddf490db802922c943f6499b14467fd7a0d0b83b04af8abf9c.json b/.sqlx/query-0a41e84a695b39ddf490db802922c943f6499b14467fd7a0d0b83b04af8abf9c.jsonnew file mode 100644index 0000000000..6d1c68b86e--- /dev/null+++ b/.sqlx/query-0a41e84a695b39ddf490db802922c943f6499b14467fd7a0d0b83b04af8abf9c.json@@ -0,0 +1,17 @@+{+ "db_name": "PostgreSQL",+ "query": "\n INSERT INTO import_run (user_id, source, status, auto_import)\n VALUES ($1, $2, 'running', $4)\n ON CONFLICT (user_id, source) DO UPDATE\n SET status = 'running', error = NULL, updated_at = NOW()\n WHERE import_run.status::text = ANY($3::text[])\n ",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": [+ "Text",+ "Text",+ "TextArray",+ "Bool"+ ]+ },+ "nullable": []+ },+ "hash": "0a41e84a695b39ddf490db802922c943f6499b14467fd7a0d0b83b04af8abf9c"+}diff --git a/.sqlx/query-18b28d89f92ce787c86984be5c68b5577218c7000631fdec7b70dfcf5de516a6.json b/.sqlx/query-18b28d89f92ce787c86984be5c68b5577218c7000631fdec7b70dfcf5de516a6.jsonnew file mode 100644index 0000000000..1af4dcfd7e--- /dev/null+++ b/.sqlx/query-18b28d89f92ce787c86984be5c68b5577218c7000631fdec7b70dfcf5de516a6.json@@ -0,0 +1,15 @@+{+ "db_name": "PostgreSQL",+ "query": "\n DELETE FROM import_entity\n WHERE user_id = $1 AND initiator = $2 AND status = 'staged'\n AND NOT (\n $2 = 'onboarding'\n AND EXISTS (\n SELECT 1\n FROM import_run\n WHERE import_run.user_id = import_entity.user_id\n AND import_run.source = import_entity.source\n AND import_run.auto_import\n AND import_run.status IN ('running', 'ready', 'importing', 'failed')\n )\n )\n ",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": [+ "Text",+ "Text"+ ]+ },+ "nullable": []+ },+ "hash": "18b28d89f92ce787c86984be5c68b5577218c7000631fdec7b70dfcf5de516a6"+}diff --git a/.sqlx/query-d919f2560dd92b03ba4cde5d6f024a94ac6e340f12eee4ac14ccec55b7008485.json b/.sqlx/query-192b53847ed98ebf7e5506cf99eff776fed05fb7926d8fba33b31ff8b49fa1c3.jsonsimilarity index 58%rename from .sqlx/query-d919f2560dd92b03ba4cde5d6f024a94ac6e340f12eee4ac14ccec55b7008485.jsonrename to .sqlx/query-192b53847ed98ebf7e5506cf99eff776fed05fb7926d8fba33b31ff8b49fa1c3.jsonindex e5801db794..9772ba66d2 100644--- a/.sqlx/query-d919f2560dd92b03ba4cde5d6f024a94ac6e340f12eee4ac14ccec55b7008485.json+++ b/.sqlx/query-192b53847ed98ebf7e5506cf99eff776fed05fb7926d8fba33b31ff8b49fa1c3.json@@ -1,6 +1,6 @@{"db_name": "PostgreSQL",- "query": "\n SELECT source, status, error, updated_at\n FROM import_run WHERE user_id = $1\n ORDER BY source\n ",+ "query": "\n SELECT source, status::text AS \"status!\", auto_import, error, updated_at\n FROM import_run WHERE user_id = $1\n ORDER BY source\n ","describe": {"columns": [{@@ -10,16 +10,21 @@},{"ordinal": 1,- "name": "status",+ "name": "status!","type_info": "Text"},{"ordinal": 2,+ "name": "auto_import",+ "type_info": "Bool"+ },+ {+ "ordinal": 3,"name": "error","type_info": "Text"},{- "ordinal": 3,+ "ordinal": 4,"name": "updated_at","type_info": "Timestamptz"}@@ -31,10 +36,11 @@},"nullable": [false,+ null,false,true,false]},- "hash": "d919f2560dd92b03ba4cde5d6f024a94ac6e340f12eee4ac14ccec55b7008485"+ "hash": "192b53847ed98ebf7e5506cf99eff776fed05fb7926d8fba33b31ff8b49fa1c3"}diff --git a/.sqlx/query-6f216a9a7f92b281059f6a8d6445c8e839dae4e1e8eaec4efec7b131a583b5ac.json b/.sqlx/query-1f632dca1f9936c1fe418219f03744955d587014e1015ff4616e05aa8491e70d.jsonsimilarity index 52%rename from .sqlx/query-6f216a9a7f92b281059f6a8d6445c8e839dae4e1e8eaec4efec7b131a583b5ac.jsonrename to .sqlx/query-1f632dca1f9936c1fe418219f03744955d587014e1015ff4616e05aa8491e70d.jsonindex 0a518a34a7..a86c575272 100644--- a/.sqlx/query-6f216a9a7f92b281059f6a8d6445c8e839dae4e1e8eaec4efec7b131a583b5ac.json+++ b/.sqlx/query-1f632dca1f9936c1fe418219f03744955d587014e1015ff4616e05aa8491e70d.json@@ -1,6 +1,6 @@{"db_name": "PostgreSQL",- "query": "\n UPDATE import_run\n SET status = $4, updated_at = NOW()\n WHERE user_id = $1 AND source = $2 AND status = ANY($3::text[])\n ",+ "query": "\n UPDATE import_run\n SET status = $4::text::import_run_status, updated_at = NOW()\n WHERE user_id = $1 AND source = $2\n AND status::text = ANY($3::text[])\n ","describe": {"columns": [],"parameters": {@@ -13,5 +13,5 @@},"nullable": []},- "hash": "6f216a9a7f92b281059f6a8d6445c8e839dae4e1e8eaec4efec7b131a583b5ac"+ "hash": "1f632dca1f9936c1fe418219f03744955d587014e1015ff4616e05aa8491e70d"}diff --git a/.sqlx/query-50dbd26f9ca1a622f73b5bb7a5729e107dd4134fe6e77fea27bffae137f17359.json b/.sqlx/query-50dbd26f9ca1a622f73b5bb7a5729e107dd4134fe6e77fea27bffae137f17359.jsondeleted file mode 100644index f7e09aff03..0000000000--- a/.sqlx/query-50dbd26f9ca1a622f73b5bb7a5729e107dd4134fe6e77fea27bffae137f17359.json+++ /dev/null@@ -1,16 +0,0 @@-{- "db_name": "PostgreSQL",- "query": "\n INSERT INTO import_run (user_id, source, status)\n VALUES ($1, $2, 'running')\n ON CONFLICT (user_id, source) DO UPDATE\n SET status = 'running', error = NULL, updated_at = NOW()\n WHERE import_run.status = ANY($3::text[])\n ",- "describe": {- "columns": [],- "parameters": {- "Left": [- "Text",- "Text",- "TextArray"- ]- },- "nullable": []- },- "hash": "50dbd26f9ca1a622f73b5bb7a5729e107dd4134fe6e77fea27bffae137f17359"-}diff --git a/.sqlx/query-6335355244b0d3b14781c6983127a65b3ef3e669b837ffa239d5b8d74dc7ef8c.json b/.sqlx/query-6335355244b0d3b14781c6983127a65b3ef3e669b837ffa239d5b8d74dc7ef8c.jsonnew file mode 100644index 0000000000..31282a5aa7--- /dev/null+++ b/.sqlx/query-6335355244b0d3b14781c6983127a65b3ef3e669b837ffa239d5b8d74dc7ef8c.json@@ -0,0 +1,24 @@+{+ "db_name": "PostgreSQL",+ "query": "\n WITH outcome AS (\n SELECT\n COUNT(*) FILTER (WHERE status = 'imported')\n = CARDINALITY($3::uuid[])::bigint AS succeeded\n FROM import_entity\n WHERE user_id = $1 AND id = ANY($3::uuid[])\n )\n UPDATE import_run\n SET status = (CASE\n WHEN outcome.succeeded THEN 'completed'\n ELSE 'failed'\n END)::import_run_status,\n error = CASE\n WHEN outcome.succeeded THEN NULL\n ELSE 'one or more automatic imports failed'\n END,\n updated_at = NOW()\n FROM outcome\n WHERE user_id = $1 AND source = $2 AND status = 'importing'\n RETURNING import_run.status::text AS \"status!\"\n ",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "status!",+ "type_info": "Text"+ }+ ],+ "parameters": {+ "Left": [+ "Text",+ "Text",+ "UuidArray"+ ]+ },+ "nullable": [+ null+ ]+ },+ "hash": "6335355244b0d3b14781c6983127a65b3ef3e669b837ffa239d5b8d74dc7ef8c"+}diff --git a/.sqlx/query-80b17499f3576f22aaddce4a8f56edd5faa866172d94dbb7cb00cbcaa64f1f4a.json b/.sqlx/query-7e99cc59b745603c368e0ad6e977848c67c188791845c18ef54b39a2fbc74df6.jsonsimilarity index 50%rename from .sqlx/query-80b17499f3576f22aaddce4a8f56edd5faa866172d94dbb7cb00cbcaa64f1f4a.jsonrename to .sqlx/query-7e99cc59b745603c368e0ad6e977848c67c188791845c18ef54b39a2fbc74df6.jsonindex 719a17222f..d52d2ac1d1 100644--- a/.sqlx/query-80b17499f3576f22aaddce4a8f56edd5faa866172d94dbb7cb00cbcaa64f1f4a.json+++ b/.sqlx/query-7e99cc59b745603c368e0ad6e977848c67c188791845c18ef54b39a2fbc74df6.json@@ -1,6 +1,6 @@{"db_name": "PostgreSQL",- "query": "\n UPDATE import_run\n SET status = $3, error = $4, updated_at = NOW()\n WHERE user_id = $1 AND source = $2 AND status = 'running'\n ",+ "query": "\n UPDATE import_run\n SET status = $3::text::import_run_status,\n error = $4,\n updated_at = NOW()\n WHERE user_id = $1 AND source = $2 AND status = 'running'\n ","describe": {"columns": [],"parameters": {@@ -13,5 +13,5 @@},"nullable": []},- "hash": "80b17499f3576f22aaddce4a8f56edd5faa866172d94dbb7cb00cbcaa64f1f4a"+ "hash": "7e99cc59b745603c368e0ad6e977848c67c188791845c18ef54b39a2fbc74df6"}diff --git a/.sqlx/query-a53c213e1d4304506ece3fb431c15a14575a6c9eee0ff348f4b01eee0eacd2be.json b/.sqlx/query-a53c213e1d4304506ece3fb431c15a14575a6c9eee0ff348f4b01eee0eacd2be.jsonnew file mode 100644index 0000000000..56be6b6bd9--- /dev/null+++ b/.sqlx/query-a53c213e1d4304506ece3fb431c15a14575a6c9eee0ff348f4b01eee0eacd2be.json@@ -0,0 +1,23 @@+{+ "db_name": "PostgreSQL",+ "query": "\n UPDATE import_run\n SET status = 'importing', error = NULL, updated_at = NOW()\n WHERE user_id = $1 AND source = $2\n AND status = 'ready' AND auto_import\n RETURNING source\n ",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "source",+ "type_info": "Text"+ }+ ],+ "parameters": {+ "Left": [+ "Text",+ "Text"+ ]+ },+ "nullable": [+ false+ ]+ },+ "hash": "a53c213e1d4304506ece3fb431c15a14575a6c9eee0ff348f4b01eee0eacd2be"+}diff --git a/.sqlx/query-c00ffb94c8b6b3c21a28360c1e7d5026d39f79645826bcb2e8136d530ebc358b.json b/.sqlx/query-c00ffb94c8b6b3c21a28360c1e7d5026d39f79645826bcb2e8136d530ebc358b.jsonnew file mode 100644index 0000000000..430fd63cfa--- /dev/null+++ b/.sqlx/query-c00ffb94c8b6b3c21a28360c1e7d5026d39f79645826bcb2e8136d530ebc358b.json@@ -0,0 +1,14 @@+{+ "db_name": "PostgreSQL",+ "query": "\n WITH settled AS (\n SELECT\n run.source,\n EXISTS (\n SELECT 1\n FROM import_entity entity\n WHERE entity.user_id = run.user_id\n AND entity.source = run.source\n AND entity.initiator = 'onboarding'\n AND entity.status = 'staged'\n AND entity.last_error IS NOT NULL\n ) AS failed\n FROM import_run run\n WHERE run.user_id = $1\n AND run.status = 'importing'\n AND run.auto_import\n AND NOT EXISTS (\n SELECT 1\n FROM import_entity entity\n WHERE entity.user_id = run.user_id\n AND entity.source = run.source\n AND entity.initiator = 'onboarding'\n AND entity.status = 'importing'\n )\n )\n UPDATE import_run run\n SET status = (\n CASE WHEN settled.failed THEN 'failed' ELSE 'completed' END\n )::import_run_status,\n error = CASE\n WHEN settled.failed THEN 'one or more automatic imports failed'\n ELSE NULL\n END,\n updated_at = NOW()\n FROM settled\n WHERE run.user_id = $1\n AND run.source = settled.source\n AND run.status = 'importing'\n ",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": [+ "Text"+ ]+ },+ "nullable": []+ },+ "hash": "c00ffb94c8b6b3c21a28360c1e7d5026d39f79645826bcb2e8136d530ebc358b"+}diff --git a/.sqlx/query-d5a9632d0330f2dfc6dc0c6f6817f2ffce500a7f816bb5af32d46d74baeb793e.json b/.sqlx/query-d5a9632d0330f2dfc6dc0c6f6817f2ffce500a7f816bb5af32d46d74baeb793e.jsondeleted file mode 100644index b7f93d687d..0000000000--- a/.sqlx/query-d5a9632d0330f2dfc6dc0c6f6817f2ffce500a7f816bb5af32d46d74baeb793e.json+++ /dev/null@@ -1,15 +0,0 @@-{- "db_name": "PostgreSQL",- "query": "\n DELETE FROM import_entity\n WHERE user_id = $1 AND initiator = $2 AND status = 'staged'\n ",- "describe": {- "columns": [],- "parameters": {- "Left": [- "Text",- "Text"- ]- },- "nullable": []- },- "hash": "d5a9632d0330f2dfc6dc0c6f6817f2ffce500a7f816bb5af32d46d74baeb793e"-}diff --git a/.sqlx/query-ff8e627d8fcb9bbcc5751ede083004cfd6f9f4ccb817a23ec8cc868946fcb276.json b/.sqlx/query-ff8e627d8fcb9bbcc5751ede083004cfd6f9f4ccb817a23ec8cc868946fcb276.jsonnew file mode 100644index 0000000000..3181be44f2--- /dev/null+++ b/.sqlx/query-ff8e627d8fcb9bbcc5751ede083004cfd6f9f4ccb817a23ec8cc868946fcb276.json@@ -0,0 +1,95 @@+{+ "db_name": "PostgreSQL",+ "query": "\n UPDATE import_entity\n SET status = 'importing', last_error = NULL, updated_at = NOW()\n WHERE user_id = $1 AND source = $2\n AND initiator = 'onboarding' AND status = 'staged'\n RETURNING id, user_id, team_id, source, foreign_id, status, initiator,\n metadata, entity_id, entity_type, last_error, created_at, updated_at\n ",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "id",+ "type_info": "Uuid"+ },+ {+ "ordinal": 1,+ "name": "user_id",+ "type_info": "Text"+ },+ {+ "ordinal": 2,+ "name": "team_id",+ "type_info": "Uuid"+ },+ {+ "ordinal": 3,+ "name": "source",+ "type_info": "Text"+ },+ {+ "ordinal": 4,+ "name": "foreign_id",+ "type_info": "Text"+ },+ {+ "ordinal": 5,+ "name": "status",+ "type_info": "Text"+ },+ {+ "ordinal": 6,+ "name": "initiator",+ "type_info": "Text"+ },+ {+ "ordinal": 7,+ "name": "metadata",+ "type_info": "Jsonb"+ },+ {+ "ordinal": 8,+ "name": "entity_id",+ "type_info": "Text"+ },+ {+ "ordinal": 9,+ "name": "entity_type",+ "type_info": "Text"+ },+ {+ "ordinal": 10,+ "name": "last_error",+ "type_info": "Text"+ },+ {+ "ordinal": 11,+ "name": "created_at",+ "type_info": "Timestamptz"+ },+ {+ "ordinal": 12,+ "name": "updated_at",+ "type_info": "Timestamptz"+ }+ ],+ "parameters": {+ "Left": [+ "Text",+ "Text"+ ]+ },+ "nullable": [+ false,+ false,+ true,+ false,+ false,+ false,+ false,+ false,+ true,+ true,+ true,+ false,+ false+ ]+ },+ "hash": "ff8e627d8fcb9bbcc5751ede083004cfd6f9f4ccb817a23ec8cc868946fcb276"+}diff --git a/Cargo.lock b/Cargo.lockindex 3cbc9982ac..f4e2631542 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -7133,6 +7133,7 @@ dependencies = ["connection_gateway_client","futures","macro_authorization",+ "macro_db_migrator","macro_user_id","mcp_client","model-entity",diff --git a/apps/web/src/lib/queries/import.ts b/apps/web/src/lib/queries/import.tsindex cfab675344..f569245897 100644--- a/apps/web/src/lib/queries/import.ts+++ b/apps/web/src/lib/queries/import.ts@@ -51,8 +51,9 @@ const PENDING_IMPORT_STATE: ImportState = { runs: [], entities: [] };function anythingInFlight(state: ImportState | undefined): boolean {if (!state) return false;return (- state.runs.some((run) => run.status === 'running') ||- state.entities.some((entity) => entity.status === 'importing')+ state.runs.some(+ (run) => run.status === 'running' || run.status === 'importing'+ ) || state.entities.some((entity) => entity.status === 'importing'));}diff --git a/apps/web/src/lib/service-clients/service-cognition/generated/client.ts b/apps/web/src/lib/service-clients/service-cognition/generated/client.tsindex f270127084..1c12625d26 100644--- a/apps/web/src/lib/service-clients/service-cognition/generated/client.ts+++ b/apps/web/src/lib/service-clients/service-cognition/generated/client.ts@@ -1823,8 +1823,8 @@ export const getMemoryHandler = async (};/**- * @summary Complete (or skip) onboarding. Leftover onboarding-staged import-candidates are discarded.+ * @summary Complete (or skip) onboarding. Unreserved leftover onboarding-staged+import candidates are deleted.*/export type completeHandlerResponse200 = {data: OnboardingRow;diff --git a/apps/web/src/lib/service-clients/service-cognition/generated/schemas/importRun.ts b/apps/web/src/lib/service-clients/service-cognition/generated/schemas/importRun.tsindex 632e8a5127..91f8541d0f 100644--- a/apps/web/src/lib/service-clients/service-cognition/generated/schemas/importRun.ts+++ b/apps/web/src/lib/service-clients/service-cognition/generated/schemas/importRun.ts@@ -12,7 +12,10 @@ import type { RunStatus } from './runStatus';* Gather-run state for one user × source.*/export interface ImportRun {- /** Failure detail, when the run failed. */+ /** Whether this run should import its onboarding-staged candidates as+soon as gathering finishes. */+ auto_import: boolean;+ /** Gather or automatic-import failure detail, when the run failed. */error?: ImportRunError;/** The source this run gathered from. */source: ImportSource;diff --git a/apps/web/src/lib/service-clients/service-cognition/generated/schemas/importRunError.ts b/apps/web/src/lib/service-clients/service-cognition/generated/schemas/importRunError.tsindex c58739b0ec..9b7ec14e05 100644--- a/apps/web/src/lib/service-clients/service-cognition/generated/schemas/importRunError.ts+++ b/apps/web/src/lib/service-clients/service-cognition/generated/schemas/importRunError.ts@@ -6,6 +6,6 @@*//**- * Failure detail, when the run failed.+ * Gather or automatic-import failure detail, when the run failed.*/export type ImportRunError = string | null;diff --git a/apps/web/src/lib/service-clients/service-cognition/generated/schemas/runStatus.ts b/apps/web/src/lib/service-clients/service-cognition/generated/schemas/runStatus.tsindex 7e8f126e8d..9976265c9a 100644--- a/apps/web/src/lib/service-clients/service-cognition/generated/schemas/runStatus.ts+++ b/apps/web/src/lib/service-clients/service-cognition/generated/schemas/runStatus.ts@@ -14,6 +14,8 @@ export type RunStatus = (typeof RunStatus)[keyof typeof RunStatus];export const RunStatus = {running: 'running',ready: 'ready',+ importing: 'importing',+ completed: 'completed',failed: 'failed',dismissed: 'dismissed',} as const;diff --git a/apps/web/src/lib/service-clients/service-cognition/import.ts b/apps/web/src/lib/service-clients/service-cognition/import.tsindex e2bf0abdb9..eceb9b1024 100644--- a/apps/web/src/lib/service-clients/service-cognition/import.ts+++ b/apps/web/src/lib/service-clients/service-cognition/import.ts@@ -43,7 +43,13 @@ export type ImportEntityStatus =export type ImportInitiator = 'onboarding' | 'chat';-export type ImportRunStatus = 'running' | 'ready' | 'failed' | 'dismissed';+export type ImportRunStatus =+ | 'running'+ | 'ready'+ | 'importing'+ | 'completed'+ | 'failed'+ | 'dismissed';/** Metadata shape for staged Linear issues (`source === 'linear'`). */export type LinearIssueMeta = {@@ -100,6 +106,7 @@ export type ImportEntity = {export type ImportRun = {source: ImportSource;status: ImportRunStatus;+ auto_import: boolean;error?: string | null;updated_at: string;};diff --git a/apps/web/src/lib/service-clients/service-cognition/openapi.json b/apps/web/src/lib/service-clients/service-cognition/openapi.jsonindex 13e6b1a9d1..ea9b06e59b 100644--- a/apps/web/src/lib/service-clients/service-cognition/openapi.json+++ b/apps/web/src/lib/service-clients/service-cognition/openapi.json@@ -1493,7 +1493,7 @@"/onboarding/complete": {"post": {"tags": ["onboarding"],- "summary": "Complete (or skip) onboarding. Leftover onboarding-staged import\ncandidates are discarded.",+ "summary": "Complete (or skip) onboarding. Unreserved leftover onboarding-staged\nimport candidates are deleted.","operationId": "complete_handler","requestBody": {"content": {@@ -3261,11 +3261,15 @@"ImportRun": {"type": "object","description": "Gather-run state for one user × source.",- "required": ["source", "status", "updated_at"],+ "required": ["source", "status", "auto_import", "updated_at"],"properties": {+ "auto_import": {+ "type": "boolean",+ "description": "Whether this run should import its onboarding-staged candidates as\nsoon as gathering finishes."+ },"error": {"type": ["string", "null"],- "description": "Failure detail, when the run failed."+ "description": "Gather or automatic-import failure detail, when the run failed."},"source": {"$ref": "#/components/schemas/ImportSource",@@ -3698,7 +3702,14 @@"RunStatus": {"type": "string","description": "Lifecycle of a gather run (one per user × source).",- "enum": ["running", "ready", "failed", "dismissed"]+ "enum": [+ "running",+ "ready",+ "importing",+ "completed",+ "failed",+ "dismissed"+ ]},"SendChatMessagePayload": {"allOf": [diff --git a/crates/import/Cargo.toml b/crates/import/Cargo.tomlindex 16856270ea..74ed10e112 100644--- a/crates/import/Cargo.toml+++ b/crates/import/Cargo.toml@@ -31,3 +31,6 @@ tracing = { workspace = true }utoipa = { workspace = true }uuid = { workspace = true }workspace-hack = { version = "0.1", path = "../workspace-hack" }++[dev-dependencies]+macro_db_migrator = { path = "../macro_db_migrator" }diff --git a/crates/import/src/domain/models.rs b/crates/import/src/domain/models.rsindex 25969cc463..de7211a5f1 100644--- a/crates/import/src/domain/models.rs+++ b/crates/import/src/domain/models.rs@@ -168,7 +168,12 @@ pub enum RunStatus {Running,/// The gather finished; staged rows (if any) are ready.Ready,- /// The gather failed; `error` has details. Retryable.+ /// The gather finished and its configured automatic import is running.+ Importing,+ /// The configured automatic import finished successfully.+ Completed,+ /// Gathering or automatic importing failed; `error` has details.+ /// Retryable.Failed,/// The user dismissed this source's import section.Dismissed,@@ -413,7 +418,10 @@ pub struct ImportRun {pub source: ImportSource,/// Where the run is in its lifecycle.pub status: RunStatus,- /// Failure detail, when the run failed.+ /// Whether this run should import its onboarding-staged candidates as+ /// soon as gathering finishes.+ pub auto_import: bool,+ /// Gather or automatic-import failure detail, when the run failed.pub error: Option<String>,/// When the run state last changed.pub updated_at: DateTime<Utc>,diff --git a/crates/import/src/domain/models/test.rs b/crates/import/src/domain/models/test.rsindex 66a3f53d32..a66f4dfcd7 100644--- a/crates/import/src/domain/models/test.rs+++ b/crates/import/src/domain/models/test.rs@@ -30,6 +30,8 @@ fn enums_round_trip_through_strings() {for run in [RunStatus::Running,RunStatus::Ready,+ RunStatus::Importing,+ RunStatus::Completed,RunStatus::Failed,RunStatus::Dismissed,] {diff --git a/crates/import/src/domain/ports.rs b/crates/import/src/domain/ports.rsindex 0985e24fc2..198475c51d 100644--- a/crates/import/src/domain/ports.rs+++ b/crates/import/src/domain/ports.rs@@ -153,10 +153,11 @@ pub trait ImportRepo: Send + Sync + 'static {initiator: Initiator,) -> impl Future<Output = Result<u64>> + Send;- /// DELETE all of the user's remaining `staged` rows with the given- /// initiator (onboarding completion cleanup: candidates the user never- /// acted on vanish from the ledger and stay re-stageable later, unlike- /// discarded rows). Returns how many were removed.+ /// DELETE the user's remaining unreserved `staged` rows with the given+ /// initiator. Onboarding candidates reserved by an active or retryable+ /// configured auto-import run survive completion cleanup; other+ /// candidates the user never acted on vanish and stay re-stageable later.+ /// Returns how many were removed.fn delete_staged_by_initiator(&self,user: &MacroUserIdStr<'static>,@@ -175,14 +176,16 @@ pub trait ImportRepo: Send + Sync + 'static {user: &MacroUserIdStr<'static>,) -> impl Future<Output = Result<Vec<ImportRun>>> + Send;- /// CAS a gather run to `running`. Wins when no run row exists yet, or- /// when the existing row's status is in `from`. Returns whether this- /// call won.+ /// CAS a gather run to `running`. New rows persist `auto_import`+ /// atomically; retries preserve the existing run's setting. Wins when no+ /// run row exists yet, or when the existing row's status is in `from`.+ /// Returns whether this call won.fn start_run(&self,user: &MacroUserIdStr<'static>,source: ImportSource,from: &[RunStatus],+ auto_import: bool,) -> impl Future<Output = Result<bool>> + Send;/// CAS the `running` run for `source` to `to` (ready/failed), recording@@ -203,6 +206,33 @@ pub trait ImportRepo: Send + Sync + 'static {from: &[RunStatus],to: RunStatus,) -> impl Future<Output = Result<bool>> + Send;++ /// Atomically claim a ready, auto-import-enabled run and all of its own+ /// onboarding-staged candidates. `None` means another caller won or the+ /// run is not ready/configured; `Some([])` is a valid empty run.+ fn begin_auto_import(+ &self,+ user: &MacroUserIdStr<'static>,+ source: ImportSource,+ ) -> impl Future<Output = Result<Option<Vec<ImportEntity>>>> + Send;++ /// Finish an auto-import run after every claimed row has settled. The run+ /// becomes `completed` only when all claimed rows are imported; otherwise+ /// it becomes `failed`. Returns the terminal status when this call won.+ fn finish_auto_import(+ &self,+ user: &MacroUserIdStr<'static>,+ source: ImportSource,+ ids: &[Uuid],+ ) -> impl Future<Output = Result<Option<RunStatus>>> + Send;++ /// Repair auto-import run rows after process interruption. An `importing`+ /// run with no importing onboarding rows is terminal: `failed` when a+ /// staged row carries an import error, otherwise `completed`.+ fn reconcile_auto_import_runs(+ &self,+ user: &MacroUserIdStr<'static>,+ ) -> impl Future<Output = Result<u64>> + Send;}/// System properties to set on an imported task, already normalized todiff --git a/crates/import/src/domain/service.rs b/crates/import/src/domain/service.rsindex b109072403..0f918c06c7 100644--- a/crates/import/src/domain/service.rs+++ b/crates/import/src/domain/service.rs@@ -131,11 +131,14 @@ pub trait ImportService: Send + Sync + 'static {) -> impl Future<Output = Result<ImportState>> + Send;/// Start a gather run for `source` if none has ever run (the- /// connector-just-authenticated hook). Returns whether a run started.+ /// connector-just-authenticated hook). `auto_import` is persisted with+ /// the new run before its background gather starts. Returns whether a+ /// run started.fn start_gather(&self,user: MacroUserIdStr<'static>,source: ImportSource,+ auto_import: bool,) -> impl Future<Output = Result<bool>> + Send;/// Restart a failed (or dismissed) gather run. Returns whether a run@@ -171,10 +174,11 @@ pub trait ImportService: Send + Sync + 'static {initiator: Initiator,) -> impl Future<Output = Result<u64>> + Send;- /// Delete all remaining staged rows with `initiator` (onboarding- /// completion cleanup). Unlike discarding, deleted candidates were- /// never reviewed and stay re-stageable by later gathers or chat.- /// Returns how many were removed.+ /// Delete remaining unreserved staged rows with `initiator` (onboarding+ /// completion cleanup). Candidates reserved by active or retryable+ /// configured auto-import runs survive. Unlike discarding, deleted+ /// candidates were never reviewed and stay re-stageable by later gathers+ /// or chat. Returns how many were removed.fn delete_staged_by_initiator(&self,user: MacroUserIdStr<'static>,@@ -305,6 +309,7 @@ wheretokio::time::timeout(GATHER_TIMEOUT, service.run_gather_session(&user, source)).await.unwrap_or_else(|_| Err(anyhow::anyhow!("gather session timed out")));+ let gather_succeeded = outcome.is_ok();let finished = match outcome {Ok(()) => {@@ -321,13 +326,67 @@ where.await}};- if let Err(e) = finished {- tracing::error!(source = source.as_ref(), error = ?e, "failed to persist gather outcome");+ match finished {+ Ok(true) if gather_succeeded => {+ let _ = service+ .maybe_start_auto_import(&user, source)+ .await+ .inspect_err(|e| {+ tracing::error!(source = source.as_ref(), error = ?e, "failed to start automatic import");+ });+ }+ Ok(_) => {}+ Err(e) => {+ tracing::error!(source = source.as_ref(), error = ?e, "failed to persist gather outcome");+ }}service.notify(&user).await;});}+ /// Claim and spawn a configured automatic import once gathering is ready.+ /// The repository CAS makes this safe when completion, configuration, and+ /// read-path reconciliation race across service replicas.+ async fn maybe_start_auto_import(+ &self,+ user: &MacroUserIdStr<'static>,+ source: ImportSource,+ ) -> Result<bool> {+ let Some(rows) = self.repo.begin_auto_import(user, source).await? else {+ return Ok(false);+ };+ let ids: Vec<Uuid> = rows.iter().map(|row| row.id).collect();+ self.notify(user).await;+ if rows.is_empty() {+ self.finish_auto_import_batch(user, source, &ids).await;+ } else {+ self.spawn_import_batch(user.clone(), rows, Some(source));+ }+ Ok(true)+ }++ async fn finish_auto_import_batch(+ &self,+ user: &MacroUserIdStr<'static>,+ source: ImportSource,+ ids: &[Uuid],+ ) {+ match self.repo.finish_auto_import(user, source, ids).await {+ Ok(Some(status)) => {+ tracing::info!(+ source = source.as_ref(),+ status = status.as_ref(),+ "automatic import finished"+ );+ self.notify(user).await;+ }+ Ok(None) => {}+ Err(e) => {+ tracing::error!(source = source.as_ref(), error = ?e, "failed to persist automatic import outcome");+ }+ }+ }+/// Run one gather session: connector MCP tools plus the locked/// `CreateImportEntity`. Staged rows land as the session runs; the final/// text is ignored.@@ -599,6 +658,122 @@ where}}}++ /// Run a claimed import batch in the background. Manual batches only+ /// update their entity rows; automatic batches also settle their owning+ /// run after every row reaches a terminal state.+ fn spawn_import_batch(+ &self,+ user: MacroUserIdStr<'static>,+ rows: Vec<ImportEntity>,+ auto_run: Option<ImportSource>,+ ) {+ let service = self.clone();+ tokio::spawn(async move {+ let batch_ids: Vec<Uuid> = rows.iter().map(|row| row.id).collect();+ let (notion_rows, direct_rows): (Vec<ImportEntity>, Vec<ImportEntity>) = rows+ .into_iter()+ .partition(|row| row.source == ImportSource::Notion);++ // Heartbeat every row this batch owns (queued and in-flight) for+ // as long as the batch runs, so the read path's stale reaper only+ // fires on rows whose process actually died.+ let _heartbeat = AbortOnDrop(tokio::spawn({+ let service = service.clone();+ let user = user.clone();+ let ids = batch_ids.clone();+ async move {+ loop {+ tokio::time::sleep(IMPORT_HEARTBEAT).await;+ let _ = service+ .repo+ .touch_importing(&user, &ids)+ .await+ .inspect_err(|e| {+ tracing::warn!(error = ?e, "import heartbeat failed");+ });+ }+ }+ }));++ for row in &direct_rows {+ service.import_deterministic(&user, row).await;+ }++ if !notion_rows.is_empty() {+ // One connector connection per batch, pages a few at a time.+ // Each page first tries direct fetch → markdown → finalize;+ // the agent session is the fallback.+ let mcp_tools = match service.connector_tools(&user, ImportSource::Notion).await {+ Ok(tools) => tools,+ Err(e) => {+ tracing::warn!(error = ?e, "notion connector unavailable");+ let ids: Vec<Uuid> = notion_rows.iter().map(|row| row.id).collect();+ service+ .fail_unfinished(&user, &ids, &format!("notion unavailable: {e}"))+ .await;+ service.notify(&user).await;+ if let Some(source) = auto_run {+ service+ .finish_auto_import_batch(&user, source, &batch_ids)+ .await;+ }+ return;+ }+ };++ futures::stream::iter(notion_rows)+ .for_each_concurrent(NOTION_IMPORT_CONCURRENCY, |row| {+ let service = service.clone();+ let user = user.clone();+ let mcp_tools = mcp_tools.clone();+ async move {+ let outcome =+ tokio::time::timeout(NOTION_PAGE_IMPORT_TIMEOUT, async {+ match service+ .import_notion_page_direct(&user, &mcp_tools, &row)+ .await+ {+ Ok(()) => Ok(()),+ Err(direct_error) => {+ tracing::info!(id = %row.id, error = ?direct_error, "direct notion import failed; trying the agent");+ service+ .run_notion_import_session(+ &user,+ std::slice::from_ref(&row),+ mcp_tools.clone(),+ )+ .await+ }+ }+ })+ .await+ .unwrap_or_else(|_| {+ Err(anyhow::anyhow!("notion import timed out"))+ });+ let _ = outcome.inspect_err(|e| {+ tracing::warn!(id = %row.id, error = ?e, "notion page import failed");+ });+ service+ .fail_unfinished(+ &user,+ &[row.id],+ "the import job did not finish this item",+ )+ .await;+ }+ })+ .await;+ service.notify(&user).await;+ }++ if let Some(source) = auto_run {+ service+ .finish_auto_import_batch(&user, source, &batch_ids)+ .await;+ }+ });+ }}impl<R, S, C> ImportService for ImportServiceImpl<R, S, C>@@ -627,7 +802,37 @@ whereErr(e) => tracing::warn!(error = ?e, "failed to reap stale imports"),}- let runs = self.repo.list_runs(&user).await?;+ match self.repo.reconcile_auto_import_runs(&user).await {+ Ok(0) => {}+ Ok(reconciled) => {+ tracing::warn!(reconciled, "reconciled interrupted automatic import runs");+ self.notify(&user).await;+ }+ // A read must never fail because recovery did.+ Err(e) => tracing::warn!(error = ?e, "failed to reconcile automatic import runs"),+ }++ // Self-heal the small window between persisting gather completion and+ // spawning its configured follow-on batch. The begin CAS makes this+ // safe across concurrent readers and replicas.+ let mut runs = self.repo.list_runs(&user).await?;+ let auto_ready: Vec<ImportSource> = runs+ .iter()+ .filter(|run| run.auto_import && run.status == RunStatus::Ready)+ .map(|run| run.source)+ .collect();+ let mut started = false;+ for source in auto_ready {+ match self.maybe_start_auto_import(&user, source).await {+ Ok(did_start) => started |= did_start,+ Err(e) => {+ tracing::warn!(source = source.as_ref(), error = ?e, "failed to self-heal automatic import start");+ }+ }+ }+ if started {+ runs = self.repo.list_runs(&user).await?;+ }let entities = self.repo.list(&user, None, None).await?;Ok(ImportState { runs, entities })}@@ -637,10 +842,11 @@ where&self,user: MacroUserIdStr<'static>,source: ImportSource,+ auto_import: bool,) -> Result<bool> {// Wins only when no run row exists — gathers run once per// connection; explicit retry is the only re-entry.- let won = self.repo.start_run(&user, source, &[]).await?;+ let won = self.repo.start_run(&user, source, &[], auto_import).await?;if won {self.spawn_gather(user.clone(), source);self.notify(&user).await;@@ -656,7 +862,12 @@ where) -> Result<bool> {let won = self.repo- .start_run(&user, source, &[RunStatus::Failed, RunStatus::Dismissed])+ .start_run(+ &user,+ source,+ &[RunStatus::Failed, RunStatus::Dismissed],+ false,+ ).await?;if won {self.spawn_gather(user.clone(), source);@@ -667,7 +878,8 @@ where#[tracing::instrument(skip(self, user), err)]async fn dismiss_run(&self, user: MacroUserIdStr<'static>, source: ImportSource) -> Result<()> {- self.repo+ let dismissed = self+ .repo.transition_run(&user,source,@@ -675,6 +887,9 @@ whereRunStatus::Dismissed,).await?;+ if dismissed {+ self.notify(&user).await;+ }Ok(())}@@ -697,106 +912,7 @@ whereself.notify(&user).await;if !rows.is_empty() {- let (notion_rows, direct_rows): (Vec<ImportEntity>, Vec<ImportEntity>) = rows- .into_iter()- .partition(|row| row.source == ImportSource::Notion);-- let service = self.clone();- let user = user.clone();- tokio::spawn(async move {- // Heartbeat every row this batch owns (queued and in-flight)- // for as long as the batch runs, so the read path's stale- // reaper only ever fires on rows whose process actually- // died. Dropped (aborted) on every exit path of this task.- let _heartbeat = AbortOnDrop(tokio::spawn({- let service = service.clone();- let user = user.clone();- let ids: Vec<Uuid> = direct_rows- .iter()- .chain(notion_rows.iter())- .map(|row| row.id)- .collect();- async move {- loop {- tokio::time::sleep(IMPORT_HEARTBEAT).await;- if let Err(e) = service.repo.touch_importing(&user, &ids).await {- tracing::warn!(error = ?e, "import heartbeat failed");- }- }- }- }));-- for row in &direct_rows {- service.import_deterministic(&user, row).await;- }-- if !notion_rows.is_empty() {- // One connector connection per batch, pages a few at a- // time. Each page first tries the direct path (fetch- // tool → markdown → finalize, no model); the Haiku- // session is the fallback for pages that path can't- // handle.- let mcp_tools = match service.connector_tools(&user, ImportSource::Notion).await- {- Ok(tools) => tools,- Err(e) => {- tracing::warn!(error = ?e, "notion connector unavailable");- let ids: Vec<Uuid> = notion_rows.iter().map(|r| r.id).collect();- service- .fail_unfinished(&user, &ids, &format!("notion unavailable: {e}"))- .await;- service.notify(&user).await;- return;- }- };-- futures::stream::iter(notion_rows)- .for_each_concurrent(NOTION_IMPORT_CONCURRENCY, |row| {- let service = service.clone();- let user = user.clone();- let mcp_tools = mcp_tools.clone();- async move {- let outcome = tokio::time::timeout(- NOTION_PAGE_IMPORT_TIMEOUT,- async {- match service- .import_notion_page_direct(&user, &mcp_tools, &row)- .await- {- Ok(()) => Ok(()),- Err(direct_error) => {- tracing::info!(id = %row.id, error = ?direct_error, "direct notion import failed; trying the agent");- service- .run_notion_import_session(- &user,- std::slice::from_ref(&row),- mcp_tools.clone(),- )- .await- }- }- },- )- .await- .unwrap_or_else(|_| {- Err(anyhow::anyhow!("notion import timed out"))- });- if let Err(e) = outcome {- tracing::warn!(id = %row.id, error = ?e, "notion page import failed");- }- service- .fail_unfinished(- &user,- &[row.id],- "the import job did not finish this item",- )- .await;- }- })- .await;- service.notify(&user).await;- }- });+ self.spawn_import_batch(user.clone(), rows, None);}Ok(RunImportOutcome {diff --git a/crates/import/src/outbound/pg_import_repo.rs b/crates/import/src/outbound/pg_import_repo.rsindex 962fc39de8..dea265f4c1 100644--- a/crates/import/src/outbound/pg_import_repo.rs+++ b/crates/import/src/outbound/pg_import_repo.rs@@ -15,6 +15,9 @@ use std::collections::HashSet;use std::str::FromStr;use uuid::Uuid;+#[cfg(test)]+mod test;+/// Postgres-backed import repository (MacroDB).#[derive(Clone)]pub struct PgImportRepo {@@ -75,6 +78,7 @@ impl TryFrom<ImportEntityDbRow> for ImportEntity {struct ImportRunDbRow {source: String,status: String,+ auto_import: bool,error: Option<String>,updated_at: DateTime<Utc>,}@@ -90,6 +94,7 @@ impl TryFrom<ImportRunDbRow> for ImportRun {Ok(ImportRun {source,status,+ auto_import: row.auto_import,error: row.error,updated_at: row.updated_at,})@@ -472,6 +477,17 @@ impl ImportRepo for PgImportRepo {r#"DELETE FROM import_entityWHERE user_id = $1 AND initiator = $2 AND status = 'staged'+ AND NOT (+ $2 = 'onboarding'+ AND EXISTS (+ SELECT 1+ FROM import_run+ WHERE import_run.user_id = import_entity.user_id+ AND import_run.source = import_entity.source+ AND import_run.auto_import+ AND import_run.status IN ('running', 'ready', 'importing', 'failed')+ )+ )"#,user.as_ref(),initiator.as_ref(),@@ -497,7 +513,7 @@ impl ImportRepo for PgImportRepo {let rows = sqlx::query_as!(ImportRunDbRow,r#"- SELECT source, status, error, updated_at+ SELECT source, status::text AS "status!", auto_import, error, updated_atFROM import_run WHERE user_id = $1ORDER BY source"#,@@ -514,19 +530,21 @@ impl ImportRepo for PgImportRepo {user: &MacroUserIdStr<'static>,source: ImportSource,from: &[RunStatus],+ auto_import: bool,) -> Result<bool> {let from = run_status_strings(from);let result = sqlx::query!(r#"- INSERT INTO import_run (user_id, source, status)- VALUES ($1, $2, 'running')+ INSERT INTO import_run (user_id, source, status, auto_import)+ VALUES ($1, $2, 'running', $4)ON CONFLICT (user_id, source) DO UPDATESET status = 'running', error = NULL, updated_at = NOW()- WHERE import_run.status = ANY($3::text[])+ WHERE import_run.status::text = ANY($3::text[])"#,user.as_ref(),source.as_ref(),&from,+ auto_import,).execute(&self.pool).await?;@@ -544,7 +562,9 @@ impl ImportRepo for PgImportRepo {let result = sqlx::query!(r#"UPDATE import_run- SET status = $3, error = $4, updated_at = NOW()+ SET status = $3::text::import_run_status,+ error = $4,+ updated_at = NOW()WHERE user_id = $1 AND source = $2 AND status = 'running'"#,user.as_ref(),@@ -569,8 +589,9 @@ impl ImportRepo for PgImportRepo {let result = sqlx::query!(r#"UPDATE import_run- SET status = $4, updated_at = NOW()- WHERE user_id = $1 AND source = $2 AND status = ANY($3::text[])+ SET status = $4::text::import_run_status, updated_at = NOW()+ WHERE user_id = $1 AND source = $2+ AND status::text = ANY($3::text[])"#,user.as_ref(),source.as_ref(),@@ -581,4 +602,145 @@ impl ImportRepo for PgImportRepo {.await?;Ok(result.rows_affected() > 0)}++ #[tracing::instrument(skip(self), err)]+ async fn begin_auto_import(+ &self,+ user: &MacroUserIdStr<'static>,+ source: ImportSource,+ ) -> Result<Option<Vec<ImportEntity>>> {+ let mut transaction = self.pool.begin().await?;+ let claimed = sqlx::query_scalar!(+ r#"+ UPDATE import_run+ SET status = 'importing', error = NULL, updated_at = NOW()+ WHERE user_id = $1 AND source = $2+ AND status = 'ready' AND auto_import+ RETURNING source+ "#,+ user.as_ref(),+ source.as_ref(),+ )+ .fetch_optional(&mut *transaction)+ .await?;+ if claimed.is_none() {+ return Ok(None);+ }++ let rows = sqlx::query_as!(+ ImportEntityDbRow,+ r#"+ UPDATE import_entity+ SET status = 'importing', last_error = NULL, updated_at = NOW()+ WHERE user_id = $1 AND source = $2+ AND initiator = 'onboarding' AND status = 'staged'+ RETURNING id, user_id, team_id, source, foreign_id, status, initiator,+ metadata, entity_id, entity_type, last_error, created_at, updated_at+ "#,+ user.as_ref(),+ source.as_ref(),+ )+ .fetch_all(&mut *transaction)+ .await?;+ transaction.commit().await?;+ Ok(Some(+ rows.into_iter()+ .map(ImportEntity::try_from)+ .collect::<Result<Vec<_>>>()?,+ ))+ }++ #[tracing::instrument(skip(self, ids), fields(count = ids.len()), err)]+ async fn finish_auto_import(+ &self,+ user: &MacroUserIdStr<'static>,+ source: ImportSource,+ ids: &[Uuid],+ ) -> Result<Option<RunStatus>> {+ let status = sqlx::query_scalar!(+ r#"+ WITH outcome AS (+ SELECT+ COUNT(*) FILTER (WHERE status = 'imported')+ = CARDINALITY($3::uuid[])::bigint AS succeeded+ FROM import_entity+ WHERE user_id = $1 AND id = ANY($3::uuid[])+ )+ UPDATE import_run+ SET status = (CASE+ WHEN outcome.succeeded THEN 'completed'+ ELSE 'failed'+ END)::import_run_status,+ error = CASE+ WHEN outcome.succeeded THEN NULL+ ELSE 'one or more automatic imports failed'+ END,+ updated_at = NOW()+ FROM outcome+ WHERE user_id = $1 AND source = $2 AND status = 'importing'+ RETURNING import_run.status::text AS "status!"+ "#,+ user.as_ref(),+ source.as_ref(),+ ids,+ )+ .fetch_optional(&self.pool)+ .await?;+ status+ .map(|status| {+ RunStatus::from_str(&status)+ .map_err(|_| anyhow::anyhow!("unknown run status: {status}").into())+ })+ .transpose()+ }++ #[tracing::instrument(skip(self), err)]+ async fn reconcile_auto_import_runs(&self, user: &MacroUserIdStr<'static>) -> Result<u64> {+ let result = sqlx::query!(+ r#"+ WITH settled AS (+ SELECT+ run.source,+ EXISTS (+ SELECT 1+ FROM import_entity entity+ WHERE entity.user_id = run.user_id+ AND entity.source = run.source+ AND entity.initiator = 'onboarding'+ AND entity.status = 'staged'+ AND entity.last_error IS NOT NULL+ ) AS failed+ FROM import_run run+ WHERE run.user_id = $1+ AND run.status = 'importing'+ AND run.auto_import+ AND NOT EXISTS (+ SELECT 1+ FROM import_entity entity+ WHERE entity.user_id = run.user_id+ AND entity.source = run.source+ AND entity.initiator = 'onboarding'+ AND entity.status = 'importing'+ )+ )+ UPDATE import_run run+ SET status = (+ CASE WHEN settled.failed THEN 'failed' ELSE 'completed' END+ )::import_run_status,+ error = CASE+ WHEN settled.failed THEN 'one or more automatic imports failed'+ ELSE NULL+ END,+ updated_at = NOW()+ FROM settled+ WHERE run.user_id = $1+ AND run.source = settled.source+ AND run.status = 'importing'+ "#,+ user.as_ref(),+ )+ .execute(&self.pool)+ .await?;+ Ok(result.rows_affected())+ }}diff --git a/crates/import/src/outbound/pg_import_repo/test.rs b/crates/import/src/outbound/pg_import_repo/test.rsnew file mode 100644index 0000000000..8167182ecd--- /dev/null+++ b/crates/import/src/outbound/pg_import_repo/test.rs@@ -0,0 +1,207 @@+use super::*;+use macro_db_migrator::MACRO_DB_MIGRATIONS;+use sqlx::{Pool, Postgres};++fn user() -> MacroUserIdStr<'static> {+ MacroUserIdStr::parse_from_str("macro|[email protected]").expect("valid user id")+}++async fn ready_run(repo: &PgImportRepo, auto_import: bool) {+ assert!(+ repo.start_run(&user(), ImportSource::Linear, &[], auto_import)+ .await+ .expect("start run")+ );+ assert!(+ repo.finish_run(&user(), ImportSource::Linear, RunStatus::Ready, None)+ .await+ .expect("finish gather")+ );+}++async fn stage(+ repo: &PgImportRepo,+ source: ImportSource,+ initiator: Initiator,+ foreign_id: &str,+) -> ImportEntity {+ let metadata = match source {+ ImportSource::Linear => serde_json::json!({ "title": foreign_id }),+ ImportSource::Notion => serde_json::json!({ "title": foreign_id }),+ ImportSource::Slack => serde_json::json!({ "name": foreign_id }),+ };+ repo.upsert_staged(&user(), source, initiator, foreign_id, &metadata)+ .await+ .expect("stage query")+ .expect("staged row")+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn auto_import_claims_only_its_onboarding_source(pool: Pool<Postgres>) {+ let repo = PgImportRepo::new(pool);+ ready_run(&repo, true).await;++ let claimed = stage(&repo, ImportSource::Linear, Initiator::Onboarding, "LIN-1").await;+ let chat = stage(&repo, ImportSource::Linear, Initiator::Chat, "LIN-CHAT").await;+ let notion = stage(+ &repo,+ ImportSource::Notion,+ Initiator::Onboarding,+ "NOTION-1",+ )+ .await;++ assert_eq!(+ repo.delete_staged_by_initiator(&user(), Initiator::Onboarding)+ .await+ .expect("onboarding cleanup"),+ 1,+ "only the unrelated Notion row should be removed"+ );+ assert!(+ repo.get(&user(), notion.id)+ .await+ .expect("notion row lookup")+ .is_none()+ );++ let rows = repo+ .begin_auto_import(&user(), ImportSource::Linear)+ .await+ .expect("begin auto import")+ .expect("claimed run");+ assert_eq!(rows.len(), 1);+ assert_eq!(rows[0].id, claimed.id);+ assert_eq!(rows[0].status, ImportStatus::Importing);++ assert_eq!(+ repo.get(&user(), chat.id)+ .await+ .expect("chat row")+ .expect("chat row exists")+ .status,+ ImportStatus::Staged+ );+ assert!(+ repo.begin_auto_import(&user(), ImportSource::Linear)+ .await+ .expect("second begin")+ .is_none(),+ "the run claim is a CAS"+ );+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn auto_import_run_reports_completed_or_failed(pool: Pool<Postgres>) {+ let repo = PgImportRepo::new(pool);+ ready_run(&repo, true).await;+ let row = stage(&repo, ImportSource::Linear, Initiator::Onboarding, "LIN-1").await;+ let claimed = repo+ .begin_auto_import(&user(), ImportSource::Linear)+ .await+ .expect("begin")+ .expect("claimed");+ assert_eq!(claimed.len(), 1);+ repo.mark_imported(&user(), row.id, "task-1", "task", None)+ .await+ .expect("mark imported")+ .expect("updated row");++ assert_eq!(+ repo.finish_auto_import(&user(), ImportSource::Linear, &[row.id])+ .await+ .expect("finish")+ .expect("won finish"),+ RunStatus::Completed+ );+ let run = repo+ .list_runs(&user())+ .await+ .expect("runs")+ .pop()+ .expect("run");+ assert_eq!(run.status, RunStatus::Completed);+ assert!(run.auto_import);++ // A retry preserves the run's configuration and can report an item+ // failure on its next automatic batch.+ assert!(+ repo.start_run(+ &user(),+ ImportSource::Linear,+ &[RunStatus::Completed],+ false,+ )+ .await+ .expect("restart for test")+ );+ assert!(+ repo.finish_run(&user(), ImportSource::Linear, RunStatus::Ready, None)+ .await+ .expect("ready again")+ );+ let failed_row = stage(&repo, ImportSource::Linear, Initiator::Onboarding, "LIN-2").await;+ repo.begin_auto_import(&user(), ImportSource::Linear)+ .await+ .expect("begin failed batch")+ .expect("claimed failed batch");+ assert!(+ repo.mark_import_failed(&user(), failed_row.id, "creator failed")+ .await+ .expect("mark failed")+ );+ assert_eq!(+ repo.finish_auto_import(&user(), ImportSource::Linear, &[failed_row.id])+ .await+ .expect("finish failed")+ .expect("won failed finish"),+ RunStatus::Failed+ );+ assert_eq!(+ repo.delete_staged_by_initiator(&user(), Initiator::Onboarding)+ .await+ .expect("cleanup after failure"),+ 0,+ "a retryable automatic-import failure must survive onboarding cleanup"+ );+ assert!(+ repo.get(&user(), failed_row.id)+ .await+ .expect("failed row lookup")+ .is_some()+ );+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn interrupted_auto_import_run_is_reconciled(pool: Pool<Postgres>) {+ let repo = PgImportRepo::new(pool);+ ready_run(&repo, true).await;+ let row = stage(&repo, ImportSource::Linear, Initiator::Onboarding, "LIN-1").await;+ repo.begin_auto_import(&user(), ImportSource::Linear)+ .await+ .expect("begin")+ .expect("claimed");+ assert!(+ repo.mark_import_failed(&user(), row.id, "process interrupted")+ .await+ .expect("mark interrupted")+ );++ assert_eq!(+ repo.reconcile_auto_import_runs(&user())+ .await+ .expect("reconcile"),+ 1+ );+ let run = repo+ .list_runs(&user())+ .await+ .expect("runs")+ .pop()+ .expect("run");+ assert_eq!(run.status, RunStatus::Failed);+ assert_eq!(+ run.error.as_deref(),+ Some("one or more automatic imports failed")+ );+}diff --git a/crates/macro_db_client/migrations/20260723150434_support_auto_import_runs.sql b/crates/macro_db_client/migrations/20260723150434_support_auto_import_runs.sqlnew file mode 100644index 0000000000..dd0121ff41--- /dev/null+++ b/crates/macro_db_client/migrations/20260723150434_support_auto_import_runs.sql@@ -0,0 +1,22 @@+-- A gather run can optionally continue straight through importing every+-- onboarding-staged candidate it discovered. The run status then exposes the+-- import batch's progress and terminal outcome to clients.++CREATE TYPE import_run_status AS ENUM (+ 'running',+ 'ready',+ 'importing',+ 'completed',+ 'failed',+ 'dismissed'+);++ALTER TABLE import_run+ ADD COLUMN auto_import BOOLEAN NOT NULL DEFAULT FALSE;++ALTER TABLE import_run+ DROP CONSTRAINT import_run_status_check;++ALTER TABLE import_run+ ALTER COLUMN status TYPE import_run_status+ USING status::import_run_status;diff --git a/crates/onboarding/src/domain/service.rs b/crates/onboarding/src/domain/service.rsindex 5ec8edefe1..8292d88b63 100644--- a/crates/onboarding/src/domain/service.rs+++ b/crates/onboarding/src/domain/service.rs@@ -3,9 +3,9 @@//!//! Reconciliation is level-triggered and idempotent: every tick lists the//! user's MCP connections and asks the import service to start a gather for-//! each authenticated connector. `start_gather` is a CAS that only wins when-//! no run has ever started, so ticking on every read (and on the MCP-//! post-auth hook) is safe.+//! each authenticated connector with automatic import enabled.+//! `start_gather` is a CAS that only wins when no run has ever started, so+//! ticking on every read (and on the MCP post-auth hook) is safe.use super::models::{ConnectedServer, OnboardingRow, OnboardingState, OnboardingStatus};use super::ports::{OnboardingError, OnboardingRepo, Result};@@ -34,9 +34,11 @@ pub trait OnboardingService: Send + Sync + 'static {/// outside onboarding must not start gathers.fn reconcile(&self, user: MacroUserIdStr<'static>) -> impl Future<Output = Result<()>> + Send;- /// Finish the flow (`skipped` records how). Leftover onboarding-staged- /// import candidates are deleted — never reviewed is not declined, so- /// they stay re-stageable by later gathers or chat.+ /// Finish the flow (`skipped` records how). Unreserved leftover+ /// onboarding-staged import candidates are deleted — never reviewed is+ /// not declined, so they stay re-stageable by later gathers or chat.+ /// Candidates reserved by configured automatic runs remain until their+ /// run settles.fn complete(&self,user: MacroUserIdStr<'static>,@@ -88,8 +90,9 @@ where.collect())}- /// Start a gather run for every authenticated connector that never had- /// one. A failing source must not block the others (or the read).+ /// Start an auto-importing gather run for every authenticated connector+ /// that never had one. A failing source must not block the others (or+ /// the read).async fn start_due_gathers(&self, user: &MacroUserIdStr<'static>, servers: &[ConnectedServer]) {for source in ImportSource::all() {let connected = servers@@ -98,7 +101,7 @@ whereif !connected {continue;}- if let Err(e) = self.import.start_gather(user.clone(), source).await {+ if let Err(e) = self.import.start_gather(user.clone(), source, true).await {tracing::warn!(source = source.as_ref(), error = ?e, "failed to start gather run");}}@@ -158,11 +161,13 @@ whereskipped: bool,) -> Result<OnboardingRow> {let row = self.repo.complete(&user, skipped).await?;- // Candidates the user never acted on are removed from the ledger,- // not discarded: discard means "the user said no" and blocks+ // Unreserved candidates the user never acted on are removed from the+ // ledger, not discarded: discard means "the user said no" and blocks// re-staging forever, which is wrong for items they never looked at- // ("Set up later" skips the whole review). Explicit declines (a- // section toggled off) were already discarded by run_import.+ // ("Set up later" skips the whole review). Candidates reserved by an+ // active or retryable configured auto-import run survive this cleanup.+ // Explicit declines (a section toggled off) were already discarded by+ // run_import.// Chat-staged rows are untouched. Best-effort: the completion above// is durable, so a cleanup failure must not fail the call — the// next reconcile tick sweeps leftovers anyway.diff --git a/crates/onboarding/src/domain/service/test.rs b/crates/onboarding/src/domain/service/test.rsindex d6ce282a1e..5302bb6f9a 100644--- a/crates/onboarding/src/domain/service/test.rs+++ b/crates/onboarding/src/domain/service/test.rs@@ -110,7 +110,7 @@ impl McpServerStore for MockStore {#[derive(Default)]struct MockImport {- gathers: Mutex<Vec<ImportSource>>,+ gathers: Mutex<Vec<(ImportSource, bool)>>,discards: Mutex<Vec<Initiator>>,deletions: Mutex<Vec<Initiator>>,}@@ -127,8 +127,9 @@ impl ImportService for MockImport {&self,_user: MacroUserIdStr<'static>,source: ImportSource,+ auto_import: bool,) -> ImportResult<bool> {- self.gathers.lock().unwrap().push(source);+ self.gathers.lock().unwrap().push((source, auto_import));Ok(true)}@@ -223,7 +224,7 @@ async fn active_flow_starts_gathers_for_authenticated_connectors_only() {assert_eq!(state.connected_servers.len(), 3);assert_eq!(import.gathers.lock().unwrap().as_slice(),- &[ImportSource::Linear]+ &[(ImportSource::Linear, true)]);}diff --git a/crates/onboarding/src/inbound/axum_router.rs b/crates/onboarding/src/inbound/axum_router.rsindex 3e09f1a9fd..cb8e0d5097 100644--- a/crates/onboarding/src/inbound/axum_router.rs+++ b/crates/onboarding/src/inbound/axum_router.rs@@ -98,8 +98,8 @@ pub async fn get_state_handler<T: OnboardingService, Auth: MacroAuthorizationSer}}-/// Complete (or skip) onboarding. Leftover onboarding-staged import-/// candidates are discarded.+/// Complete (or skip) onboarding. Unreserved leftover onboarding-staged+/// import candidates are deleted.#[utoipa::path(post,path = "/onboarding/complete",diff --git a/crates/onboarding/src/lib.rs b/crates/onboarding/src/lib.rsindex 72daee34d3..c5cc689b47 100644--- a/crates/onboarding/src/lib.rs+++ b/crates/onboarding/src/lib.rs@@ -4,8 +4,9 @@//!//! All the actual import machinery — staging, dedup, gather/import jobs —//! lives in the `import` crate. Onboarding is just a feature that uses it:-//! it starts gathers for authenticated connectors while the flow is active,-//! and discards leftover onboarding-staged rows when the flow completes.+//! it starts auto-importing gathers for authenticated connectors while the+//! flow is active, and deletes unreserved onboarding-staged rows when the+//! flow completes.#![deny(missing_docs)]diff --git a/services/document_cognition_service/src/main.rs b/services/document_cognition_service/src/main.rsindex 4fceed396c..7908ede2c5 100644--- a/services/document_cognition_service/src/main.rs+++ b/services/document_cognition_service/src/main.rs@@ -558,8 +558,8 @@ async fn main() -> anyhow::Result<()> {tracing::info!("initialized ai projections service");// The onboarding flow drives the import pipeline: reads/hooks start- // gather runs for authenticated connectors, completion discards leftover- // onboarding-staged candidates.+ // auto-importing gather runs for authenticated connectors, and+ // completion deletes unreserved onboarding-staged candidates.let onboarding_service = Arc::new(onboarding::domain::service::OnboardingServiceImpl::new(onboarding::outbound::pg_onboarding_repo::PgOnboardingRepo::new(db.clone()),Arc::new(mcp_server_repo.clone()),