Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

feat(import/onboarding): support auto-import in onboarding (#5122)

teo10bd79c2a1ec
diff --git a/.sqlx/query-0a41e84a695b39ddf490db802922c943f6499b14467fd7a0d0b83b04af8abf9c.json b/.sqlx/query-0a41e84a695b39ddf490db802922c943f6499b14467fd7a0d0b83b04af8abf9c.json
new file mode 100644
index 0000000000..6d1c68b86e
--- /dev/null
+++ b/.sqlx/query-0a41e84a695b39ddf490db802922c943f6499b14467fd7a0d0b83b04af8abf9c.json
@@ -0,0 +1,17 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n INSERT INTO import_run (user_id, source, status, auto_import)\n VALUES ($1, $2, 'running', $4)\n ON CONFLICT (user_id, source) DO UPDATE\n SET status = 'running', error = NULL, updated_at = NOW()\n WHERE import_run.status::text = ANY($3::text[])\n ",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Text",
+ "Text",
+ "TextArray",
+ "Bool"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "0a41e84a695b39ddf490db802922c943f6499b14467fd7a0d0b83b04af8abf9c"
+}
diff --git a/.sqlx/query-18b28d89f92ce787c86984be5c68b5577218c7000631fdec7b70dfcf5de516a6.json b/.sqlx/query-18b28d89f92ce787c86984be5c68b5577218c7000631fdec7b70dfcf5de516a6.json
new file mode 100644
index 0000000000..1af4dcfd7e
--- /dev/null
+++ b/.sqlx/query-18b28d89f92ce787c86984be5c68b5577218c7000631fdec7b70dfcf5de516a6.json
@@ -0,0 +1,15 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n DELETE FROM import_entity\n WHERE user_id = $1 AND initiator = $2 AND status = 'staged'\n AND NOT (\n $2 = 'onboarding'\n AND EXISTS (\n SELECT 1\n FROM import_run\n WHERE import_run.user_id = import_entity.user_id\n AND import_run.source = import_entity.source\n AND import_run.auto_import\n AND import_run.status IN ('running', 'ready', 'importing', 'failed')\n )\n )\n ",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Text",
+ "Text"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "18b28d89f92ce787c86984be5c68b5577218c7000631fdec7b70dfcf5de516a6"
+}
diff --git a/.sqlx/query-d919f2560dd92b03ba4cde5d6f024a94ac6e340f12eee4ac14ccec55b7008485.json b/.sqlx/query-192b53847ed98ebf7e5506cf99eff776fed05fb7926d8fba33b31ff8b49fa1c3.json
similarity index 58%
rename from .sqlx/query-d919f2560dd92b03ba4cde5d6f024a94ac6e340f12eee4ac14ccec55b7008485.json
rename to .sqlx/query-192b53847ed98ebf7e5506cf99eff776fed05fb7926d8fba33b31ff8b49fa1c3.json
index e5801db794..9772ba66d2 100644
--- a/.sqlx/query-d919f2560dd92b03ba4cde5d6f024a94ac6e340f12eee4ac14ccec55b7008485.json
+++ b/.sqlx/query-192b53847ed98ebf7e5506cf99eff776fed05fb7926d8fba33b31ff8b49fa1c3.json
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
- "query": "\n SELECT source, status, error, updated_at\n FROM import_run WHERE user_id = $1\n ORDER BY source\n ",
+ "query": "\n SELECT source, status::text AS \"status!\", auto_import, error, updated_at\n FROM import_run WHERE user_id = $1\n ORDER BY source\n ",
"describe": {
"columns": [
{
@@ -10,16 +10,21 @@
},
{
"ordinal": 1,
- "name": "status",
+ "name": "status!",
"type_info": "Text"
},
{
"ordinal": 2,
+ "name": "auto_import",
+ "type_info": "Bool"
+ },
+ {
+ "ordinal": 3,
"name": "error",
"type_info": "Text"
},
{
- "ordinal": 3,
+ "ordinal": 4,
"name": "updated_at",
"type_info": "Timestamptz"
}
@@ -31,10 +36,11 @@
},
"nullable": [
false,
+ null,
false,
true,
false
]
},
- "hash": "d919f2560dd92b03ba4cde5d6f024a94ac6e340f12eee4ac14ccec55b7008485"
+ "hash": "192b53847ed98ebf7e5506cf99eff776fed05fb7926d8fba33b31ff8b49fa1c3"
}
diff --git a/.sqlx/query-6f216a9a7f92b281059f6a8d6445c8e839dae4e1e8eaec4efec7b131a583b5ac.json b/.sqlx/query-1f632dca1f9936c1fe418219f03744955d587014e1015ff4616e05aa8491e70d.json
similarity index 52%
rename from .sqlx/query-6f216a9a7f92b281059f6a8d6445c8e839dae4e1e8eaec4efec7b131a583b5ac.json
rename to .sqlx/query-1f632dca1f9936c1fe418219f03744955d587014e1015ff4616e05aa8491e70d.json
index 0a518a34a7..a86c575272 100644
--- a/.sqlx/query-6f216a9a7f92b281059f6a8d6445c8e839dae4e1e8eaec4efec7b131a583b5ac.json
+++ b/.sqlx/query-1f632dca1f9936c1fe418219f03744955d587014e1015ff4616e05aa8491e70d.json
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
- "query": "\n UPDATE import_run\n SET status = $4, updated_at = NOW()\n WHERE user_id = $1 AND source = $2 AND status = ANY($3::text[])\n ",
+ "query": "\n UPDATE import_run\n SET status = $4::text::import_run_status, updated_at = NOW()\n WHERE user_id = $1 AND source = $2\n AND status::text = ANY($3::text[])\n ",
"describe": {
"columns": [],
"parameters": {
@@ -13,5 +13,5 @@
},
"nullable": []
},
- "hash": "6f216a9a7f92b281059f6a8d6445c8e839dae4e1e8eaec4efec7b131a583b5ac"
+ "hash": "1f632dca1f9936c1fe418219f03744955d587014e1015ff4616e05aa8491e70d"
}
diff --git a/.sqlx/query-50dbd26f9ca1a622f73b5bb7a5729e107dd4134fe6e77fea27bffae137f17359.json b/.sqlx/query-50dbd26f9ca1a622f73b5bb7a5729e107dd4134fe6e77fea27bffae137f17359.json
deleted file mode 100644
index f7e09aff03..0000000000
--- a/.sqlx/query-50dbd26f9ca1a622f73b5bb7a5729e107dd4134fe6e77fea27bffae137f17359.json
+++ /dev/null
@@ -1,16 +0,0 @@
-{
- "db_name": "PostgreSQL",
- "query": "\n INSERT INTO import_run (user_id, source, status)\n VALUES ($1, $2, 'running')\n ON CONFLICT (user_id, source) DO UPDATE\n SET status = 'running', error = NULL, updated_at = NOW()\n WHERE import_run.status = ANY($3::text[])\n ",
- "describe": {
- "columns": [],
- "parameters": {
- "Left": [
- "Text",
- "Text",
- "TextArray"
- ]
- },
- "nullable": []
- },
- "hash": "50dbd26f9ca1a622f73b5bb7a5729e107dd4134fe6e77fea27bffae137f17359"
-}
diff --git a/.sqlx/query-6335355244b0d3b14781c6983127a65b3ef3e669b837ffa239d5b8d74dc7ef8c.json b/.sqlx/query-6335355244b0d3b14781c6983127a65b3ef3e669b837ffa239d5b8d74dc7ef8c.json
new file mode 100644
index 0000000000..31282a5aa7
--- /dev/null
+++ b/.sqlx/query-6335355244b0d3b14781c6983127a65b3ef3e669b837ffa239d5b8d74dc7ef8c.json
@@ -0,0 +1,24 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n WITH outcome AS (\n SELECT\n COUNT(*) FILTER (WHERE status = 'imported')\n = CARDINALITY($3::uuid[])::bigint AS succeeded\n FROM import_entity\n WHERE user_id = $1 AND id = ANY($3::uuid[])\n )\n UPDATE import_run\n SET status = (CASE\n WHEN outcome.succeeded THEN 'completed'\n ELSE 'failed'\n END)::import_run_status,\n error = CASE\n WHEN outcome.succeeded THEN NULL\n ELSE 'one or more automatic imports failed'\n END,\n updated_at = NOW()\n FROM outcome\n WHERE user_id = $1 AND source = $2 AND status = 'importing'\n RETURNING import_run.status::text AS \"status!\"\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "status!",
+ "type_info": "Text"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Text",
+ "Text",
+ "UuidArray"
+ ]
+ },
+ "nullable": [
+ null
+ ]
+ },
+ "hash": "6335355244b0d3b14781c6983127a65b3ef3e669b837ffa239d5b8d74dc7ef8c"
+}
diff --git a/.sqlx/query-80b17499f3576f22aaddce4a8f56edd5faa866172d94dbb7cb00cbcaa64f1f4a.json b/.sqlx/query-7e99cc59b745603c368e0ad6e977848c67c188791845c18ef54b39a2fbc74df6.json
similarity index 50%
rename from .sqlx/query-80b17499f3576f22aaddce4a8f56edd5faa866172d94dbb7cb00cbcaa64f1f4a.json
rename to .sqlx/query-7e99cc59b745603c368e0ad6e977848c67c188791845c18ef54b39a2fbc74df6.json
index 719a17222f..d52d2ac1d1 100644
--- a/.sqlx/query-80b17499f3576f22aaddce4a8f56edd5faa866172d94dbb7cb00cbcaa64f1f4a.json
+++ b/.sqlx/query-7e99cc59b745603c368e0ad6e977848c67c188791845c18ef54b39a2fbc74df6.json
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
- "query": "\n UPDATE import_run\n SET status = $3, error = $4, updated_at = NOW()\n WHERE user_id = $1 AND source = $2 AND status = 'running'\n ",
+ "query": "\n UPDATE import_run\n SET status = $3::text::import_run_status,\n error = $4,\n updated_at = NOW()\n WHERE user_id = $1 AND source = $2 AND status = 'running'\n ",
"describe": {
"columns": [],
"parameters": {
@@ -13,5 +13,5 @@
},
"nullable": []
},
- "hash": "80b17499f3576f22aaddce4a8f56edd5faa866172d94dbb7cb00cbcaa64f1f4a"
+ "hash": "7e99cc59b745603c368e0ad6e977848c67c188791845c18ef54b39a2fbc74df6"
}
diff --git a/.sqlx/query-a53c213e1d4304506ece3fb431c15a14575a6c9eee0ff348f4b01eee0eacd2be.json b/.sqlx/query-a53c213e1d4304506ece3fb431c15a14575a6c9eee0ff348f4b01eee0eacd2be.json
new file mode 100644
index 0000000000..56be6b6bd9
--- /dev/null
+++ b/.sqlx/query-a53c213e1d4304506ece3fb431c15a14575a6c9eee0ff348f4b01eee0eacd2be.json
@@ -0,0 +1,23 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n UPDATE import_run\n SET status = 'importing', error = NULL, updated_at = NOW()\n WHERE user_id = $1 AND source = $2\n AND status = 'ready' AND auto_import\n RETURNING source\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "source",
+ "type_info": "Text"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Text",
+ "Text"
+ ]
+ },
+ "nullable": [
+ false
+ ]
+ },
+ "hash": "a53c213e1d4304506ece3fb431c15a14575a6c9eee0ff348f4b01eee0eacd2be"
+}
diff --git a/.sqlx/query-c00ffb94c8b6b3c21a28360c1e7d5026d39f79645826bcb2e8136d530ebc358b.json b/.sqlx/query-c00ffb94c8b6b3c21a28360c1e7d5026d39f79645826bcb2e8136d530ebc358b.json
new file mode 100644
index 0000000000..430fd63cfa
--- /dev/null
+++ b/.sqlx/query-c00ffb94c8b6b3c21a28360c1e7d5026d39f79645826bcb2e8136d530ebc358b.json
@@ -0,0 +1,14 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n WITH settled AS (\n SELECT\n run.source,\n EXISTS (\n SELECT 1\n FROM import_entity entity\n WHERE entity.user_id = run.user_id\n AND entity.source = run.source\n AND entity.initiator = 'onboarding'\n AND entity.status = 'staged'\n AND entity.last_error IS NOT NULL\n ) AS failed\n FROM import_run run\n WHERE run.user_id = $1\n AND run.status = 'importing'\n AND run.auto_import\n AND NOT EXISTS (\n SELECT 1\n FROM import_entity entity\n WHERE entity.user_id = run.user_id\n AND entity.source = run.source\n AND entity.initiator = 'onboarding'\n AND entity.status = 'importing'\n )\n )\n UPDATE import_run run\n SET status = (\n CASE WHEN settled.failed THEN 'failed' ELSE 'completed' END\n )::import_run_status,\n error = CASE\n WHEN settled.failed THEN 'one or more automatic imports failed'\n ELSE NULL\n END,\n updated_at = NOW()\n FROM settled\n WHERE run.user_id = $1\n AND run.source = settled.source\n AND run.status = 'importing'\n ",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Text"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "c00ffb94c8b6b3c21a28360c1e7d5026d39f79645826bcb2e8136d530ebc358b"
+}
diff --git a/.sqlx/query-d5a9632d0330f2dfc6dc0c6f6817f2ffce500a7f816bb5af32d46d74baeb793e.json b/.sqlx/query-d5a9632d0330f2dfc6dc0c6f6817f2ffce500a7f816bb5af32d46d74baeb793e.json
deleted file mode 100644
index b7f93d687d..0000000000
--- a/.sqlx/query-d5a9632d0330f2dfc6dc0c6f6817f2ffce500a7f816bb5af32d46d74baeb793e.json
+++ /dev/null
@@ -1,15 +0,0 @@
-{
- "db_name": "PostgreSQL",
- "query": "\n DELETE FROM import_entity\n WHERE user_id = $1 AND initiator = $2 AND status = 'staged'\n ",
- "describe": {
- "columns": [],
- "parameters": {
- "Left": [
- "Text",
- "Text"
- ]
- },
- "nullable": []
- },
- "hash": "d5a9632d0330f2dfc6dc0c6f6817f2ffce500a7f816bb5af32d46d74baeb793e"
-}
diff --git a/.sqlx/query-ff8e627d8fcb9bbcc5751ede083004cfd6f9f4ccb817a23ec8cc868946fcb276.json b/.sqlx/query-ff8e627d8fcb9bbcc5751ede083004cfd6f9f4ccb817a23ec8cc868946fcb276.json
new file mode 100644
index 0000000000..3181be44f2
--- /dev/null
+++ b/.sqlx/query-ff8e627d8fcb9bbcc5751ede083004cfd6f9f4ccb817a23ec8cc868946fcb276.json
@@ -0,0 +1,95 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n UPDATE import_entity\n SET status = 'importing', last_error = NULL, updated_at = NOW()\n WHERE user_id = $1 AND source = $2\n AND initiator = 'onboarding' AND status = 'staged'\n RETURNING id, user_id, team_id, source, foreign_id, status, initiator,\n metadata, entity_id, entity_type, last_error, created_at, updated_at\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "id",
+ "type_info": "Uuid"
+ },
+ {
+ "ordinal": 1,
+ "name": "user_id",
+ "type_info": "Text"
+ },
+ {
+ "ordinal": 2,
+ "name": "team_id",
+ "type_info": "Uuid"
+ },
+ {
+ "ordinal": 3,
+ "name": "source",
+ "type_info": "Text"
+ },
+ {
+ "ordinal": 4,
+ "name": "foreign_id",
+ "type_info": "Text"
+ },
+ {
+ "ordinal": 5,
+ "name": "status",
+ "type_info": "Text"
+ },
+ {
+ "ordinal": 6,
+ "name": "initiator",
+ "type_info": "Text"
+ },
+ {
+ "ordinal": 7,
+ "name": "metadata",
+ "type_info": "Jsonb"
+ },
+ {
+ "ordinal": 8,
+ "name": "entity_id",
+ "type_info": "Text"
+ },
+ {
+ "ordinal": 9,
+ "name": "entity_type",
+ "type_info": "Text"
+ },
+ {
+ "ordinal": 10,
+ "name": "last_error",
+ "type_info": "Text"
+ },
+ {
+ "ordinal": 11,
+ "name": "created_at",
+ "type_info": "Timestamptz"
+ },
+ {
+ "ordinal": 12,
+ "name": "updated_at",
+ "type_info": "Timestamptz"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Text",
+ "Text"
+ ]
+ },
+ "nullable": [
+ false,
+ false,
+ true,
+ false,
+ false,
+ false,
+ false,
+ false,
+ true,
+ true,
+ true,
+ false,
+ false
+ ]
+ },
+ "hash": "ff8e627d8fcb9bbcc5751ede083004cfd6f9f4ccb817a23ec8cc868946fcb276"
+}
diff --git a/Cargo.lock b/Cargo.lock
index 3cbc9982ac..f4e2631542 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -7133,6 +7133,7 @@ dependencies = [
"connection_gateway_client",
"futures",
"macro_authorization",
+ "macro_db_migrator",
"macro_user_id",
"mcp_client",
"model-entity",
diff --git a/apps/web/src/lib/queries/import.ts b/apps/web/src/lib/queries/import.ts
index cfab675344..f569245897 100644
--- a/apps/web/src/lib/queries/import.ts
+++ b/apps/web/src/lib/queries/import.ts
@@ -51,8 +51,9 @@ const PENDING_IMPORT_STATE: ImportState = { runs: [], entities: [] };
function anythingInFlight(state: ImportState | undefined): boolean {
if (!state) return false;
return (
- state.runs.some((run) => run.status === 'running') ||
- state.entities.some((entity) => entity.status === 'importing')
+ state.runs.some(
+ (run) => run.status === 'running' || run.status === 'importing'
+ ) || state.entities.some((entity) => entity.status === 'importing')
);
}
diff --git a/apps/web/src/lib/service-clients/service-cognition/generated/client.ts b/apps/web/src/lib/service-clients/service-cognition/generated/client.ts
index f270127084..1c12625d26 100644
--- a/apps/web/src/lib/service-clients/service-cognition/generated/client.ts
+++ b/apps/web/src/lib/service-clients/service-cognition/generated/client.ts
@@ -1823,8 +1823,8 @@ export const getMemoryHandler = async (
};
/**
- * @summary Complete (or skip) onboarding. Leftover onboarding-staged import
-candidates are discarded.
+ * @summary Complete (or skip) onboarding. Unreserved leftover onboarding-staged
+import candidates are deleted.
*/
export type completeHandlerResponse200 = {
data: OnboardingRow;
diff --git a/apps/web/src/lib/service-clients/service-cognition/generated/schemas/importRun.ts b/apps/web/src/lib/service-clients/service-cognition/generated/schemas/importRun.ts
index 632e8a5127..91f8541d0f 100644
--- a/apps/web/src/lib/service-clients/service-cognition/generated/schemas/importRun.ts
+++ b/apps/web/src/lib/service-clients/service-cognition/generated/schemas/importRun.ts
@@ -12,7 +12,10 @@ import type { RunStatus } from './runStatus';
* Gather-run state for one user × source.
*/
export interface ImportRun {
- /** Failure detail, when the run failed. */
+ /** Whether this run should import its onboarding-staged candidates as
+soon as gathering finishes. */
+ auto_import: boolean;
+ /** Gather or automatic-import failure detail, when the run failed. */
error?: ImportRunError;
/** The source this run gathered from. */
source: ImportSource;
diff --git a/apps/web/src/lib/service-clients/service-cognition/generated/schemas/importRunError.ts b/apps/web/src/lib/service-clients/service-cognition/generated/schemas/importRunError.ts
index c58739b0ec..9b7ec14e05 100644
--- a/apps/web/src/lib/service-clients/service-cognition/generated/schemas/importRunError.ts
+++ b/apps/web/src/lib/service-clients/service-cognition/generated/schemas/importRunError.ts
@@ -6,6 +6,6 @@
*/
/**
- * Failure detail, when the run failed.
+ * Gather or automatic-import failure detail, when the run failed.
*/
export type ImportRunError = string | null;
diff --git a/apps/web/src/lib/service-clients/service-cognition/generated/schemas/runStatus.ts b/apps/web/src/lib/service-clients/service-cognition/generated/schemas/runStatus.ts
index 7e8f126e8d..9976265c9a 100644
--- a/apps/web/src/lib/service-clients/service-cognition/generated/schemas/runStatus.ts
+++ b/apps/web/src/lib/service-clients/service-cognition/generated/schemas/runStatus.ts
@@ -14,6 +14,8 @@ export type RunStatus = (typeof RunStatus)[keyof typeof RunStatus];
export const RunStatus = {
running: 'running',
ready: 'ready',
+ importing: 'importing',
+ completed: 'completed',
failed: 'failed',
dismissed: 'dismissed',
} as const;
diff --git a/apps/web/src/lib/service-clients/service-cognition/import.ts b/apps/web/src/lib/service-clients/service-cognition/import.ts
index e2bf0abdb9..eceb9b1024 100644
--- a/apps/web/src/lib/service-clients/service-cognition/import.ts
+++ b/apps/web/src/lib/service-clients/service-cognition/import.ts
@@ -43,7 +43,13 @@ export type ImportEntityStatus =
export type ImportInitiator = 'onboarding' | 'chat';
-export type ImportRunStatus = 'running' | 'ready' | 'failed' | 'dismissed';
+export type ImportRunStatus =
+ | 'running'
+ | 'ready'
+ | 'importing'
+ | 'completed'
+ | 'failed'
+ | 'dismissed';
/** Metadata shape for staged Linear issues (`source === 'linear'`). */
export type LinearIssueMeta = {
@@ -100,6 +106,7 @@ export type ImportEntity = {
export type ImportRun = {
source: ImportSource;
status: ImportRunStatus;
+ auto_import: boolean;
error?: string | null;
updated_at: string;
};
diff --git a/apps/web/src/lib/service-clients/service-cognition/openapi.json b/apps/web/src/lib/service-clients/service-cognition/openapi.json
index 13e6b1a9d1..ea9b06e59b 100644
--- a/apps/web/src/lib/service-clients/service-cognition/openapi.json
+++ b/apps/web/src/lib/service-clients/service-cognition/openapi.json
@@ -1493,7 +1493,7 @@
"/onboarding/complete": {
"post": {
"tags": ["onboarding"],
- "summary": "Complete (or skip) onboarding. Leftover onboarding-staged import\ncandidates are discarded.",
+ "summary": "Complete (or skip) onboarding. Unreserved leftover onboarding-staged\nimport candidates are deleted.",
"operationId": "complete_handler",
"requestBody": {
"content": {
@@ -3261,11 +3261,15 @@
"ImportRun": {
"type": "object",
"description": "Gather-run state for one user × source.",
- "required": ["source", "status", "updated_at"],
+ "required": ["source", "status", "auto_import", "updated_at"],
"properties": {
+ "auto_import": {
+ "type": "boolean",
+ "description": "Whether this run should import its onboarding-staged candidates as\nsoon as gathering finishes."
+ },
"error": {
"type": ["string", "null"],
- "description": "Failure detail, when the run failed."
+ "description": "Gather or automatic-import failure detail, when the run failed."
},
"source": {
"$ref": "#/components/schemas/ImportSource",
@@ -3698,7 +3702,14 @@
"RunStatus": {
"type": "string",
"description": "Lifecycle of a gather run (one per user × source).",
- "enum": ["running", "ready", "failed", "dismissed"]
+ "enum": [
+ "running",
+ "ready",
+ "importing",
+ "completed",
+ "failed",
+ "dismissed"
+ ]
},
"SendChatMessagePayload": {
"allOf": [
diff --git a/crates/import/Cargo.toml b/crates/import/Cargo.toml
index 16856270ea..74ed10e112 100644
--- a/crates/import/Cargo.toml
+++ b/crates/import/Cargo.toml
@@ -31,3 +31,6 @@ tracing = { workspace = true }
utoipa = { workspace = true }
uuid = { workspace = true }
workspace-hack = { version = "0.1", path = "../workspace-hack" }
+
+[dev-dependencies]
+macro_db_migrator = { path = "../macro_db_migrator" }
diff --git a/crates/import/src/domain/models.rs b/crates/import/src/domain/models.rs
index 25969cc463..de7211a5f1 100644
--- a/crates/import/src/domain/models.rs
+++ b/crates/import/src/domain/models.rs
@@ -168,7 +168,12 @@ pub enum RunStatus {
Running,
/// The gather finished; staged rows (if any) are ready.
Ready,
- /// The gather failed; `error` has details. Retryable.
+ /// The gather finished and its configured automatic import is running.
+ Importing,
+ /// The configured automatic import finished successfully.
+ Completed,
+ /// Gathering or automatic importing failed; `error` has details.
+ /// Retryable.
Failed,
/// The user dismissed this source's import section.
Dismissed,
@@ -413,7 +418,10 @@ pub struct ImportRun {
pub source: ImportSource,
/// Where the run is in its lifecycle.
pub status: RunStatus,
- /// Failure detail, when the run failed.
+ /// Whether this run should import its onboarding-staged candidates as
+ /// soon as gathering finishes.
+ pub auto_import: bool,
+ /// Gather or automatic-import failure detail, when the run failed.
pub error: Option<String>,
/// When the run state last changed.
pub updated_at: DateTime<Utc>,
diff --git a/crates/import/src/domain/models/test.rs b/crates/import/src/domain/models/test.rs
index 66a3f53d32..a66f4dfcd7 100644
--- a/crates/import/src/domain/models/test.rs
+++ b/crates/import/src/domain/models/test.rs
@@ -30,6 +30,8 @@ fn enums_round_trip_through_strings() {
for run in [
RunStatus::Running,
RunStatus::Ready,
+ RunStatus::Importing,
+ RunStatus::Completed,
RunStatus::Failed,
RunStatus::Dismissed,
] {
diff --git a/crates/import/src/domain/ports.rs b/crates/import/src/domain/ports.rs
index 0985e24fc2..198475c51d 100644
--- a/crates/import/src/domain/ports.rs
+++ b/crates/import/src/domain/ports.rs
@@ -153,10 +153,11 @@ pub trait ImportRepo: Send + Sync + 'static {
initiator: Initiator,
) -> impl Future<Output = Result<u64>> + Send;
- /// DELETE all of the user's remaining `staged` rows with the given
- /// initiator (onboarding completion cleanup: candidates the user never
- /// acted on vanish from the ledger and stay re-stageable later, unlike
- /// discarded rows). Returns how many were removed.
+ /// DELETE the user's remaining unreserved `staged` rows with the given
+ /// initiator. Onboarding candidates reserved by an active or retryable
+ /// configured auto-import run survive completion cleanup; other
+ /// candidates the user never acted on vanish and stay re-stageable later.
+ /// Returns how many were removed.
fn delete_staged_by_initiator(
&self,
user: &MacroUserIdStr<'static>,
@@ -175,14 +176,16 @@ pub trait ImportRepo: Send + Sync + 'static {
user: &MacroUserIdStr<'static>,
) -> impl Future<Output = Result<Vec<ImportRun>>> + Send;
- /// CAS a gather run to `running`. Wins when no run row exists yet, or
- /// when the existing row's status is in `from`. Returns whether this
- /// call won.
+ /// CAS a gather run to `running`. New rows persist `auto_import`
+ /// atomically; retries preserve the existing run's setting. Wins when no
+ /// run row exists yet, or when the existing row's status is in `from`.
+ /// Returns whether this call won.
fn start_run(
&self,
user: &MacroUserIdStr<'static>,
source: ImportSource,
from: &[RunStatus],
+ auto_import: bool,
) -> impl Future<Output = Result<bool>> + Send;
/// CAS the `running` run for `source` to `to` (ready/failed), recording
@@ -203,6 +206,33 @@ pub trait ImportRepo: Send + Sync + 'static {
from: &[RunStatus],
to: RunStatus,
) -> impl Future<Output = Result<bool>> + Send;
+
+ /// Atomically claim a ready, auto-import-enabled run and all of its own
+ /// onboarding-staged candidates. `None` means another caller won or the
+ /// run is not ready/configured; `Some([])` is a valid empty run.
+ fn begin_auto_import(
+ &self,
+ user: &MacroUserIdStr<'static>,
+ source: ImportSource,
+ ) -> impl Future<Output = Result<Option<Vec<ImportEntity>>>> + Send;
+
+ /// Finish an auto-import run after every claimed row has settled. The run
+ /// becomes `completed` only when all claimed rows are imported; otherwise
+ /// it becomes `failed`. Returns the terminal status when this call won.
+ fn finish_auto_import(
+ &self,
+ user: &MacroUserIdStr<'static>,
+ source: ImportSource,
+ ids: &[Uuid],
+ ) -> impl Future<Output = Result<Option<RunStatus>>> + Send;
+
+ /// Repair auto-import run rows after process interruption. An `importing`
+ /// run with no importing onboarding rows is terminal: `failed` when a
+ /// staged row carries an import error, otherwise `completed`.
+ fn reconcile_auto_import_runs(
+ &self,
+ user: &MacroUserIdStr<'static>,
+ ) -> impl Future<Output = Result<u64>> + Send;
}
/// System properties to set on an imported task, already normalized to
diff --git a/crates/import/src/domain/service.rs b/crates/import/src/domain/service.rs
index b109072403..0f918c06c7 100644
--- a/crates/import/src/domain/service.rs
+++ b/crates/import/src/domain/service.rs
@@ -131,11 +131,14 @@ pub trait ImportService: Send + Sync + 'static {
) -> impl Future<Output = Result<ImportState>> + Send;
/// Start a gather run for `source` if none has ever run (the
- /// connector-just-authenticated hook). Returns whether a run started.
+ /// connector-just-authenticated hook). `auto_import` is persisted with
+ /// the new run before its background gather starts. Returns whether a
+ /// run started.
fn start_gather(
&self,
user: MacroUserIdStr<'static>,
source: ImportSource,
+ auto_import: bool,
) -> impl Future<Output = Result<bool>> + Send;
/// Restart a failed (or dismissed) gather run. Returns whether a run
@@ -171,10 +174,11 @@ pub trait ImportService: Send + Sync + 'static {
initiator: Initiator,
) -> impl Future<Output = Result<u64>> + Send;
- /// Delete all remaining staged rows with `initiator` (onboarding
- /// completion cleanup). Unlike discarding, deleted candidates were
- /// never reviewed and stay re-stageable by later gathers or chat.
- /// Returns how many were removed.
+ /// Delete remaining unreserved staged rows with `initiator` (onboarding
+ /// completion cleanup). Candidates reserved by active or retryable
+ /// configured auto-import runs survive. Unlike discarding, deleted
+ /// candidates were never reviewed and stay re-stageable by later gathers
+ /// or chat. Returns how many were removed.
fn delete_staged_by_initiator(
&self,
user: MacroUserIdStr<'static>,
@@ -305,6 +309,7 @@ where
tokio::time::timeout(GATHER_TIMEOUT, service.run_gather_session(&user, source))
.await
.unwrap_or_else(|_| Err(anyhow::anyhow!("gather session timed out")));
+ let gather_succeeded = outcome.is_ok();
let finished = match outcome {
Ok(()) => {
@@ -321,13 +326,67 @@ where
.await
}
};
- if let Err(e) = finished {
- tracing::error!(source = source.as_ref(), error = ?e, "failed to persist gather outcome");
+ match finished {
+ Ok(true) if gather_succeeded => {
+ let _ = service
+ .maybe_start_auto_import(&user, source)
+ .await
+ .inspect_err(|e| {
+ tracing::error!(source = source.as_ref(), error = ?e, "failed to start automatic import");
+ });
+ }
+ Ok(_) => {}
+ Err(e) => {
+ tracing::error!(source = source.as_ref(), error = ?e, "failed to persist gather outcome");
+ }
}
service.notify(&user).await;
});
}
+ /// Claim and spawn a configured automatic import once gathering is ready.
+ /// The repository CAS makes this safe when completion, configuration, and
+ /// read-path reconciliation race across service replicas.
+ async fn maybe_start_auto_import(
+ &self,
+ user: &MacroUserIdStr<'static>,
+ source: ImportSource,
+ ) -> Result<bool> {
+ let Some(rows) = self.repo.begin_auto_import(user, source).await? else {
+ return Ok(false);
+ };
+ let ids: Vec<Uuid> = rows.iter().map(|row| row.id).collect();
+ self.notify(user).await;
+ if rows.is_empty() {
+ self.finish_auto_import_batch(user, source, &ids).await;
+ } else {
+ self.spawn_import_batch(user.clone(), rows, Some(source));
+ }
+ Ok(true)
+ }
+
+ async fn finish_auto_import_batch(
+ &self,
+ user: &MacroUserIdStr<'static>,
+ source: ImportSource,
+ ids: &[Uuid],
+ ) {
+ match self.repo.finish_auto_import(user, source, ids).await {
+ Ok(Some(status)) => {
+ tracing::info!(
+ source = source.as_ref(),
+ status = status.as_ref(),
+ "automatic import finished"
+ );
+ self.notify(user).await;
+ }
+ Ok(None) => {}
+ Err(e) => {
+ tracing::error!(source = source.as_ref(), error = ?e, "failed to persist automatic import outcome");
+ }
+ }
+ }
+
/// Run one gather session: connector MCP tools plus the locked
/// `CreateImportEntity`. Staged rows land as the session runs; the final
/// text is ignored.
@@ -599,6 +658,122 @@ where
}
}
}
+
+ /// Run a claimed import batch in the background. Manual batches only
+ /// update their entity rows; automatic batches also settle their owning
+ /// run after every row reaches a terminal state.
+ fn spawn_import_batch(
+ &self,
+ user: MacroUserIdStr<'static>,
+ rows: Vec<ImportEntity>,
+ auto_run: Option<ImportSource>,
+ ) {
+ let service = self.clone();
+ tokio::spawn(async move {
+ let batch_ids: Vec<Uuid> = rows.iter().map(|row| row.id).collect();
+ let (notion_rows, direct_rows): (Vec<ImportEntity>, Vec<ImportEntity>) = rows
+ .into_iter()
+ .partition(|row| row.source == ImportSource::Notion);
+
+ // Heartbeat every row this batch owns (queued and in-flight) for
+ // as long as the batch runs, so the read path's stale reaper only
+ // fires on rows whose process actually died.
+ let _heartbeat = AbortOnDrop(tokio::spawn({
+ let service = service.clone();
+ let user = user.clone();
+ let ids = batch_ids.clone();
+ async move {
+ loop {
+ tokio::time::sleep(IMPORT_HEARTBEAT).await;
+ let _ = service
+ .repo
+ .touch_importing(&user, &ids)
+ .await
+ .inspect_err(|e| {
+ tracing::warn!(error = ?e, "import heartbeat failed");
+ });
+ }
+ }
+ }));
+
+ for row in &direct_rows {
+ service.import_deterministic(&user, row).await;
+ }
+
+ if !notion_rows.is_empty() {
+ // One connector connection per batch, pages a few at a time.
+ // Each page first tries direct fetch → markdown → finalize;
+ // the agent session is the fallback.
+ let mcp_tools = match service.connector_tools(&user, ImportSource::Notion).await {
+ Ok(tools) => tools,
+ Err(e) => {
+ tracing::warn!(error = ?e, "notion connector unavailable");
+ let ids: Vec<Uuid> = notion_rows.iter().map(|row| row.id).collect();
+ service
+ .fail_unfinished(&user, &ids, &format!("notion unavailable: {e}"))
+ .await;
+ service.notify(&user).await;
+ if let Some(source) = auto_run {
+ service
+ .finish_auto_import_batch(&user, source, &batch_ids)
+ .await;
+ }
+ return;
+ }
+ };
+
+ futures::stream::iter(notion_rows)
+ .for_each_concurrent(NOTION_IMPORT_CONCURRENCY, |row| {
+ let service = service.clone();
+ let user = user.clone();
+ let mcp_tools = mcp_tools.clone();
+ async move {
+ let outcome =
+ tokio::time::timeout(NOTION_PAGE_IMPORT_TIMEOUT, async {
+ match service
+ .import_notion_page_direct(&user, &mcp_tools, &row)
+ .await
+ {
+ Ok(()) => Ok(()),
+ Err(direct_error) => {
+ tracing::info!(id = %row.id, error = ?direct_error, "direct notion import failed; trying the agent");
+ service
+ .run_notion_import_session(
+ &user,
+ std::slice::from_ref(&row),
+ mcp_tools.clone(),
+ )
+ .await
+ }
+ }
+ })
+ .await
+ .unwrap_or_else(|_| {
+ Err(anyhow::anyhow!("notion import timed out"))
+ });
+ let _ = outcome.inspect_err(|e| {
+ tracing::warn!(id = %row.id, error = ?e, "notion page import failed");
+ });
+ service
+ .fail_unfinished(
+ &user,
+ &[row.id],
+ "the import job did not finish this item",
+ )
+ .await;
+ }
+ })
+ .await;
+ service.notify(&user).await;
+ }
+
+ if let Some(source) = auto_run {
+ service
+ .finish_auto_import_batch(&user, source, &batch_ids)
+ .await;
+ }
+ });
+ }
}
impl<R, S, C> ImportService for ImportServiceImpl<R, S, C>
@@ -627,7 +802,37 @@ where
Err(e) => tracing::warn!(error = ?e, "failed to reap stale imports"),
}
- let runs = self.repo.list_runs(&user).await?;
+ match self.repo.reconcile_auto_import_runs(&user).await {
+ Ok(0) => {}
+ Ok(reconciled) => {
+ tracing::warn!(reconciled, "reconciled interrupted automatic import runs");
+ self.notify(&user).await;
+ }
+ // A read must never fail because recovery did.
+ Err(e) => tracing::warn!(error = ?e, "failed to reconcile automatic import runs"),
+ }
+
+ // Self-heal the small window between persisting gather completion and
+ // spawning its configured follow-on batch. The begin CAS makes this
+ // safe across concurrent readers and replicas.
+ let mut runs = self.repo.list_runs(&user).await?;
+ let auto_ready: Vec<ImportSource> = runs
+ .iter()
+ .filter(|run| run.auto_import && run.status == RunStatus::Ready)
+ .map(|run| run.source)
+ .collect();
+ let mut started = false;
+ for source in auto_ready {
+ match self.maybe_start_auto_import(&user, source).await {
+ Ok(did_start) => started |= did_start,
+ Err(e) => {
+ tracing::warn!(source = source.as_ref(), error = ?e, "failed to self-heal automatic import start");
+ }
+ }
+ }
+ if started {
+ runs = self.repo.list_runs(&user).await?;
+ }
let entities = self.repo.list(&user, None, None).await?;
Ok(ImportState { runs, entities })
}
@@ -637,10 +842,11 @@ where
&self,
user: MacroUserIdStr<'static>,
source: ImportSource,
+ auto_import: bool,
) -> Result<bool> {
// Wins only when no run row exists — gathers run once per
// connection; explicit retry is the only re-entry.
- let won = self.repo.start_run(&user, source, &[]).await?;
+ let won = self.repo.start_run(&user, source, &[], auto_import).await?;
if won {
self.spawn_gather(user.clone(), source);
self.notify(&user).await;
@@ -656,7 +862,12 @@ where
) -> Result<bool> {
let won = self
.repo
- .start_run(&user, source, &[RunStatus::Failed, RunStatus::Dismissed])
+ .start_run(
+ &user,
+ source,
+ &[RunStatus::Failed, RunStatus::Dismissed],
+ false,
+ )
.await?;
if won {
self.spawn_gather(user.clone(), source);
@@ -667,7 +878,8 @@ where
#[tracing::instrument(skip(self, user), err)]
async fn dismiss_run(&self, user: MacroUserIdStr<'static>, source: ImportSource) -> Result<()> {
- self.repo
+ let dismissed = self
+ .repo
.transition_run(
&user,
source,
@@ -675,6 +887,9 @@ where
RunStatus::Dismissed,
)
.await?;
+ if dismissed {
+ self.notify(&user).await;
+ }
Ok(())
}
@@ -697,106 +912,7 @@ where
self.notify(&user).await;
if !rows.is_empty() {
- let (notion_rows, direct_rows): (Vec<ImportEntity>, Vec<ImportEntity>) = rows
- .into_iter()
- .partition(|row| row.source == ImportSource::Notion);
-
- let service = self.clone();
- let user = user.clone();
- tokio::spawn(async move {
- // Heartbeat every row this batch owns (queued and in-flight)
- // for as long as the batch runs, so the read path's stale
- // reaper only ever fires on rows whose process actually
- // died. Dropped (aborted) on every exit path of this task.
- let _heartbeat = AbortOnDrop(tokio::spawn({
- let service = service.clone();
- let user = user.clone();
- let ids: Vec<Uuid> = direct_rows
- .iter()
- .chain(notion_rows.iter())
- .map(|row| row.id)
- .collect();
- async move {
- loop {
- tokio::time::sleep(IMPORT_HEARTBEAT).await;
- if let Err(e) = service.repo.touch_importing(&user, &ids).await {
- tracing::warn!(error = ?e, "import heartbeat failed");
- }
- }
- }
- }));
-
- for row in &direct_rows {
- service.import_deterministic(&user, row).await;
- }
-
- if !notion_rows.is_empty() {
- // One connector connection per batch, pages a few at a
- // time. Each page first tries the direct path (fetch
- // tool → markdown → finalize, no model); the Haiku
- // session is the fallback for pages that path can't
- // handle.
- let mcp_tools = match service.connector_tools(&user, ImportSource::Notion).await
- {
- Ok(tools) => tools,
- Err(e) => {
- tracing::warn!(error = ?e, "notion connector unavailable");
- let ids: Vec<Uuid> = notion_rows.iter().map(|r| r.id).collect();
- service
- .fail_unfinished(&user, &ids, &format!("notion unavailable: {e}"))
- .await;
- service.notify(&user).await;
- return;
- }
- };
-
- futures::stream::iter(notion_rows)
- .for_each_concurrent(NOTION_IMPORT_CONCURRENCY, |row| {
- let service = service.clone();
- let user = user.clone();
- let mcp_tools = mcp_tools.clone();
- async move {
- let outcome = tokio::time::timeout(
- NOTION_PAGE_IMPORT_TIMEOUT,
- async {
- match service
- .import_notion_page_direct(&user, &mcp_tools, &row)
- .await
- {
- Ok(()) => Ok(()),
- Err(direct_error) => {
- tracing::info!(id = %row.id, error = ?direct_error, "direct notion import failed; trying the agent");
- service
- .run_notion_import_session(
- &user,
- std::slice::from_ref(&row),
- mcp_tools.clone(),
- )
- .await
- }
- }
- },
- )
- .await
- .unwrap_or_else(|_| {
- Err(anyhow::anyhow!("notion import timed out"))
- });
- if let Err(e) = outcome {
- tracing::warn!(id = %row.id, error = ?e, "notion page import failed");
- }
- service
- .fail_unfinished(
- &user,
- &[row.id],
- "the import job did not finish this item",
- )
- .await;
- }
- })
- .await;
- service.notify(&user).await;
- }
- });
+ self.spawn_import_batch(user.clone(), rows, None);
}
Ok(RunImportOutcome {
diff --git a/crates/import/src/outbound/pg_import_repo.rs b/crates/import/src/outbound/pg_import_repo.rs
index 962fc39de8..dea265f4c1 100644
--- a/crates/import/src/outbound/pg_import_repo.rs
+++ b/crates/import/src/outbound/pg_import_repo.rs
@@ -15,6 +15,9 @@ use std::collections::HashSet;
use std::str::FromStr;
use uuid::Uuid;
+#[cfg(test)]
+mod test;
+
/// Postgres-backed import repository (MacroDB).
#[derive(Clone)]
pub struct PgImportRepo {
@@ -75,6 +78,7 @@ impl TryFrom<ImportEntityDbRow> for ImportEntity {
struct ImportRunDbRow {
source: String,
status: String,
+ auto_import: bool,
error: Option<String>,
updated_at: DateTime<Utc>,
}
@@ -90,6 +94,7 @@ impl TryFrom<ImportRunDbRow> for ImportRun {
Ok(ImportRun {
source,
status,
+ auto_import: row.auto_import,
error: row.error,
updated_at: row.updated_at,
})
@@ -472,6 +477,17 @@ impl ImportRepo for PgImportRepo {
r#"
DELETE FROM import_entity
WHERE user_id = $1 AND initiator = $2 AND status = 'staged'
+ AND NOT (
+ $2 = 'onboarding'
+ AND EXISTS (
+ SELECT 1
+ FROM import_run
+ WHERE import_run.user_id = import_entity.user_id
+ AND import_run.source = import_entity.source
+ AND import_run.auto_import
+ AND import_run.status IN ('running', 'ready', 'importing', 'failed')
+ )
+ )
"#,
user.as_ref(),
initiator.as_ref(),
@@ -497,7 +513,7 @@ impl ImportRepo for PgImportRepo {
let rows = sqlx::query_as!(
ImportRunDbRow,
r#"
- SELECT source, status, error, updated_at
+ SELECT source, status::text AS "status!", auto_import, error, updated_at
FROM import_run WHERE user_id = $1
ORDER BY source
"#,
@@ -514,19 +530,21 @@ impl ImportRepo for PgImportRepo {
user: &MacroUserIdStr<'static>,
source: ImportSource,
from: &[RunStatus],
+ auto_import: bool,
) -> Result<bool> {
let from = run_status_strings(from);
let result = sqlx::query!(
r#"
- INSERT INTO import_run (user_id, source, status)
- VALUES ($1, $2, 'running')
+ INSERT INTO import_run (user_id, source, status, auto_import)
+ VALUES ($1, $2, 'running', $4)
ON CONFLICT (user_id, source) DO UPDATE
SET status = 'running', error = NULL, updated_at = NOW()
- WHERE import_run.status = ANY($3::text[])
+ WHERE import_run.status::text = ANY($3::text[])
"#,
user.as_ref(),
source.as_ref(),
&from,
+ auto_import,
)
.execute(&self.pool)
.await?;
@@ -544,7 +562,9 @@ impl ImportRepo for PgImportRepo {
let result = sqlx::query!(
r#"
UPDATE import_run
- SET status = $3, error = $4, updated_at = NOW()
+ SET status = $3::text::import_run_status,
+ error = $4,
+ updated_at = NOW()
WHERE user_id = $1 AND source = $2 AND status = 'running'
"#,
user.as_ref(),
@@ -569,8 +589,9 @@ impl ImportRepo for PgImportRepo {
let result = sqlx::query!(
r#"
UPDATE import_run
- SET status = $4, updated_at = NOW()
- WHERE user_id = $1 AND source = $2 AND status = ANY($3::text[])
+ SET status = $4::text::import_run_status, updated_at = NOW()
+ WHERE user_id = $1 AND source = $2
+ AND status::text = ANY($3::text[])
"#,
user.as_ref(),
source.as_ref(),
@@ -581,4 +602,145 @@ impl ImportRepo for PgImportRepo {
.await?;
Ok(result.rows_affected() > 0)
}
+
+ #[tracing::instrument(skip(self), err)]
+ async fn begin_auto_import(
+ &self,
+ user: &MacroUserIdStr<'static>,
+ source: ImportSource,
+ ) -> Result<Option<Vec<ImportEntity>>> {
+ let mut transaction = self.pool.begin().await?;
+ let claimed = sqlx::query_scalar!(
+ r#"
+ UPDATE import_run
+ SET status = 'importing', error = NULL, updated_at = NOW()
+ WHERE user_id = $1 AND source = $2
+ AND status = 'ready' AND auto_import
+ RETURNING source
+ "#,
+ user.as_ref(),
+ source.as_ref(),
+ )
+ .fetch_optional(&mut *transaction)
+ .await?;
+ if claimed.is_none() {
+ return Ok(None);
+ }
+
+ let rows = sqlx::query_as!(
+ ImportEntityDbRow,
+ r#"
+ UPDATE import_entity
+ SET status = 'importing', last_error = NULL, updated_at = NOW()
+ WHERE user_id = $1 AND source = $2
+ AND initiator = 'onboarding' AND status = 'staged'
+ RETURNING id, user_id, team_id, source, foreign_id, status, initiator,
+ metadata, entity_id, entity_type, last_error, created_at, updated_at
+ "#,
+ user.as_ref(),
+ source.as_ref(),
+ )
+ .fetch_all(&mut *transaction)
+ .await?;
+ transaction.commit().await?;
+ Ok(Some(
+ rows.into_iter()
+ .map(ImportEntity::try_from)
+ .collect::<Result<Vec<_>>>()?,
+ ))
+ }
+
+ #[tracing::instrument(skip(self, ids), fields(count = ids.len()), err)]
+ async fn finish_auto_import(
+ &self,
+ user: &MacroUserIdStr<'static>,
+ source: ImportSource,
+ ids: &[Uuid],
+ ) -> Result<Option<RunStatus>> {
+ let status = sqlx::query_scalar!(
+ r#"
+ WITH outcome AS (
+ SELECT
+ COUNT(*) FILTER (WHERE status = 'imported')
+ = CARDINALITY($3::uuid[])::bigint AS succeeded
+ FROM import_entity
+ WHERE user_id = $1 AND id = ANY($3::uuid[])
+ )
+ UPDATE import_run
+ SET status = (CASE
+ WHEN outcome.succeeded THEN 'completed'
+ ELSE 'failed'
+ END)::import_run_status,
+ error = CASE
+ WHEN outcome.succeeded THEN NULL
+ ELSE 'one or more automatic imports failed'
+ END,
+ updated_at = NOW()
+ FROM outcome
+ WHERE user_id = $1 AND source = $2 AND status = 'importing'
+ RETURNING import_run.status::text AS "status!"
+ "#,
+ user.as_ref(),
+ source.as_ref(),
+ ids,
+ )
+ .fetch_optional(&self.pool)
+ .await?;
+ status
+ .map(|status| {
+ RunStatus::from_str(&status)
+ .map_err(|_| anyhow::anyhow!("unknown run status: {status}").into())
+ })
+ .transpose()
+ }
+
+ #[tracing::instrument(skip(self), err)]
+ async fn reconcile_auto_import_runs(&self, user: &MacroUserIdStr<'static>) -> Result<u64> {
+ let result = sqlx::query!(
+ r#"
+ WITH settled AS (
+ SELECT
+ run.source,
+ EXISTS (
+ SELECT 1
+ FROM import_entity entity
+ WHERE entity.user_id = run.user_id
+ AND entity.source = run.source
+ AND entity.initiator = 'onboarding'
+ AND entity.status = 'staged'
+ AND entity.last_error IS NOT NULL
+ ) AS failed
+ FROM import_run run
+ WHERE run.user_id = $1
+ AND run.status = 'importing'
+ AND run.auto_import
+ AND NOT EXISTS (
+ SELECT 1
+ FROM import_entity entity
+ WHERE entity.user_id = run.user_id
+ AND entity.source = run.source
+ AND entity.initiator = 'onboarding'
+ AND entity.status = 'importing'
+ )
+ )
+ UPDATE import_run run
+ SET status = (
+ CASE WHEN settled.failed THEN 'failed' ELSE 'completed' END
+ )::import_run_status,
+ error = CASE
+ WHEN settled.failed THEN 'one or more automatic imports failed'
+ ELSE NULL
+ END,
+ updated_at = NOW()
+ FROM settled
+ WHERE run.user_id = $1
+ AND run.source = settled.source
+ AND run.status = 'importing'
+ "#,
+ user.as_ref(),
+ )
+ .execute(&self.pool)
+ .await?;
+ Ok(result.rows_affected())
+ }
}
diff --git a/crates/import/src/outbound/pg_import_repo/test.rs b/crates/import/src/outbound/pg_import_repo/test.rs
new file mode 100644
index 0000000000..8167182ecd
--- /dev/null
+++ b/crates/import/src/outbound/pg_import_repo/test.rs
@@ -0,0 +1,207 @@
+use super::*;
+use macro_db_migrator::MACRO_DB_MIGRATIONS;
+use sqlx::{Pool, Postgres};
+
+fn user() -> MacroUserIdStr<'static> {
+ MacroUserIdStr::parse_from_str("macro|[email protected]").expect("valid user id")
+}
+
+async fn ready_run(repo: &PgImportRepo, auto_import: bool) {
+ assert!(
+ repo.start_run(&user(), ImportSource::Linear, &[], auto_import)
+ .await
+ .expect("start run")
+ );
+ assert!(
+ repo.finish_run(&user(), ImportSource::Linear, RunStatus::Ready, None)
+ .await
+ .expect("finish gather")
+ );
+}
+
+async fn stage(
+ repo: &PgImportRepo,
+ source: ImportSource,
+ initiator: Initiator,
+ foreign_id: &str,
+) -> ImportEntity {
+ let metadata = match source {
+ ImportSource::Linear => serde_json::json!({ "title": foreign_id }),
+ ImportSource::Notion => serde_json::json!({ "title": foreign_id }),
+ ImportSource::Slack => serde_json::json!({ "name": foreign_id }),
+ };
+ repo.upsert_staged(&user(), source, initiator, foreign_id, &metadata)
+ .await
+ .expect("stage query")
+ .expect("staged row")
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn auto_import_claims_only_its_onboarding_source(pool: Pool<Postgres>) {
+ let repo = PgImportRepo::new(pool);
+ ready_run(&repo, true).await;
+
+ let claimed = stage(&repo, ImportSource::Linear, Initiator::Onboarding, "LIN-1").await;
+ let chat = stage(&repo, ImportSource::Linear, Initiator::Chat, "LIN-CHAT").await;
+ let notion = stage(
+ &repo,
+ ImportSource::Notion,
+ Initiator::Onboarding,
+ "NOTION-1",
+ )
+ .await;
+
+ assert_eq!(
+ repo.delete_staged_by_initiator(&user(), Initiator::Onboarding)
+ .await
+ .expect("onboarding cleanup"),
+ 1,
+ "only the unrelated Notion row should be removed"
+ );
+ assert!(
+ repo.get(&user(), notion.id)
+ .await
+ .expect("notion row lookup")
+ .is_none()
+ );
+
+ let rows = repo
+ .begin_auto_import(&user(), ImportSource::Linear)
+ .await
+ .expect("begin auto import")
+ .expect("claimed run");
+ assert_eq!(rows.len(), 1);
+ assert_eq!(rows[0].id, claimed.id);
+ assert_eq!(rows[0].status, ImportStatus::Importing);
+
+ assert_eq!(
+ repo.get(&user(), chat.id)
+ .await
+ .expect("chat row")
+ .expect("chat row exists")
+ .status,
+ ImportStatus::Staged
+ );
+ assert!(
+ repo.begin_auto_import(&user(), ImportSource::Linear)
+ .await
+ .expect("second begin")
+ .is_none(),
+ "the run claim is a CAS"
+ );
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn auto_import_run_reports_completed_or_failed(pool: Pool<Postgres>) {
+ let repo = PgImportRepo::new(pool);
+ ready_run(&repo, true).await;
+ let row = stage(&repo, ImportSource::Linear, Initiator::Onboarding, "LIN-1").await;
+ let claimed = repo
+ .begin_auto_import(&user(), ImportSource::Linear)
+ .await
+ .expect("begin")
+ .expect("claimed");
+ assert_eq!(claimed.len(), 1);
+ repo.mark_imported(&user(), row.id, "task-1", "task", None)
+ .await
+ .expect("mark imported")
+ .expect("updated row");
+
+ assert_eq!(
+ repo.finish_auto_import(&user(), ImportSource::Linear, &[row.id])
+ .await
+ .expect("finish")
+ .expect("won finish"),
+ RunStatus::Completed
+ );
+ let run = repo
+ .list_runs(&user())
+ .await
+ .expect("runs")
+ .pop()
+ .expect("run");
+ assert_eq!(run.status, RunStatus::Completed);
+ assert!(run.auto_import);
+
+ // A retry preserves the run's configuration and can report an item
+ // failure on its next automatic batch.
+ assert!(
+ repo.start_run(
+ &user(),
+ ImportSource::Linear,
+ &[RunStatus::Completed],
+ false,
+ )
+ .await
+ .expect("restart for test")
+ );
+ assert!(
+ repo.finish_run(&user(), ImportSource::Linear, RunStatus::Ready, None)
+ .await
+ .expect("ready again")
+ );
+ let failed_row = stage(&repo, ImportSource::Linear, Initiator::Onboarding, "LIN-2").await;
+ repo.begin_auto_import(&user(), ImportSource::Linear)
+ .await
+ .expect("begin failed batch")
+ .expect("claimed failed batch");
+ assert!(
+ repo.mark_import_failed(&user(), failed_row.id, "creator failed")
+ .await
+ .expect("mark failed")
+ );
+ assert_eq!(
+ repo.finish_auto_import(&user(), ImportSource::Linear, &[failed_row.id])
+ .await
+ .expect("finish failed")
+ .expect("won failed finish"),
+ RunStatus::Failed
+ );
+ assert_eq!(
+ repo.delete_staged_by_initiator(&user(), Initiator::Onboarding)
+ .await
+ .expect("cleanup after failure"),
+ 0,
+ "a retryable automatic-import failure must survive onboarding cleanup"
+ );
+ assert!(
+ repo.get(&user(), failed_row.id)
+ .await
+ .expect("failed row lookup")
+ .is_some()
+ );
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn interrupted_auto_import_run_is_reconciled(pool: Pool<Postgres>) {
+ let repo = PgImportRepo::new(pool);
+ ready_run(&repo, true).await;
+ let row = stage(&repo, ImportSource::Linear, Initiator::Onboarding, "LIN-1").await;
+ repo.begin_auto_import(&user(), ImportSource::Linear)
+ .await
+ .expect("begin")
+ .expect("claimed");
+ assert!(
+ repo.mark_import_failed(&user(), row.id, "process interrupted")
+ .await
+ .expect("mark interrupted")
+ );
+
+ assert_eq!(
+ repo.reconcile_auto_import_runs(&user())
+ .await
+ .expect("reconcile"),
+ 1
+ );
+ let run = repo
+ .list_runs(&user())
+ .await
+ .expect("runs")
+ .pop()
+ .expect("run");
+ assert_eq!(run.status, RunStatus::Failed);
+ assert_eq!(
+ run.error.as_deref(),
+ Some("one or more automatic imports failed")
+ );
+}
diff --git a/crates/macro_db_client/migrations/20260723150434_support_auto_import_runs.sql b/crates/macro_db_client/migrations/20260723150434_support_auto_import_runs.sql
new file mode 100644
index 0000000000..dd0121ff41
--- /dev/null
+++ b/crates/macro_db_client/migrations/20260723150434_support_auto_import_runs.sql
@@ -0,0 +1,22 @@
+-- A gather run can optionally continue straight through importing every
+-- onboarding-staged candidate it discovered. The run status then exposes the
+-- import batch's progress and terminal outcome to clients.
+
+CREATE TYPE import_run_status AS ENUM (
+ 'running',
+ 'ready',
+ 'importing',
+ 'completed',
+ 'failed',
+ 'dismissed'
+);
+
+ALTER TABLE import_run
+ ADD COLUMN auto_import BOOLEAN NOT NULL DEFAULT FALSE;
+
+ALTER TABLE import_run
+ DROP CONSTRAINT import_run_status_check;
+
+ALTER TABLE import_run
+ ALTER COLUMN status TYPE import_run_status
+ USING status::import_run_status;
diff --git a/crates/onboarding/src/domain/service.rs b/crates/onboarding/src/domain/service.rs
index 5ec8edefe1..8292d88b63 100644
--- a/crates/onboarding/src/domain/service.rs
+++ b/crates/onboarding/src/domain/service.rs
@@ -3,9 +3,9 @@
//!
//! Reconciliation is level-triggered and idempotent: every tick lists the
//! user's MCP connections and asks the import service to start a gather for
-//! each authenticated connector. `start_gather` is a CAS that only wins when
-//! no run has ever started, so ticking on every read (and on the MCP
-//! post-auth hook) is safe.
+//! each authenticated connector with automatic import enabled.
+//! `start_gather` is a CAS that only wins when no run has ever started, so
+//! ticking on every read (and on the MCP post-auth hook) is safe.
use super::models::{ConnectedServer, OnboardingRow, OnboardingState, OnboardingStatus};
use super::ports::{OnboardingError, OnboardingRepo, Result};
@@ -34,9 +34,11 @@ pub trait OnboardingService: Send + Sync + 'static {
/// outside onboarding must not start gathers.
fn reconcile(&self, user: MacroUserIdStr<'static>) -> impl Future<Output = Result<()>> + Send;
- /// Finish the flow (`skipped` records how). Leftover onboarding-staged
- /// import candidates are deleted — never reviewed is not declined, so
- /// they stay re-stageable by later gathers or chat.
+ /// Finish the flow (`skipped` records how). Unreserved leftover
+ /// onboarding-staged import candidates are deleted — never reviewed is
+ /// not declined, so they stay re-stageable by later gathers or chat.
+ /// Candidates reserved by configured automatic runs remain until their
+ /// run settles.
fn complete(
&self,
user: MacroUserIdStr<'static>,
@@ -88,8 +90,9 @@ where
.collect())
}
- /// Start a gather run for every authenticated connector that never had
- /// one. A failing source must not block the others (or the read).
+ /// Start an auto-importing gather run for every authenticated connector
+ /// that never had one. A failing source must not block the others (or
+ /// the read).
async fn start_due_gathers(&self, user: &MacroUserIdStr<'static>, servers: &[ConnectedServer]) {
for source in ImportSource::all() {
let connected = servers
@@ -98,7 +101,7 @@ where
if !connected {
continue;
}
- if let Err(e) = self.import.start_gather(user.clone(), source).await {
+ if let Err(e) = self.import.start_gather(user.clone(), source, true).await {
tracing::warn!(source = source.as_ref(), error = ?e, "failed to start gather run");
}
}
@@ -158,11 +161,13 @@ where
skipped: bool,
) -> Result<OnboardingRow> {
let row = self.repo.complete(&user, skipped).await?;
- // Candidates the user never acted on are removed from the ledger,
- // not discarded: discard means "the user said no" and blocks
+ // Unreserved candidates the user never acted on are removed from the
+ // ledger, not discarded: discard means "the user said no" and blocks
// re-staging forever, which is wrong for items they never looked at
- // ("Set up later" skips the whole review). Explicit declines (a
- // section toggled off) were already discarded by run_import.
+ // ("Set up later" skips the whole review). Candidates reserved by an
+ // active or retryable configured auto-import run survive this cleanup.
+ // Explicit declines (a section toggled off) were already discarded by
+ // run_import.
// Chat-staged rows are untouched. Best-effort: the completion above
// is durable, so a cleanup failure must not fail the call — the
// next reconcile tick sweeps leftovers anyway.
diff --git a/crates/onboarding/src/domain/service/test.rs b/crates/onboarding/src/domain/service/test.rs
index d6ce282a1e..5302bb6f9a 100644
--- a/crates/onboarding/src/domain/service/test.rs
+++ b/crates/onboarding/src/domain/service/test.rs
@@ -110,7 +110,7 @@ impl McpServerStore for MockStore {
#[derive(Default)]
struct MockImport {
- gathers: Mutex<Vec<ImportSource>>,
+ gathers: Mutex<Vec<(ImportSource, bool)>>,
discards: Mutex<Vec<Initiator>>,
deletions: Mutex<Vec<Initiator>>,
}
@@ -127,8 +127,9 @@ impl ImportService for MockImport {
&self,
_user: MacroUserIdStr<'static>,
source: ImportSource,
+ auto_import: bool,
) -> ImportResult<bool> {
- self.gathers.lock().unwrap().push(source);
+ self.gathers.lock().unwrap().push((source, auto_import));
Ok(true)
}
@@ -223,7 +224,7 @@ async fn active_flow_starts_gathers_for_authenticated_connectors_only() {
assert_eq!(state.connected_servers.len(), 3);
assert_eq!(
import.gathers.lock().unwrap().as_slice(),
- &[ImportSource::Linear]
+ &[(ImportSource::Linear, true)]
);
}
diff --git a/crates/onboarding/src/inbound/axum_router.rs b/crates/onboarding/src/inbound/axum_router.rs
index 3e09f1a9fd..cb8e0d5097 100644
--- a/crates/onboarding/src/inbound/axum_router.rs
+++ b/crates/onboarding/src/inbound/axum_router.rs
@@ -98,8 +98,8 @@ pub async fn get_state_handler<T: OnboardingService, Auth: MacroAuthorizationSer
}
}
-/// Complete (or skip) onboarding. Leftover onboarding-staged import
-/// candidates are discarded.
+/// Complete (or skip) onboarding. Unreserved leftover onboarding-staged
+/// import candidates are deleted.
#[utoipa::path(
post,
path = "/onboarding/complete",
diff --git a/crates/onboarding/src/lib.rs b/crates/onboarding/src/lib.rs
index 72daee34d3..c5cc689b47 100644
--- a/crates/onboarding/src/lib.rs
+++ b/crates/onboarding/src/lib.rs
@@ -4,8 +4,9 @@
//!
//! All the actual import machinery — staging, dedup, gather/import jobs —
//! lives in the `import` crate. Onboarding is just a feature that uses it:
-//! it starts gathers for authenticated connectors while the flow is active,
-//! and discards leftover onboarding-staged rows when the flow completes.
+//! it starts auto-importing gathers for authenticated connectors while the
+//! flow is active, and deletes unreserved onboarding-staged rows when the
+//! flow completes.
#![deny(missing_docs)]
diff --git a/services/document_cognition_service/src/main.rs b/services/document_cognition_service/src/main.rs
index 4fceed396c..7908ede2c5 100644
--- a/services/document_cognition_service/src/main.rs
+++ b/services/document_cognition_service/src/main.rs
@@ -558,8 +558,8 @@ async fn main() -> anyhow::Result<()> {
tracing::info!("initialized ai projections service");
// The onboarding flow drives the import pipeline: reads/hooks start
- // gather runs for authenticated connectors, completion discards leftover
- // onboarding-staged candidates.
+ // auto-importing gather runs for authenticated connectors, and
+ // completion deletes unreserved onboarding-staged candidates.
let onboarding_service = Arc::new(onboarding::domain::service::OnboardingServiceImpl::new(
onboarding::outbound::pg_onboarding_repo::PgOnboardingRepo::new(db.clone()),
Arc::new(mcp_server_repo.clone()),