Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

feat(messages): idempotent import of legacy document comments (#6437)

gbirman43cb2efaec2e
diff --git a/.github/workspace-dep-closures.json b/.github/workspace-dep-closures.json
index 431d81b864..25e5a992d9 100644
--- a/.github/workspace-dep-closures.json
+++ b/.github/workspace-dep-closures.json
@@ -5690,7 +5690,12 @@
"crates/workspace-hack"
],
"macro_db_migrator": [
+ "crates/database_env_vars",
"crates/macro_db_migrator",
+ "crates/macro_entrypoint",
+ "crates/macro_env",
+ "crates/macro_env_var",
+ "crates/macro_uuid",
"crates/workspace-hack"
],
"macro_entrypoint": [
diff --git a/.sqlx/query-038db5b91138c574e5927816834bdedef8ac3d00b378999c7370d1b6f8ec22be.json b/.sqlx/query-038db5b91138c574e5927816834bdedef8ac3d00b378999c7370d1b6f8ec22be.json
new file mode 100644
index 0000000000..94d118fc91
--- /dev/null
+++ b/.sqlx/query-038db5b91138c574e5927816834bdedef8ac3d00b378999c7370d1b6f8ec22be.json
@@ -0,0 +1,22 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "INSERT INTO comms_messages (\n id, parent_entity_type, parent_entity_id, sender_id, content,\n created_at, updated_at, deleted_at\n )\n SELECT tm.root_id, 'document', t.\"documentId\", t.owner, '',\n t.\"createdAt\" AT TIME ZONE 'UTC', t.\"updatedAt\" AT TIME ZONE 'UTC',\n COALESCE(t.\"deletedAt\", t.\"updatedAt\")\n FROM \"Thread\" t\n JOIN migrated_comment_thread_id tm ON tm.thread_id = t.id\n WHERE t.\"documentId\" = ANY($1)\n AND NOT EXISTS (SELECT 1 FROM \"Comment\" c WHERE c.\"threadId\" = t.id)\n ON CONFLICT (id) DO UPDATE SET\n updated_at = EXCLUDED.updated_at,\n deleted_at = EXCLUDED.deleted_at\n WHERE comms_messages.updated_at <= EXCLUDED.updated_at\n AND (comms_messages.updated_at, comms_messages.deleted_at)\n IS DISTINCT FROM (EXCLUDED.updated_at, EXCLUDED.deleted_at)\n RETURNING (xmax = 0) AS \"inserted!\"",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "inserted!",
+ "type_info": "Bool"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "TextArray"
+ ]
+ },
+ "nullable": [
+ null
+ ]
+ },
+ "hash": "038db5b91138c574e5927816834bdedef8ac3d00b378999c7370d1b6f8ec22be"
+}
diff --git a/.sqlx/query-099de7baaa6a5bbb26a23719ae554f5e71c055fd4a36eba60c158d663d8d65b6.json b/.sqlx/query-099de7baaa6a5bbb26a23719ae554f5e71c055fd4a36eba60c158d663d8d65b6.json
new file mode 100644
index 0000000000..605567a034
--- /dev/null
+++ b/.sqlx/query-099de7baaa6a5bbb26a23719ae554f5e71c055fd4a36eba60c158d663d8d65b6.json
@@ -0,0 +1,16 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "INSERT INTO migrated_comment_thread_id (thread_id, root_id, document_id)\n SELECT * FROM UNNEST($1::bigint[], $2::uuid[], $3::text[])",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Int8Array",
+ "UuidArray",
+ "TextArray"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "099de7baaa6a5bbb26a23719ae554f5e71c055fd4a36eba60c158d663d8d65b6"
+}
diff --git a/.sqlx/query-12eba8d33668ec91d11bf8787de9c90499802d0ebe46f7a5b2e5d0f958c12364.json b/.sqlx/query-12eba8d33668ec91d11bf8787de9c90499802d0ebe46f7a5b2e5d0f958c12364.json
new file mode 100644
index 0000000000..c16520dedf
--- /dev/null
+++ b/.sqlx/query-12eba8d33668ec91d11bf8787de9c90499802d0ebe46f7a5b2e5d0f958c12364.json
@@ -0,0 +1,28 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "SELECT comment_id, message_id FROM migrated_comment_id WHERE document_id = $1",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "comment_id",
+ "type_info": "Int8"
+ },
+ {
+ "ordinal": 1,
+ "name": "message_id",
+ "type_info": "Uuid"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Text"
+ ]
+ },
+ "nullable": [
+ false,
+ false
+ ]
+ },
+ "hash": "12eba8d33668ec91d11bf8787de9c90499802d0ebe46f7a5b2e5d0f958c12364"
+}
diff --git a/.sqlx/query-180a9c059064bca1eab390de5c76bf41bc489612d22ae37c8bf52989ba8e062a.json b/.sqlx/query-180a9c059064bca1eab390de5c76bf41bc489612d22ae37c8bf52989ba8e062a.json
new file mode 100644
index 0000000000..261ad9c71a
--- /dev/null
+++ b/.sqlx/query-180a9c059064bca1eab390de5c76bf41bc489612d22ae37c8bf52989ba8e062a.json
@@ -0,0 +1,14 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "UPDATE \"PdfPlaceableCommentAnchor\" a SET root_id = tm.root_id\n FROM migrated_comment_thread_id tm\n WHERE a.\"threadId\" = tm.thread_id AND a.\"documentId\" = ANY($1)\n AND a.root_id IS DISTINCT FROM tm.root_id",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "TextArray"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "180a9c059064bca1eab390de5c76bf41bc489612d22ae37c8bf52989ba8e062a"
+}
diff --git a/.sqlx/query-27dba71899c33b28e6f3daf31030b8e9f609ee7e6b0a62bba74ac168ac76f8cd.json b/.sqlx/query-27dba71899c33b28e6f3daf31030b8e9f609ee7e6b0a62bba74ac168ac76f8cd.json
new file mode 100644
index 0000000000..9cf00258a3
--- /dev/null
+++ b/.sqlx/query-27dba71899c33b28e6f3daf31030b8e9f609ee7e6b0a62bba74ac168ac76f8cd.json
@@ -0,0 +1,23 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "SELECT count(*) AS \"count!\" FROM notification n\n WHERE n.event_item_type = 'document' AND n.event_item_id = ANY($1)\n AND n.notification_event_type = ANY($2)\n AND (n.metadata->>'commentId' ~ '^[0-9]+$' OR n.metadata->>'threadId' ~ '^[0-9]+$')",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "count!",
+ "type_info": "Int8"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "TextArray",
+ "TextArray"
+ ]
+ },
+ "nullable": [
+ null
+ ]
+ },
+ "hash": "27dba71899c33b28e6f3daf31030b8e9f609ee7e6b0a62bba74ac168ac76f8cd"
+}
diff --git a/.sqlx/query-3d92901e27e9ca5ff9f78b9f09e02c76b9e44ee49efec6c4afc9ea620cb27f2f.json b/.sqlx/query-3d92901e27e9ca5ff9f78b9f09e02c76b9e44ee49efec6c4afc9ea620cb27f2f.json
new file mode 100644
index 0000000000..7fbf5c1431
--- /dev/null
+++ b/.sqlx/query-3d92901e27e9ca5ff9f78b9f09e02c76b9e44ee49efec6c4afc9ea620cb27f2f.json
@@ -0,0 +1,35 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "SELECT m.id, i.\"documentId\" AS document_id, m.\"modificationData\" AS payload\n FROM \"DocumentInstanceModificationData\" m\n JOIN \"DocumentInstance\" i ON i.id = m.\"documentInstanceId\"\n JOIN \"Document\" d ON d.id = i.\"documentId\"\n WHERE i.\"documentId\" = ANY($1) AND d.\"fileType\" = 'pdf'\n AND jsonb_path_exists(m.\"modificationData\", $2::text::jsonpath)\n ORDER BY m.id",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "id",
+ "type_info": "Int8"
+ },
+ {
+ "ordinal": 1,
+ "name": "document_id",
+ "type_info": "Text"
+ },
+ {
+ "ordinal": 2,
+ "name": "payload",
+ "type_info": "Jsonb"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "TextArray",
+ "Text"
+ ]
+ },
+ "nullable": [
+ false,
+ false,
+ false
+ ]
+ },
+ "hash": "3d92901e27e9ca5ff9f78b9f09e02c76b9e44ee49efec6c4afc9ea620cb27f2f"
+}
diff --git a/.sqlx/query-4027823662c88d28cb2a95792b615423ead933fb94bbd3dad5e6e5e7ff59ff53.json b/.sqlx/query-4027823662c88d28cb2a95792b615423ead933fb94bbd3dad5e6e5e7ff59ff53.json
new file mode 100644
index 0000000000..733b7ba987
--- /dev/null
+++ b/.sqlx/query-4027823662c88d28cb2a95792b615423ead933fb94bbd3dad5e6e5e7ff59ff53.json
@@ -0,0 +1,22 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "SELECT pg_advisory_unlock($1) AS \"released!\"",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "released!",
+ "type_info": "Bool"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Int8"
+ ]
+ },
+ "nullable": [
+ null
+ ]
+ },
+ "hash": "4027823662c88d28cb2a95792b615423ead933fb94bbd3dad5e6e5e7ff59ff53"
+}
diff --git a/.sqlx/query-48fbddcebd915244c16d53a6f2c8b68ad4ffd6735cd44817ee2cec34aa6bbe9c.json b/.sqlx/query-48fbddcebd915244c16d53a6f2c8b68ad4ffd6735cd44817ee2cec34aa6bbe9c.json
new file mode 100644
index 0000000000..f94c37e726
--- /dev/null
+++ b/.sqlx/query-48fbddcebd915244c16d53a6f2c8b68ad4ffd6735cd44817ee2cec34aa6bbe9c.json
@@ -0,0 +1,22 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "SELECT t.id FROM \"Thread\" t\n LEFT JOIN \"PdfPlaceableCommentAnchor\" p ON p.\"threadId\" = t.id AND NOT p.\"wasDeleted\"\n LEFT JOIN \"PdfHighlightAnchor\" h ON h.\"threadId\" = t.id AND h.\"deletedAt\" IS NULL\n WHERE t.\"deletedAt\" IS NULL\n GROUP BY t.id HAVING count(DISTINCT p.uuid) + count(DISTINCT h.uuid) > 1\n ORDER BY t.id LIMIT $1",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "id",
+ "type_info": "Int8"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Int8"
+ ]
+ },
+ "nullable": [
+ false
+ ]
+ },
+ "hash": "48fbddcebd915244c16d53a6f2c8b68ad4ffd6735cd44817ee2cec34aa6bbe9c"
+}
diff --git a/.sqlx/query-576c0d1c1c043d4418c8f137138c5a0cd749cc32100c3d95c0cc6792e697e8a9.json b/.sqlx/query-576c0d1c1c043d4418c8f137138c5a0cd749cc32100c3d95c0cc6792e697e8a9.json
new file mode 100644
index 0000000000..38304c51cb
--- /dev/null
+++ b/.sqlx/query-576c0d1c1c043d4418c8f137138c5a0cd749cc32100c3d95c0cc6792e697e8a9.json
@@ -0,0 +1,22 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "SELECT count(*) AS \"count!\" FROM \"Thread\" t\n JOIN migrated_comment_thread_id tm ON tm.thread_id = t.id\n JOIN LATERAL (\n SELECT cm.message_id FROM \"Comment\" c\n JOIN migrated_comment_id cm ON cm.comment_id = c.id\n WHERE c.\"threadId\" = t.id\n ORDER BY c.\"order\" NULLS LAST, c.\"createdAt\", c.id LIMIT 1\n ) first ON true\n WHERE t.\"documentId\" = ANY($1) AND first.message_id <> tm.root_id",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "count!",
+ "type_info": "Int8"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "TextArray"
+ ]
+ },
+ "nullable": [
+ null
+ ]
+ },
+ "hash": "576c0d1c1c043d4418c8f137138c5a0cd749cc32100c3d95c0cc6792e697e8a9"
+}
diff --git a/.sqlx/query-624ac87b24d542ae21dabeb58d067643329cd92029864abda17d5376dc999739.json b/.sqlx/query-624ac87b24d542ae21dabeb58d067643329cd92029864abda17d5376dc999739.json
new file mode 100644
index 0000000000..a0d957a6c9
--- /dev/null
+++ b/.sqlx/query-624ac87b24d542ae21dabeb58d067643329cd92029864abda17d5376dc999739.json
@@ -0,0 +1,15 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "UPDATE \"DocumentInstanceModificationData\" SET \"modificationData\" = $1 WHERE id = $2",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Jsonb",
+ "Int8"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "624ac87b24d542ae21dabeb58d067643329cd92029864abda17d5376dc999739"
+}
diff --git a/.sqlx/query-72c6260a4c0cdfb61d8273040c5d79d6c615c3e1dcbb4bbfabc9a61e2cead4cd.json b/.sqlx/query-72c6260a4c0cdfb61d8273040c5d79d6c615c3e1dcbb4bbfabc9a61e2cead4cd.json
new file mode 100644
index 0000000000..57c0fe1328
--- /dev/null
+++ b/.sqlx/query-72c6260a4c0cdfb61d8273040c5d79d6c615c3e1dcbb4bbfabc9a61e2cead4cd.json
@@ -0,0 +1,20 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "SELECT count(*) AS \"count!\" FROM \"Comment\" c\n WHERE NOT EXISTS (SELECT 1 FROM migrated_comment_id m WHERE m.comment_id = c.id)",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "count!",
+ "type_info": "Int8"
+ }
+ ],
+ "parameters": {
+ "Left": []
+ },
+ "nullable": [
+ null
+ ]
+ },
+ "hash": "72c6260a4c0cdfb61d8273040c5d79d6c615c3e1dcbb4bbfabc9a61e2cead4cd"
+}
diff --git a/.sqlx/query-74275afbf7e9f3348efbbd7d8c7eb565faee20148c38b8ca42582c02628af0e7.json b/.sqlx/query-74275afbf7e9f3348efbbd7d8c7eb565faee20148c38b8ca42582c02628af0e7.json
new file mode 100644
index 0000000000..c2e34c72c1
--- /dev/null
+++ b/.sqlx/query-74275afbf7e9f3348efbbd7d8c7eb565faee20148c38b8ca42582c02628af0e7.json
@@ -0,0 +1,16 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "INSERT INTO migrated_comment_id (comment_id, message_id, document_id)\n SELECT * FROM UNNEST($1::bigint[], $2::uuid[], $3::text[])",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Int8Array",
+ "UuidArray",
+ "TextArray"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "74275afbf7e9f3348efbbd7d8c7eb565faee20148c38b8ca42582c02628af0e7"
+}
diff --git a/.sqlx/query-74842f5645a90d4bdefd392a080062648d83a046b36d7d83ed963c3842657a41.json b/.sqlx/query-74842f5645a90d4bdefd392a080062648d83a046b36d7d83ed963c3842657a41.json
new file mode 100644
index 0000000000..27ff206084
--- /dev/null
+++ b/.sqlx/query-74842f5645a90d4bdefd392a080062648d83a046b36d7d83ed963c3842657a41.json
@@ -0,0 +1,35 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "SELECT t.\"documentId\" AS document_id, t.metadata->>'markId' AS \"mark_id!\",\n array_agg(t.id ORDER BY t.id) AS \"thread_ids!\"\n FROM \"Thread\" t\n WHERE t.\"deletedAt\" IS NULL\n AND t.metadata->>'markId' ~ $1\n AND EXISTS (SELECT 1 FROM \"Comment\" c WHERE c.\"threadId\" = t.id)\n AND NOT EXISTS (SELECT 1 FROM \"PdfPlaceableCommentAnchor\" p WHERE p.\"threadId\" = t.id AND NOT p.\"wasDeleted\")\n AND NOT EXISTS (SELECT 1 FROM \"PdfHighlightAnchor\" h\n WHERE h.\"threadId\" = t.id AND h.\"deletedAt\" IS NULL)\n GROUP BY 1, 2 HAVING count(*) > 1\n ORDER BY 1, 2 LIMIT $2",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "document_id",
+ "type_info": "Text"
+ },
+ {
+ "ordinal": 1,
+ "name": "mark_id!",
+ "type_info": "Text"
+ },
+ {
+ "ordinal": 2,
+ "name": "thread_ids!",
+ "type_info": "Int8Array"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Text",
+ "Int8"
+ ]
+ },
+ "nullable": [
+ false,
+ null,
+ null
+ ]
+ },
+ "hash": "74842f5645a90d4bdefd392a080062648d83a046b36d7d83ed963c3842657a41"
+}
diff --git a/.sqlx/query-76f3e71eaea836800a3e5857a00621edab609f4eaba0ce9a4da11743cc2b1482.json b/.sqlx/query-76f3e71eaea836800a3e5857a00621edab609f4eaba0ce9a4da11743cc2b1482.json
new file mode 100644
index 0000000000..fb2702be6c
--- /dev/null
+++ b/.sqlx/query-76f3e71eaea836800a3e5857a00621edab609f4eaba0ce9a4da11743cc2b1482.json
@@ -0,0 +1,15 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "UPDATE notification n SET metadata = jsonb_set(\n jsonb_set(n.metadata, '{commentId}', to_jsonb(cm.message_id::text)),\n '{threadId}', to_jsonb(tm.root_id::text)\n )\n FROM migrated_comment_id cm, migrated_comment_thread_id tm\n WHERE n.event_item_type = 'document' AND n.event_item_id = ANY($1)\n AND n.notification_event_type = ANY($2)\n AND cm.document_id = n.event_item_id AND tm.document_id = n.event_item_id\n AND n.metadata->>'commentId' = cm.comment_id::text\n AND n.metadata->>'threadId' = tm.thread_id::text",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "TextArray",
+ "TextArray"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "76f3e71eaea836800a3e5857a00621edab609f4eaba0ce9a4da11743cc2b1482"
+}
diff --git a/.sqlx/query-7a590cadb532aa035907f3c2889166dd7c5cea5d61ab9e7fedf9f8a5186295e7.json b/.sqlx/query-7a590cadb532aa035907f3c2889166dd7c5cea5d61ab9e7fedf9f8a5186295e7.json
new file mode 100644
index 0000000000..3ef0c29241
--- /dev/null
+++ b/.sqlx/query-7a590cadb532aa035907f3c2889166dd7c5cea5d61ab9e7fedf9f8a5186295e7.json
@@ -0,0 +1,14 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "UPDATE \"PdfHighlightAnchor\" a SET root_id = tm.root_id\n FROM migrated_comment_thread_id tm\n WHERE a.\"threadId\" = tm.thread_id AND a.\"documentId\" = ANY($1)\n AND a.root_id IS DISTINCT FROM tm.root_id",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "TextArray"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "7a590cadb532aa035907f3c2889166dd7c5cea5d61ab9e7fedf9f8a5186295e7"
+}
diff --git a/.sqlx/query-8da419734f41296de7dd848d4b2659623a2e31379ba795b68a366b2d6439a516.json b/.sqlx/query-8da419734f41296de7dd848d4b2659623a2e31379ba795b68a366b2d6439a516.json
new file mode 100644
index 0000000000..c7196a52c7
--- /dev/null
+++ b/.sqlx/query-8da419734f41296de7dd848d4b2659623a2e31379ba795b68a366b2d6439a516.json
@@ -0,0 +1,22 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "SELECT pg_try_advisory_lock($1) AS \"acquired!\"",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "acquired!",
+ "type_info": "Bool"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Int8"
+ ]
+ },
+ "nullable": [
+ null
+ ]
+ },
+ "hash": "8da419734f41296de7dd848d4b2659623a2e31379ba795b68a366b2d6439a516"
+}
diff --git a/.sqlx/query-99086fa5c096f71daeb129691382932436bf3ce8a810b8a868655f399e0fc3f5.json b/.sqlx/query-99086fa5c096f71daeb129691382932436bf3ce8a810b8a868655f399e0fc3f5.json
new file mode 100644
index 0000000000..baa475902d
--- /dev/null
+++ b/.sqlx/query-99086fa5c096f71daeb129691382932436bf3ce8a810b8a868655f399e0fc3f5.json
@@ -0,0 +1,22 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "UPDATE comms_message_threads s SET deleted_at = now(), updated_at = now()\n FROM migrated_comment_thread_id tm\n WHERE tm.root_id = s.root_id AND tm.document_id = ANY($1) AND s.deleted_at IS NULL\n AND NOT EXISTS (SELECT 1 FROM \"Thread\" t WHERE t.id = tm.thread_id)\n RETURNING s.root_id",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "root_id",
+ "type_info": "Uuid"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "TextArray"
+ ]
+ },
+ "nullable": [
+ false
+ ]
+ },
+ "hash": "99086fa5c096f71daeb129691382932436bf3ce8a810b8a868655f399e0fc3f5"
+}
diff --git a/.sqlx/query-a65e5f3ae0b0506fb5204aa83ea49fa70745c460e53ec316be184a309f492754.json b/.sqlx/query-a65e5f3ae0b0506fb5204aa83ea49fa70745c460e53ec316be184a309f492754.json
new file mode 100644
index 0000000000..102b4ff269
--- /dev/null
+++ b/.sqlx/query-a65e5f3ae0b0506fb5204aa83ea49fa70745c460e53ec316be184a309f492754.json
@@ -0,0 +1,14 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "UPDATE comms_messages SET deleted_at = now(), updated_at = now()\n WHERE (id = ANY($1) OR thread_id = ANY($1)) AND deleted_at IS NULL",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "UuidArray"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "a65e5f3ae0b0506fb5204aa83ea49fa70745c460e53ec316be184a309f492754"
+}
diff --git a/.sqlx/query-a6fe5e93cc6b7ed25934216cfa647b8591213be9dc848d874b3b3f2a78226669.json b/.sqlx/query-a6fe5e93cc6b7ed25934216cfa647b8591213be9dc848d874b3b3f2a78226669.json
new file mode 100644
index 0000000000..5dd86cd9e5
--- /dev/null
+++ b/.sqlx/query-a6fe5e93cc6b7ed25934216cfa647b8591213be9dc848d874b3b3f2a78226669.json
@@ -0,0 +1,22 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "INSERT INTO comms_messages (\n id, parent_entity_type, parent_entity_id, thread_id, sender_id, imported_author,\n content, created_at, updated_at, edited_at, deleted_at, import_metadata, import_order\n )\n SELECT cm.message_id, 'document', t.\"documentId\",\n CASE WHEN cm.message_id = tm.root_id THEN NULL ELSE tm.root_id END,\n c.owner, c.sender, c.text,\n c.\"createdAt\" AT TIME ZONE 'UTC', c.\"updatedAt\" AT TIME ZONE 'UTC',\n CASE WHEN c.\"updatedAt\" > c.\"createdAt\" THEN c.\"updatedAt\" END,\n COALESCE(c.\"deletedAt\", t.\"deletedAt\"), c.metadata,\n row_number() OVER (PARTITION BY t.id ORDER BY c.\"order\" NULLS LAST, c.\"createdAt\", c.id)\n FROM \"Comment\" c\n JOIN \"Thread\" t ON t.id = c.\"threadId\"\n JOIN migrated_comment_id cm ON cm.comment_id = c.id\n JOIN migrated_comment_thread_id tm ON tm.thread_id = t.id\n WHERE t.\"documentId\" = ANY($1)\n ON CONFLICT (id) DO UPDATE SET\n content = EXCLUDED.content,\n updated_at = EXCLUDED.updated_at,\n edited_at = EXCLUDED.edited_at,\n deleted_at = EXCLUDED.deleted_at,\n imported_author = EXCLUDED.imported_author,\n import_metadata = EXCLUDED.import_metadata,\n import_order = EXCLUDED.import_order\n WHERE comms_messages.updated_at <= EXCLUDED.updated_at\n AND (comms_messages.content, comms_messages.updated_at, comms_messages.edited_at,\n comms_messages.deleted_at, comms_messages.imported_author,\n comms_messages.import_metadata, comms_messages.import_order)\n IS DISTINCT FROM\n (EXCLUDED.content, EXCLUDED.updated_at, EXCLUDED.edited_at, EXCLUDED.deleted_at,\n EXCLUDED.imported_author, EXCLUDED.import_metadata, EXCLUDED.import_order)\n RETURNING (xmax = 0) AS \"inserted!\"",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "inserted!",
+ "type_info": "Bool"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "TextArray"
+ ]
+ },
+ "nullable": [
+ null
+ ]
+ },
+ "hash": "a6fe5e93cc6b7ed25934216cfa647b8591213be9dc848d874b3b3f2a78226669"
+}
diff --git a/.sqlx/query-b44f233a1a834c1ae81dbfda9c248a7b7daf2813c73521fbb8691d4f34ca28ea.json b/.sqlx/query-b44f233a1a834c1ae81dbfda9c248a7b7daf2813c73521fbb8691d4f34ca28ea.json
new file mode 100644
index 0000000000..bae7d68881
--- /dev/null
+++ b/.sqlx/query-b44f233a1a834c1ae81dbfda9c248a7b7daf2813c73521fbb8691d4f34ca28ea.json
@@ -0,0 +1,20 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "SELECT count(*) AS \"count!\" FROM \"Thread\" t\n WHERE NOT EXISTS (SELECT 1 FROM migrated_comment_thread_id m WHERE m.thread_id = t.id)",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "count!",
+ "type_info": "Int8"
+ }
+ ],
+ "parameters": {
+ "Left": []
+ },
+ "nullable": [
+ null
+ ]
+ },
+ "hash": "b44f233a1a834c1ae81dbfda9c248a7b7daf2813c73521fbb8691d4f34ca28ea"
+}
diff --git a/.sqlx/query-bc996dcd91b86d30e494387bacf11fc307ac363e4d1701439b0bc6ad59e29d16.json b/.sqlx/query-bc996dcd91b86d30e494387bacf11fc307ac363e4d1701439b0bc6ad59e29d16.json
new file mode 100644
index 0000000000..05bb70f569
--- /dev/null
+++ b/.sqlx/query-bc996dcd91b86d30e494387bacf11fc307ac363e4d1701439b0bc6ad59e29d16.json
@@ -0,0 +1,23 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "SELECT d.id FROM \"Document\" d\n WHERE ($1::text IS NULL OR d.id > $1)\n AND (EXISTS (SELECT 1 FROM \"Thread\" t WHERE t.\"documentId\" = d.id)\n OR EXISTS (SELECT 1 FROM migrated_comment_thread_id m WHERE m.document_id = d.id))\n ORDER BY d.id LIMIT $2",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "id",
+ "type_info": "Text"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Text",
+ "Int8"
+ ]
+ },
+ "nullable": [
+ false
+ ]
+ },
+ "hash": "bc996dcd91b86d30e494387bacf11fc307ac363e4d1701439b0bc6ad59e29d16"
+}
diff --git a/.sqlx/query-c32423d596dca1eb14bef93e22d6b2a37d52cfbf3cf650d1965df042056ad9ce.json b/.sqlx/query-c32423d596dca1eb14bef93e22d6b2a37d52cfbf3cf650d1965df042056ad9ce.json
new file mode 100644
index 0000000000..62ee47591e
--- /dev/null
+++ b/.sqlx/query-c32423d596dca1eb14bef93e22d6b2a37d52cfbf3cf650d1965df042056ad9ce.json
@@ -0,0 +1,24 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "SELECT count(*) AS \"count!\" FROM \"Thread\" t\n WHERE t.\"documentId\" = ANY($1) AND t.\"deletedAt\" IS NULL\n AND t.metadata->>'markId' IS NOT NULL\n AND t.metadata->>'markId' !~ $2\n AND t.metadata->>'markId' NOT LIKE $3\n AND NOT EXISTS (SELECT 1 FROM \"PdfPlaceableCommentAnchor\" p WHERE p.\"threadId\" = t.id AND NOT p.\"wasDeleted\")\n AND NOT EXISTS (SELECT 1 FROM \"PdfHighlightAnchor\" h\n WHERE h.\"threadId\" = t.id AND h.\"deletedAt\" IS NULL)",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "count!",
+ "type_info": "Int8"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "TextArray",
+ "Text",
+ "Text"
+ ]
+ },
+ "nullable": [
+ null
+ ]
+ },
+ "hash": "c32423d596dca1eb14bef93e22d6b2a37d52cfbf3cf650d1965df042056ad9ce"
+}
diff --git a/.sqlx/query-c5e01d489c1b1316ab38953bf81a99b5d98b5f73d5a4ce47c5499d9bf958ce6e.json b/.sqlx/query-c5e01d489c1b1316ab38953bf81a99b5d98b5f73d5a4ce47c5499d9bf958ce6e.json
new file mode 100644
index 0000000000..e0b1610c7f
--- /dev/null
+++ b/.sqlx/query-c5e01d489c1b1316ab38953bf81a99b5d98b5f73d5a4ce47c5499d9bf958ce6e.json
@@ -0,0 +1,16 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "INSERT INTO comms_message_threads (\n root_id, parent_entity_type, parent_entity_id, user_id, resolved, anchor,\n import_metadata, created_at, updated_at, deleted_at\n )\n SELECT tm.root_id, 'document', t.\"documentId\", t.owner, t.resolved,\n CASE\n WHEN pa.uuid IS NOT NULL\n THEN jsonb_build_object('type', 'pdf_placeable', 'anchor_id', pa.uuid)\n WHEN ph.uuid IS NOT NULL\n THEN jsonb_build_object('type', 'pdf_highlight', 'anchor_id', ph.uuid)\n WHEN t.metadata->>'markId' ~ $3\n THEN jsonb_build_object('type', 'markdown', 'mark_id', (t.metadata->>'markId')::uuid)\n END,\n t.metadata, t.\"createdAt\" AT TIME ZONE 'UTC', t.\"updatedAt\" AT TIME ZONE 'UTC',\n CASE\n WHEN t.\"deletedAt\" IS NOT NULL THEN t.\"deletedAt\" AT TIME ZONE 'UTC'\n WHEN NOT EXISTS (SELECT 1 FROM \"Comment\" c WHERE c.\"threadId\" = t.id)\n THEN t.\"updatedAt\" AT TIME ZONE 'UTC'\n END\n FROM \"Thread\" t\n JOIN migrated_comment_thread_id tm ON tm.thread_id = t.id\n LEFT JOIN LATERAL (\n SELECT a.uuid FROM \"PdfPlaceableCommentAnchor\" a\n WHERE a.\"threadId\" = t.id AND NOT a.\"wasDeleted\"\n ORDER BY a.uuid LIMIT 1\n ) pa ON true\n LEFT JOIN LATERAL (\n SELECT a.uuid FROM \"PdfHighlightAnchor\" a\n WHERE a.\"threadId\" = t.id AND a.\"deletedAt\" IS NULL\n ORDER BY a.uuid LIMIT 1\n ) ph ON true\n WHERE t.\"documentId\" = ANY($1)\n ON CONFLICT (root_id) DO UPDATE SET\n user_id = EXCLUDED.user_id,\n resolved = EXCLUDED.resolved,\n anchor = EXCLUDED.anchor,\n import_metadata = EXCLUDED.import_metadata,\n created_at = EXCLUDED.created_at,\n updated_at = EXCLUDED.updated_at,\n deleted_at = EXCLUDED.deleted_at\n WHERE (comms_message_threads.root_id = ANY($2)\n OR comms_message_threads.updated_at <= EXCLUDED.updated_at)\n AND (comms_message_threads.user_id, comms_message_threads.resolved,\n comms_message_threads.anchor, comms_message_threads.import_metadata,\n comms_message_threads.created_at, comms_message_threads.updated_at,\n comms_message_threads.deleted_at)\n IS DISTINCT FROM\n (EXCLUDED.user_id, EXCLUDED.resolved, EXCLUDED.anchor, EXCLUDED.import_metadata,\n EXCLUDED.created_at, EXCLUDED.updated_at, EXCLUDED.deleted_at)",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "TextArray",
+ "UuidArray",
+ "Text"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "c5e01d489c1b1316ab38953bf81a99b5d98b5f73d5a4ce47c5499d9bf958ce6e"
+}
diff --git a/.sqlx/query-cad533fdf9dff2cb374956bb810b6672179e9e5afc7bc6e059b3ba14f4cd0dd0.json b/.sqlx/query-cad533fdf9dff2cb374956bb810b6672179e9e5afc7bc6e059b3ba14f4cd0dd0.json
new file mode 100644
index 0000000000..63e2c7761a
--- /dev/null
+++ b/.sqlx/query-cad533fdf9dff2cb374956bb810b6672179e9e5afc7bc6e059b3ba14f4cd0dd0.json
@@ -0,0 +1,34 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "SELECT t.id, t.\"documentId\" AS document_id, first.message_id AS \"root_id?\"\n FROM \"Thread\" t\n LEFT JOIN LATERAL (\n SELECT m.message_id FROM \"Comment\" c\n JOIN migrated_comment_id m ON m.comment_id = c.id\n WHERE c.\"threadId\" = t.id\n ORDER BY c.\"order\" NULLS LAST, c.\"createdAt\", c.id LIMIT 1\n ) first ON true\n WHERE t.\"documentId\" = ANY($1)\n AND NOT EXISTS (SELECT 1 FROM migrated_comment_thread_id m WHERE m.thread_id = t.id)\n ORDER BY t.id",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "id",
+ "type_info": "Int8"
+ },
+ {
+ "ordinal": 1,
+ "name": "document_id",
+ "type_info": "Text"
+ },
+ {
+ "ordinal": 2,
+ "name": "root_id?",
+ "type_info": "Uuid"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "TextArray"
+ ]
+ },
+ "nullable": [
+ false,
+ false,
+ false
+ ]
+ },
+ "hash": "cad533fdf9dff2cb374956bb810b6672179e9e5afc7bc6e059b3ba14f4cd0dd0"
+}
diff --git a/.sqlx/query-e558ac1741835623b34dea5518c43f58913909a51b4f5f6ef057f8cccdd8eab7.json b/.sqlx/query-e558ac1741835623b34dea5518c43f58913909a51b4f5f6ef057f8cccdd8eab7.json
new file mode 100644
index 0000000000..c831efb930
--- /dev/null
+++ b/.sqlx/query-e558ac1741835623b34dea5518c43f58913909a51b4f5f6ef057f8cccdd8eab7.json
@@ -0,0 +1,28 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "SELECT c.id, t.\"documentId\" AS document_id FROM \"Comment\" c\n JOIN \"Thread\" t ON t.id = c.\"threadId\"\n WHERE t.\"documentId\" = ANY($1)\n AND NOT EXISTS (SELECT 1 FROM migrated_comment_id m WHERE m.comment_id = c.id)\n ORDER BY c.id",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "id",
+ "type_info": "Int8"
+ },
+ {
+ "ordinal": 1,
+ "name": "document_id",
+ "type_info": "Text"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "TextArray"
+ ]
+ },
+ "nullable": [
+ false,
+ false
+ ]
+ },
+ "hash": "e558ac1741835623b34dea5518c43f58913909a51b4f5f6ef057f8cccdd8eab7"
+}
diff --git a/.sqlx/query-e9f33d6edca87d88c7403b13d19fe3aa835c20b17993e9e8cdabfe20ed5efa34.json b/.sqlx/query-e9f33d6edca87d88c7403b13d19fe3aa835c20b17993e9e8cdabfe20ed5efa34.json
new file mode 100644
index 0000000000..d4071cbc7a
--- /dev/null
+++ b/.sqlx/query-e9f33d6edca87d88c7403b13d19fe3aa835c20b17993e9e8cdabfe20ed5efa34.json
@@ -0,0 +1,14 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "UPDATE comms_messages m SET deleted_at = now(), updated_at = now()\n FROM migrated_comment_id cm\n WHERE cm.message_id = m.id AND cm.document_id = ANY($1) AND m.deleted_at IS NULL\n AND NOT EXISTS (SELECT 1 FROM \"Comment\" c WHERE c.id = cm.comment_id)",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "TextArray"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "e9f33d6edca87d88c7403b13d19fe3aa835c20b17993e9e8cdabfe20ed5efa34"
+}
diff --git a/.sqlx/query-f4387247669703803329aa0d090c0b6427a7b17676d1fdf8fdfd950dbf685dd0.json b/.sqlx/query-f4387247669703803329aa0d090c0b6427a7b17676d1fdf8fdfd950dbf685dd0.json
new file mode 100644
index 0000000000..e0e7f4d8d0
--- /dev/null
+++ b/.sqlx/query-f4387247669703803329aa0d090c0b6427a7b17676d1fdf8fdfd950dbf685dd0.json
@@ -0,0 +1,20 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "SELECT count(DISTINCT \"documentId\") AS \"count!\" FROM \"Thread\"",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "count!",
+ "type_info": "Int8"
+ }
+ ],
+ "parameters": {
+ "Left": []
+ },
+ "nullable": [
+ null
+ ]
+ },
+ "hash": "f4387247669703803329aa0d090c0b6427a7b17676d1fdf8fdfd950dbf685dd0"
+}
diff --git a/Cargo.lock b/Cargo.lock
index c35d080dfa..e53cb6ac5e 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -10134,7 +10134,17 @@ dependencies = [
name = "macro_db_migrator"
version = "0.1.0"
dependencies = [
+ "chrono",
+ "clap",
+ "database_env_vars",
+ "macro_entrypoint",
+ "macro_env",
+ "macro_uuid",
+ "rootcause",
+ "serde_json",
"sqlx",
+ "tokio",
+ "tracing",
"workspace-hack",
]
diff --git a/crates/macro_db_client/migrations/20260917190902_messages_mapping_foreign_keys.sql b/crates/macro_db_client/migrations/20260917190902_messages_mapping_foreign_keys.sql
new file mode 100644
index 0000000000..770566ff01
--- /dev/null
+++ b/crates/macro_db_client/migrations/20260917190902_messages_mapping_foreign_keys.sql
@@ -0,0 +1,20 @@
+-- The legacy comment importer allocates a mapping row and writes the message
+-- it points at inside the same transaction, so these references hold at every
+-- commit boundary. Deferred so the mapping can be inserted before the message.
+ALTER TABLE migrated_comment_id
+ ADD CONSTRAINT migrated_comment_id_message_id_fkey
+ FOREIGN KEY (message_id) REFERENCES comms_messages (id)
+ ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
+
+ALTER TABLE migrated_comment_thread_id
+ ADD CONSTRAINT migrated_comment_thread_id_root_id_fkey
+ FOREIGN KEY (root_id) REFERENCES comms_message_threads (root_id)
+ ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
+
+-- The importer works one batch of documents at a time and the Document FK
+-- cascades by document; neither had an index on the referencing column.
+CREATE INDEX idx_migrated_comment_id_document
+ ON migrated_comment_id (document_id);
+
+CREATE INDEX idx_migrated_comment_thread_id_document
+ ON migrated_comment_thread_id (document_id);
diff --git a/crates/macro_db_migrator/Cargo.toml b/crates/macro_db_migrator/Cargo.toml
index 15efd1049b..6d827198e5 100644
--- a/crates/macro_db_migrator/Cargo.toml
+++ b/crates/macro_db_migrator/Cargo.toml
@@ -4,6 +4,35 @@ name = "macro_db_migrator"
publish = false
version = "0.1.0"
+[features]
+comment-import = [
+ "dep:clap",
+ "dep:database_env_vars",
+ "dep:macro_entrypoint",
+ "dep:macro_env",
+ "dep:macro_uuid",
+ "dep:rootcause",
+ "dep:serde_json",
+ "dep:tokio",
+ "dep:tracing",
+]
+
+[[bin]]
+name = "comment_import"
+required-features = ["comment-import"]
+
[dependencies]
+clap = { workspace = true, optional = true }
+database_env_vars = { path = "../database_env_vars", optional = true }
+macro_entrypoint = { path = "../macro_entrypoint", optional = true }
+macro_env = { path = "../macro_env", optional = true }
+macro_uuid = { path = "../macro_uuid", optional = true }
+rootcause = { workspace = true, optional = true }
+serde_json = { workspace = true, optional = true }
sqlx = { workspace = true, features = ["migrate"] }
+tokio = { workspace = true, features = ["macros", "rt-multi-thread", "time"], optional = true }
+tracing = { workspace = true, optional = true }
workspace-hack = { version = "0.1", path = "../workspace-hack" }
+
+[dev-dependencies]
+chrono = { workspace = true }
diff --git a/crates/macro_db_migrator/src/bin/comment_import.rs b/crates/macro_db_migrator/src/bin/comment_import.rs
new file mode 100644
index 0000000000..6496b370b1
--- /dev/null
+++ b/crates/macro_db_migrator/src/bin/comment_import.rs
@@ -0,0 +1,45 @@
+//! Online, idempotent import of legacy document comments into the shared
+//! message store. Safe to rerun while the application keeps serving requests.
+
+#[path = "comment_import/runner.rs"]
+mod runner;
+
+use clap::Parser;
+use database_env_vars::DatabaseUrl;
+use macro_entrypoint::MacroEntrypoint;
+use macro_env::Environment;
+use sqlx::{Connection, PgConnection};
+
+#[derive(Debug, Parser)]
+#[command(
+ about = "Import legacy document comments into comms_messages while the application stays live"
+)]
+struct Args {
+ /// Documents imported per transaction. Smaller batches hold the legacy
+ /// table lock for less time; larger batches finish sooner.
+ #[arg(long, default_value_t = runner::DEFAULT_DOCUMENTS_PER_BATCH)]
+ documents_per_batch: i64,
+ /// Run the preflight and report pending work without writing anything.
+ #[arg(long)]
+ check: bool,
+}
+
+#[tokio::main]
+async fn main() -> Result<(), rootcause::Report> {
+ let args = Args::parse();
+ let entrypoint = MacroEntrypoint::new(Environment::Local).init();
+ let database_url = DatabaseUrl::new()?;
+ let mut connection = PgConnection::connect(database_url.as_ref()).await?;
+ let report = if args.check {
+ runner::check(&mut connection).await?.to_string()
+ } else {
+ let options = runner::Options {
+ documents_per_batch: args.documents_per_batch,
+ };
+ runner::run(&mut connection, options).await?.to_string()
+ };
+ println!("{report}");
+ connection.close().await?;
+ entrypoint.shutdown();
+ Ok(())
+}
diff --git a/crates/macro_db_migrator/src/bin/comment_import/legacy_fixture.sql b/crates/macro_db_migrator/src/bin/comment_import/legacy_fixture.sql
new file mode 100644
index 0000000000..d404905602
--- /dev/null
+++ b/crates/macro_db_migrator/src/bin/comment_import/legacy_fixture.sql
@@ -0,0 +1,46 @@
+INSERT INTO macro_user(id, username, email, stripe_customer_id)
+VALUES ('01990000-0000-7000-8000-000000000001', 'migration', '[email protected]', 'migration');
+INSERT INTO "User"(id, email, macro_user_id)
+VALUES ('macro|[email protected]', '[email protected]', '01990000-0000-7000-8000-000000000001');
+INSERT INTO comms_channels(id, name, channel_type, owner_id)
+VALUES ('01990000-0000-7000-8000-000000000005', 'Existing channel', 'private', 'macro|[email protected]');
+INSERT INTO comms_messages(id, channel_id, thread_id, sender_id, content) VALUES
+ ('01990000-0000-7000-8000-000000000006', '01990000-0000-7000-8000-000000000005', null, 'macro|[email protected]', 'Channel root'),
+ ('01990000-0000-7000-8000-000000000007', '01990000-0000-7000-8000-000000000005', '01990000-0000-7000-8000-000000000006', 'macro|[email protected]', 'Channel reply');
+INSERT INTO "Document"(id, name, owner, "fileType") VALUES
+ ('legacy-md', 'Legacy', 'macro|[email protected]', 'md'),
+ ('legacy-pdf', 'Legacy PDF', 'macro|[email protected]', 'pdf');
+INSERT INTO "Thread"(id, owner, "documentId", resolved, metadata, "createdAt", "updatedAt") VALUES
+ (1, 'macro|[email protected]', 'legacy-md', true, '{"markId":"01990000-0000-7000-8000-000000000002"}', '2020-01-01', '2020-01-02'),
+ (2, 'macro|[email protected]', 'legacy-md', false, '{"markId":"DISCUSSION:legacy"}', '2020-01-05', '2020-01-06'),
+ (3, 'macro|[email protected]', 'legacy-pdf', false, '{}', '2020-01-01', '2020-01-01'),
+ (4, 'macro|[email protected]', 'legacy-pdf', false, '{}', '2020-01-01', '2020-01-01'),
+ (5, 'macro|[email protected]', 'legacy-md', false, '{"markId":"DISCUSSION:01990000-0000-7000-8000-000000000009"}', '2020-01-07', '2020-01-07');
+INSERT INTO "Comment"(id, "threadId", owner, sender, text, "order", "createdAt", "updatedAt", "deletedAt") VALUES
+ (10, 1, 'macro|[email protected]', 'Original author', 'Deleted root', 1, '2020-01-01', '2020-01-01', '2020-01-03'),
+ (11, 1, 'macro|[email protected]', null, 'Surviving reply', 2, '2020-01-02', '2020-01-02', null),
+ (12, 3, 'macro|[email protected]', 'PDF Author', 'PDF original', 1, '2020-01-01', '2020-01-01', null),
+ (13, 4, 'macro|[email protected]', null, 'Highlight comment', 1, '2020-01-01', '2020-01-01', null),
+ (14, 5, 'macro|[email protected]', null, 'Discussion root', null, '2020-01-07', '2020-01-08', null);
+INSERT INTO "PdfPlaceableCommentAnchor"(
+ uuid, "documentId", owner, page, "wasEdited", "wasDeleted", "shouldLockOnSave",
+ "originalPage", "originalIndex", "xPct", "yPct", "widthPct", "heightPct", rotation, "threadId"
+) VALUES ('01990000-0000-7000-8000-000000000003', 'legacy-pdf', 'macro|[email protected]',
+ 2, false, false, false, 2, 0, 0.1, 0.2, 0.3, 0.4, 0, 3);
+INSERT INTO "PdfHighlightAnchor"(
+ uuid, "documentId", owner, page, red, green, blue, alpha, type, text,
+ "pageViewportWidth", "pageViewportHeight", "threadId"
+) VALUES ('01990000-0000-7000-8000-000000000004', 'legacy-pdf', 'macro|[email protected]',
+ 1, 255, 255, 0, 0.5, 0, 'Highlighted text', 600, 800, 4);
+INSERT INTO "PdfHighlightRect"(top, "left", width, height, "pdfHighlightAnchorId")
+VALUES (0.1, 0.2, 0.3, 0.4, '01990000-0000-7000-8000-000000000004');
+INSERT INTO "DocumentInstance"(id, "documentId", sha) VALUES (1, 'legacy-pdf', 'fixture');
+INSERT INTO "DocumentInstanceModificationData"("documentInstanceId", "modificationData")
+VALUES (1, '{"pages":[{"highlights":[{"comments":[{"id":12,"text":"PDF original"},{"id":"999","text":"Stale"},{"id":"external-pdf-id","text":"External"}]}]}]}');
+INSERT INTO notification(id, notification_event_type, event_item_id, event_item_type, service_sender, metadata)
+SELECT gen_random_uuid(), kind, 'legacy-md', 'document', 'dss',
+ '{"commentId":11,"threadId":1,"commentText":"Surviving reply"}'::jsonb
+FROM unnest(ARRAY['commented_on_document', 'replied_to_document_comment_thread', 'mentioned_in_document_comment']) kind;
+INSERT INTO notification(id, notification_event_type, event_item_id, event_item_type, service_sender, metadata)
+VALUES (gen_random_uuid(), 'commented_on_document', 'legacy-md', 'document', 'dss',
+ '{"commentId":424242,"threadId":1,"commentText":"Removed long ago"}'::jsonb);
diff --git a/crates/macro_db_migrator/src/bin/comment_import/runner.rs b/crates/macro_db_migrator/src/bin/comment_import/runner.rs
new file mode 100644
index 0000000000..9b7c11394c
--- /dev/null
+++ b/crates/macro_db_migrator/src/bin/comment_import/runner.rs
@@ -0,0 +1,781 @@
+//! Database-only operator tooling. No application request or authorization path.
+//!
+//! Every run projects the legacy `"Comment"` / `"Thread"` rows onto the shared
+//! message tables one batch of documents at a time. Rows whose projection
+//! already matches are left alone, so a rerun is a no-op and a run after new
+//! legacy writes carries only the delta.
+
+#[path = "summary.rs"]
+mod summary;
+
+use std::collections::HashMap;
+use std::time::Duration;
+
+use macro_uuid::{Uuid, generate_uuid_v7};
+use sqlx::{Connection, PgConnection, Postgres, Transaction};
+pub(super) use summary::{Pending, Summary};
+
+const LOCK_ID: i64 = 732_847_301;
+/// Documents handled per transaction when the operator gives no override.
+pub(super) const DEFAULT_DOCUMENTS_PER_BATCH: i64 = 250;
+const MAPPING_CHUNK: usize = 1_000;
+const LOCK_RETRIES: u32 = 5;
+const LOCK_RETRY_DELAY: Duration = Duration::from_secs(2);
+const LOCK_NOT_AVAILABLE: &str = "55P03";
+const DEADLOCK_DETECTED: &str = "40P01";
+const PREFLIGHT_SAMPLE: i64 = 20;
+const UUID_PATTERN: &str =
+ "^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$";
+const DISCUSSION_MARK_PREFIX: &str = "DISCUSSION:";
+const COMMENT_NOTIFICATION_TYPES: [&str; 3] = [
+ "commented_on_document",
+ "replied_to_document_comment_thread",
+ "mentioned_in_document_comment",
+];
+/// Saved PDF export payloads whose `comments[].id` still carries a legacy id.
+const NUMERIC_COMMENT_ID_PATH: &str =
+ r#"$.**.comments[*].id ? (@.type() == "number" || @ like_regex "^[0-9]+$")"#;
+
+#[derive(Debug, Clone, Copy)]
+pub(super) struct Options {
+ pub documents_per_batch: i64,
+}
+
+impl Default for Options {
+ fn default() -> Self {
+ Self {
+ documents_per_batch: DEFAULT_DOCUMENTS_PER_BATCH,
+ }
+ }
+}
+
+/// The caller owns this dedicated connection; dropping it also releases the lock.
+pub(super) async fn run(
+ connection: &mut PgConnection,
+ options: Options,
+) -> Result<Summary, rootcause::Report> {
+ if options.documents_per_batch < 1 {
+ rootcause::bail!("documents_per_batch must be at least 1");
+ }
+ with_lock(connection, |connection| {
+ Box::pin(run_locked(connection, options))
+ })
+ .await
+}
+
+/// Preflight and pending counts; never writes.
+pub(super) async fn check(connection: &mut PgConnection) -> Result<Pending, rootcause::Report> {
+ with_lock(connection, |connection| Box::pin(check_locked(connection))).await
+}
+
+type Locked<'a, T> =
+ std::pin::Pin<Box<dyn Future<Output = Result<T, rootcause::Report>> + Send + 'a>>;
+
+async fn with_lock<T>(
+ connection: &mut PgConnection,
+ body: impl for<'a> FnOnce(&'a mut PgConnection) -> Locked<'a, T>,
+) -> Result<T, rootcause::Report> {
+ let acquired =
+ sqlx::query_scalar!(r#"SELECT pg_try_advisory_lock($1) AS "acquired!""#, LOCK_ID)
+ .fetch_one(&mut *connection)
+ .await?;
+ if !acquired {
+ rootcause::bail!("Another comment import is running");
+ }
+ let result = body(connection).await;
+ let unlock = sqlx::query_scalar!(r#"SELECT pg_advisory_unlock($1) AS "released!""#, LOCK_ID)
+ .fetch_one(&mut *connection)
+ .await;
+ let outcome = result?;
+ unlock?;
+ Ok(outcome)
+}
+
+async fn check_locked(connection: &mut PgConnection) -> Result<Pending, rootcause::Report> {
+ preflight(connection).await?;
+ let comments_without_mapping = sqlx::query_scalar!(
+ r#"SELECT count(*) AS "count!" FROM "Comment" c
+ WHERE NOT EXISTS (SELECT 1 FROM migrated_comment_id m WHERE m.comment_id = c.id)"#
+ )
+ .fetch_one(&mut *connection)
+ .await?;
+ let threads_without_mapping = sqlx::query_scalar!(
+ r#"SELECT count(*) AS "count!" FROM "Thread" t
+ WHERE NOT EXISTS (SELECT 1 FROM migrated_comment_thread_id m WHERE m.thread_id = t.id)"#
+ )
+ .fetch_one(&mut *connection)
+ .await?;
+ let documents_with_legacy_comments =
+ sqlx::query_scalar!(r#"SELECT count(DISTINCT "documentId") AS "count!" FROM "Thread""#)
+ .fetch_one(&mut *connection)
+ .await?;
+ Ok(Pending {
+ comments_without_mapping,
+ threads_without_mapping,
+ documents_with_legacy_comments,
+ })
+}
+
+async fn run_locked(
+ connection: &mut PgConnection,
+ options: Options,
+) -> Result<Summary, rootcause::Report> {
+ preflight(connection).await?;
+ let mut summary = Summary::default();
+ let mut cursor: Option<String> = None;
+ loop {
+ let documents = sqlx::query_scalar!(
+ r#"SELECT d.id FROM "Document" d
+ WHERE ($1::text IS NULL OR d.id > $1)
+ AND (EXISTS (SELECT 1 FROM "Thread" t WHERE t."documentId" = d.id)
+ OR EXISTS (SELECT 1 FROM migrated_comment_thread_id m WHERE m.document_id = d.id))
+ ORDER BY d.id LIMIT $2"#,
+ cursor,
+ options.documents_per_batch,
+ )
+ .fetch_all(&mut *connection)
+ .await?;
+ let Some(last) = documents.last() else { break };
+ cursor = Some(last.clone());
+ import_batch_with_retry(connection, &documents, &mut summary).await?;
+ summary.documents += documents.len() as u64;
+ summary.batches += 1;
+ }
+ Ok(summary)
+}
+
+/// Legacy data that cannot be represented and must be resolved by hand first.
+async fn preflight(connection: &mut PgConnection) -> Result<(), rootcause::Report> {
+ let ambiguous = sqlx::query_scalar!(
+ r#"SELECT t.id FROM "Thread" t
+ LEFT JOIN "PdfPlaceableCommentAnchor" p ON p."threadId" = t.id AND NOT p."wasDeleted"
+ LEFT JOIN "PdfHighlightAnchor" h ON h."threadId" = t.id AND h."deletedAt" IS NULL
+ WHERE t."deletedAt" IS NULL
+ GROUP BY t.id HAVING count(DISTINCT p.uuid) + count(DISTINCT h.uuid) > 1
+ ORDER BY t.id LIMIT $1"#,
+ PREFLIGHT_SAMPLE,
+ )
+ .fetch_all(&mut *connection)
+ .await?;
+ if !ambiguous.is_empty() {
+ rootcause::bail!(
+ "Multiple PDF anchors on a legacy thread; resolve before importing (thread ids {ambiguous:?})"
+ );
+ }
+ let duplicates = sqlx::query!(
+ r#"SELECT t."documentId" AS document_id, t.metadata->>'markId' AS "mark_id!",
+ array_agg(t.id ORDER BY t.id) AS "thread_ids!"
+ FROM "Thread" t
+ WHERE t."deletedAt" IS NULL
+ AND t.metadata->>'markId' ~ $1
+ AND EXISTS (SELECT 1 FROM "Comment" c WHERE c."threadId" = t.id)
+ AND NOT EXISTS (SELECT 1 FROM "PdfPlaceableCommentAnchor" p WHERE p."threadId" = t.id AND NOT p."wasDeleted")
+ AND NOT EXISTS (SELECT 1 FROM "PdfHighlightAnchor" h
+ WHERE h."threadId" = t.id AND h."deletedAt" IS NULL)
+ GROUP BY 1, 2 HAVING count(*) > 1
+ ORDER BY 1, 2 LIMIT $2"#,
+ UUID_PATTERN,
+ PREFLIGHT_SAMPLE,
+ )
+ .fetch_all(&mut *connection)
+ .await?;
+ if let Some(first) = duplicates.first() {
+ rootcause::bail!(
+ "Multiple active legacy threads share the same Markdown mark; delete the duplicates before importing (document {} mark {} threads {:?}, {} conflicts sampled)",
+ first.document_id,
+ first.mark_id,
+ first.thread_ids,
+ duplicates.len()
+ );
+ }
+ Ok(())
+}
+
+async fn import_batch_with_retry(
+ connection: &mut PgConnection,
+ documents: &[String],
+ summary: &mut Summary,
+) -> Result<(), rootcause::Report> {
+ let mut attempt = 0;
+ loop {
+ match import_batch(connection, documents, summary).await {
+ Ok(()) => return Ok(()),
+ Err(error) if attempt < LOCK_RETRIES && is_retryable(&error) => {
+ attempt += 1;
+ tracing::warn!(
+ error = %error,
+ attempt,
+ documents = documents.len(),
+ "legacy tables busy; retrying batch"
+ );
+ tokio::time::sleep(LOCK_RETRY_DELAY).await;
+ }
+ Err(error) => return Err(error.into()),
+ }
+ }
+}
+
+fn is_retryable(error: &sqlx::Error) -> bool {
+ error
+ .as_database_error()
+ .and_then(|e| e.code())
+ .is_some_and(|code| code == LOCK_NOT_AVAILABLE || code == DEADLOCK_DETECTED)
+}
+
+/// One transaction per batch: mapping allocation and the rows those mappings
+/// point at commit together, so the mapping foreign keys hold at every boundary.
+/// Counts join the run summary only once the batch has committed, so a retried
+/// batch is counted once.
+async fn import_batch(
+ connection: &mut PgConnection,
+ documents: &[String],
+ summary: &mut Summary,
+) -> Result<(), sqlx::Error> {
+ let mut tx = connection.begin().await?;
+ let mut batch = Summary::default();
+ lock_legacy_tables(&mut tx).await?;
+ batch.comment_mappings_allocated = allocate_comment_mappings(&mut tx, documents).await?;
+ let new_roots = allocate_thread_mappings(&mut tx, documents).await?;
+ batch.thread_mappings_allocated = new_roots.len() as u64;
+
+ let (inserted, updated) = upsert_messages(&mut tx, documents).await?;
+ batch.messages_inserted += inserted;
+ batch.messages_updated += updated;
+ let (inserted, updated) = upsert_structural_roots(&mut tx, documents).await?;
+ batch.messages_inserted += inserted;
+ batch.messages_updated += updated;
+ batch.messages_tombstoned += tombstone_orphan_messages(&mut tx, documents).await?;
+
+ batch.threads_written = upsert_thread_state(&mut tx, documents, &new_roots).await?;
+ let (threads, messages) = tombstone_orphan_threads(&mut tx, documents).await?;
+ batch.threads_tombstoned = threads;
+ batch.messages_tombstoned += messages;
+
+ batch.anchors_linked = link_anchors(&mut tx, documents).await?;
+ let (remapped, unmapped) = remap_notifications(&mut tx, documents).await?;
+ batch.notifications_remapped = remapped;
+ batch.warnings.unmapped_notifications = unmapped;
+ let (remapped, unmapped) = remap_pdf_payloads(&mut tx, documents).await?;
+ batch.pdf_payloads_remapped = remapped;
+ batch.warnings.unmapped_pdf_comment_ids = unmapped;
+
+ batch.warnings.invalid_mark_ids = count_invalid_mark_ids(&mut tx, documents).await?;
+ batch.warnings.root_order_drift = count_root_order_drift(&mut tx, documents).await?;
+ tx.commit().await?;
+ summary.absorb(batch);
+ Ok(())
+}
+
+#[expect(
+ clippy::disallowed_methods,
+ reason = "utility statements carry no result shape for SQLx to check"
+)]
+async fn lock_legacy_tables(tx: &mut Transaction<'_, Postgres>) -> Result<(), sqlx::Error> {
+ sqlx::query("SET LOCAL lock_timeout = '5s'")
+ .execute(&mut **tx)
+ .await?;
+ sqlx::query(r#"LOCK TABLE "Comment", "Thread" IN SHARE ROW EXCLUSIVE MODE"#)
+ .execute(&mut **tx)
+ .await?;
+ // The 5s timeout bounds only the table-lock wait. Clear it so later batch
+ // statements wait normally for unrelated row locks instead of aborting.
+ sqlx::query("SET LOCAL lock_timeout = 0")
+ .execute(&mut **tx)
+ .await?;
+ Ok(())
+}
+
+async fn allocate_comment_mappings(
+ tx: &mut Transaction<'_, Postgres>,
+ documents: &[String],
+) -> Result<u64, sqlx::Error> {
+ let missing = sqlx::query!(
+ r#"SELECT c.id, t."documentId" AS document_id FROM "Comment" c
+ JOIN "Thread" t ON t.id = c."threadId"
+ WHERE t."documentId" = ANY($1)
+ AND NOT EXISTS (SELECT 1 FROM migrated_comment_id m WHERE m.comment_id = c.id)
+ ORDER BY c.id"#,
+ documents,
+ )
+ .fetch_all(&mut **tx)
+ .await?;
+ for chunk in missing.chunks(MAPPING_CHUNK) {
+ let ids: Vec<i64> = chunk.iter().map(|row| row.id).collect();
+ let uuids: Vec<Uuid> = chunk.iter().map(|_| generate_uuid_v7()).collect();
+ let document_ids: Vec<String> = chunk.iter().map(|row| row.document_id.clone()).collect();
+ sqlx::query!(
+ "INSERT INTO migrated_comment_id (comment_id, message_id, document_id)
+ SELECT * FROM UNNEST($1::bigint[], $2::uuid[], $3::text[])",
+ &ids,
+ &uuids,
+ &document_ids,
+ )
+ .execute(&mut **tx)
+ .await?;
+ }
+ Ok(missing.len() as u64)
+}
+
+/// Roots follow legacy display order, then creation time, then id, including
+/// deleted comments. Empty threads receive a fresh root id for a structural tombstone.
+async fn allocate_thread_mappings(
+ tx: &mut Transaction<'_, Postgres>,
+ documents: &[String],
+) -> Result<Vec<Uuid>, sqlx::Error> {
+ let missing = sqlx::query!(
+ r#"SELECT t.id, t."documentId" AS document_id, first.message_id AS "root_id?"
+ FROM "Thread" t
+ LEFT JOIN LATERAL (
+ SELECT m.message_id FROM "Comment" c
+ JOIN migrated_comment_id m ON m.comment_id = c.id
+ WHERE c."threadId" = t.id
+ ORDER BY c."order" NULLS LAST, c."createdAt", c.id LIMIT 1
+ ) first ON true
+ WHERE t."documentId" = ANY($1)
+ AND NOT EXISTS (SELECT 1 FROM migrated_comment_thread_id m WHERE m.thread_id = t.id)
+ ORDER BY t.id"#,
+ documents,
+ )
+ .fetch_all(&mut **tx)
+ .await?;
+ let mut roots = Vec::with_capacity(missing.len());
+ for chunk in missing.chunks(MAPPING_CHUNK) {
+ let ids: Vec<i64> = chunk.iter().map(|row| row.id).collect();
+ let chunk_roots: Vec<Uuid> = chunk
+ .iter()
+ .map(|row| row.root_id.unwrap_or_else(generate_uuid_v7))
+ .collect();
+ let document_ids: Vec<String> = chunk.iter().map(|row| row.document_id.clone()).collect();
+ sqlx::query!(
+ "INSERT INTO migrated_comment_thread_id (thread_id, root_id, document_id)
+ SELECT * FROM UNNEST($1::bigint[], $2::uuid[], $3::text[])",
+ &ids,
+ &chunk_roots,
+ &document_ids,
+ )
+ .execute(&mut **tx)
+ .await?;
+ roots.extend(chunk_roots);
+ }
+ Ok(roots)
+}
+
+fn count_inserts(rows: &[bool]) -> (u64, u64) {
+ let inserted = rows.iter().filter(|inserted| **inserted).count() as u64;
+ (inserted, rows.len() as u64 - inserted)
+}
+
+/// Every mapped comment. Existing rows change only when the legacy row differs
+/// and is not older than the stored row, so an edit made through the new API
+/// after a legacy row went stale is never reverted.
+async fn upsert_messages(
+ tx: &mut Transaction<'_, Postgres>,
+ documents: &[String],
+) -> Result<(u64, u64), sqlx::Error> {
+ let rows = sqlx::query_scalar!(
+ r#"INSERT INTO comms_messages (
+ id, parent_entity_type, parent_entity_id, thread_id, sender_id, imported_author,
+ content, created_at, updated_at, edited_at, deleted_at, import_metadata, import_order
+ )
+ SELECT cm.message_id, 'document', t."documentId",
+ CASE WHEN cm.message_id = tm.root_id THEN NULL ELSE tm.root_id END,
+ c.owner, c.sender, c.text,
+ c."createdAt" AT TIME ZONE 'UTC', c."updatedAt" AT TIME ZONE 'UTC',
+ CASE WHEN c."updatedAt" > c."createdAt" THEN c."updatedAt" END,
+ COALESCE(c."deletedAt", t."deletedAt"), c.metadata,
+ row_number() OVER (PARTITION BY t.id ORDER BY c."order" NULLS LAST, c."createdAt", c.id)
+ FROM "Comment" c
+ JOIN "Thread" t ON t.id = c."threadId"
+ JOIN migrated_comment_id cm ON cm.comment_id = c.id
+ JOIN migrated_comment_thread_id tm ON tm.thread_id = t.id
+ WHERE t."documentId" = ANY($1)
+ ON CONFLICT (id) DO UPDATE SET
+ content = EXCLUDED.content,
+ updated_at = EXCLUDED.updated_at,
+ edited_at = EXCLUDED.edited_at,
+ deleted_at = EXCLUDED.deleted_at,
+ imported_author = EXCLUDED.imported_author,
+ import_metadata = EXCLUDED.import_metadata,
+ import_order = EXCLUDED.import_order
+ WHERE comms_messages.updated_at <= EXCLUDED.updated_at
+ AND (comms_messages.content, comms_messages.updated_at, comms_messages.edited_at,
+ comms_messages.deleted_at, comms_messages.imported_author,
+ comms_messages.import_metadata, comms_messages.import_order)
+ IS DISTINCT FROM
+ (EXCLUDED.content, EXCLUDED.updated_at, EXCLUDED.edited_at, EXCLUDED.deleted_at,
+ EXCLUDED.imported_author, EXCLUDED.import_metadata, EXCLUDED.import_order)
+ RETURNING (xmax = 0) AS "inserted!""#,
+ documents,
+ )
+ .fetch_all(&mut **tx)
+ .await?;
+ Ok(count_inserts(&rows))
+}
+
+/// Empty legacy threads keep a deleted root so their identity, anchors, and
+/// external links stay representable without rendering as a discussion.
+async fn upsert_structural_roots(
+ tx: &mut Transaction<'_, Postgres>,
+ documents: &[String],
+) -> Result<(u64, u64), sqlx::Error> {
+ let rows = sqlx::query_scalar!(
+ r#"INSERT INTO comms_messages (
+ id, parent_entity_type, parent_entity_id, sender_id, content,
+ created_at, updated_at, deleted_at
+ )
+ SELECT tm.root_id, 'document', t."documentId", t.owner, '',
+ t."createdAt" AT TIME ZONE 'UTC', t."updatedAt" AT TIME ZONE 'UTC',
+ COALESCE(t."deletedAt", t."updatedAt")
+ FROM "Thread" t
+ JOIN migrated_comment_thread_id tm ON tm.thread_id = t.id
+ WHERE t."documentId" = ANY($1)
+ AND NOT EXISTS (SELECT 1 FROM "Comment" c WHERE c."threadId" = t.id)
+ ON CONFLICT (id) DO UPDATE SET
+ updated_at = EXCLUDED.updated_at,
+ deleted_at = EXCLUDED.deleted_at
+ WHERE comms_messages.updated_at <= EXCLUDED.updated_at
+ AND (comms_messages.updated_at, comms_messages.deleted_at)
+ IS DISTINCT FROM (EXCLUDED.updated_at, EXCLUDED.deleted_at)
+ RETURNING (xmax = 0) AS "inserted!""#,
+ documents,
+ )
+ .fetch_all(&mut **tx)
+ .await?;
+ Ok(count_inserts(&rows))
+}
+
+/// A mapped comment whose legacy row was removed outright reads as deleted.
+async fn tombstone_orphan_messages(
+ tx: &mut Transaction<'_, Postgres>,
+ documents: &[String],
+) -> Result<u64, sqlx::Error> {
+ let result = sqlx::query!(
+ r#"UPDATE comms_messages m SET deleted_at = now(), updated_at = now()
+ FROM migrated_comment_id cm
+ WHERE cm.message_id = m.id AND cm.document_id = ANY($1) AND m.deleted_at IS NULL
+ AND NOT EXISTS (SELECT 1 FROM "Comment" c WHERE c.id = cm.comment_id)"#,
+ documents,
+ )
+ .execute(&mut **tx)
+ .await?;
+ Ok(result.rows_affected())
+}
+
+/// Thread state for every mapped thread. The root insert trigger has already
+/// created a placeholder row for roots written in this batch; those are always
+/// overwritten. Older rows follow the same newer-wins rule as messages.
+async fn upsert_thread_state(
+ tx: &mut Transaction<'_, Postgres>,
+ documents: &[String],
+ new_roots: &[Uuid],
+) -> Result<u64, sqlx::Error> {
+ let result = sqlx::query!(
+ r#"INSERT INTO comms_message_threads (
+ root_id, parent_entity_type, parent_entity_id, user_id, resolved, anchor,
+ import_metadata, created_at, updated_at, deleted_at
+ )
+ SELECT tm.root_id, 'document', t."documentId", t.owner, t.resolved,
+ CASE
+ WHEN pa.uuid IS NOT NULL
+ THEN jsonb_build_object('type', 'pdf_placeable', 'anchor_id', pa.uuid)
+ WHEN ph.uuid IS NOT NULL
+ THEN jsonb_build_object('type', 'pdf_highlight', 'anchor_id', ph.uuid)
+ WHEN t.metadata->>'markId' ~ $3
+ THEN jsonb_build_object('type', 'markdown', 'mark_id', (t.metadata->>'markId')::uuid)
+ END,
+ t.metadata, t."createdAt" AT TIME ZONE 'UTC', t."updatedAt" AT TIME ZONE 'UTC',
+ CASE
+ WHEN t."deletedAt" IS NOT NULL THEN t."deletedAt" AT TIME ZONE 'UTC'
+ WHEN NOT EXISTS (SELECT 1 FROM "Comment" c WHERE c."threadId" = t.id)
+ THEN t."updatedAt" AT TIME ZONE 'UTC'
+ END
+ FROM "Thread" t
+ JOIN migrated_comment_thread_id tm ON tm.thread_id = t.id
+ LEFT JOIN LATERAL (
+ SELECT a.uuid FROM "PdfPlaceableCommentAnchor" a
+ WHERE a."threadId" = t.id AND NOT a."wasDeleted"
+ ORDER BY a.uuid LIMIT 1
+ ) pa ON true
+ LEFT JOIN LATERAL (
+ SELECT a.uuid FROM "PdfHighlightAnchor" a
+ WHERE a."threadId" = t.id AND a."deletedAt" IS NULL
+ ORDER BY a.uuid LIMIT 1
+ ) ph ON true
+ WHERE t."documentId" = ANY($1)
+ ON CONFLICT (root_id) DO UPDATE SET
+ user_id = EXCLUDED.user_id,
+ resolved = EXCLUDED.resolved,
+ anchor = EXCLUDED.anchor,
+ import_metadata = EXCLUDED.import_metadata,
+ created_at = EXCLUDED.created_at,
+ updated_at = EXCLUDED.updated_at,
+ deleted_at = EXCLUDED.deleted_at
+ WHERE (comms_message_threads.root_id = ANY($2)
+ OR comms_message_threads.updated_at <= EXCLUDED.updated_at)
+ AND (comms_message_threads.user_id, comms_message_threads.resolved,
+ comms_message_threads.anchor, comms_message_threads.import_metadata,
+ comms_message_threads.created_at, comms_message_threads.updated_at,
+ comms_message_threads.deleted_at)
+ IS DISTINCT FROM
+ (EXCLUDED.user_id, EXCLUDED.resolved, EXCLUDED.anchor, EXCLUDED.import_metadata,
+ EXCLUDED.created_at, EXCLUDED.updated_at, EXCLUDED.deleted_at)"#,
+ documents,
+ new_roots,
+ UUID_PATTERN,
+ )
+ .execute(&mut **tx)
+ .await?;
+ Ok(result.rows_affected())
+}
+
+/// A mapped thread whose legacy row was removed outright reads as deleted, with its messages.
+async fn tombstone_orphan_threads(
+ tx: &mut Transaction<'_, Postgres>,
+ documents: &[String],
+) -> Result<(u64, u64), sqlx::Error> {
+ let roots = sqlx::query_scalar!(
+ r#"UPDATE comms_message_threads s SET deleted_at = now(), updated_at = now()
+ FROM migrated_comment_thread_id tm
+ WHERE tm.root_id = s.root_id AND tm.document_id = ANY($1) AND s.deleted_at IS NULL
+ AND NOT EXISTS (SELECT 1 FROM "Thread" t WHERE t.id = tm.thread_id)
+ RETURNING s.root_id"#,
+ documents,
+ )
+ .fetch_all(&mut **tx)
+ .await?;
+ if roots.is_empty() {
+ return Ok((0, 0));
+ }
+ let messages = sqlx::query!(
+ r#"UPDATE comms_messages SET deleted_at = now(), updated_at = now()
+ WHERE (id = ANY($1) OR thread_id = ANY($1)) AND deleted_at IS NULL"#,
+ &roots,
+ )
+ .execute(&mut **tx)
+ .await?;
+ Ok((roots.len() as u64, messages.rows_affected()))
+}
+
+async fn link_anchors(
+ tx: &mut Transaction<'_, Postgres>,
+ documents: &[String],
+) -> Result<u64, sqlx::Error> {
+ let placeables = sqlx::query!(
+ r#"UPDATE "PdfPlaceableCommentAnchor" a SET root_id = tm.root_id
+ FROM migrated_comment_thread_id tm
+ WHERE a."threadId" = tm.thread_id AND a."documentId" = ANY($1)
+ AND a.root_id IS DISTINCT FROM tm.root_id"#,
+ documents,
+ )
+ .execute(&mut **tx)
+ .await?;
+ let highlights = sqlx::query!(
+ r#"UPDATE "PdfHighlightAnchor" a SET root_id = tm.root_id
+ FROM migrated_comment_thread_id tm
+ WHERE a."threadId" = tm.thread_id AND a."documentId" = ANY($1)
+ AND a.root_id IS DISTINCT FROM tm.root_id"#,
+ documents,
+ )
+ .execute(&mut **tx)
+ .await?;
+ Ok(placeables.rows_affected() + highlights.rows_affected())
+}
+
+/// Historical document comment notifications point at message ids once both
+/// the comment and its thread have mappings. Rewritten ids never match again.
+async fn remap_notifications(
+ tx: &mut Transaction<'_, Postgres>,
+ documents: &[String],
+) -> Result<(u64, u64), sqlx::Error> {
+ let types: Vec<String> = COMMENT_NOTIFICATION_TYPES
+ .iter()
+ .map(|kind| (*kind).to_owned())
+ .collect();
+ let remapped = sqlx::query!(
+ r#"UPDATE notification n SET metadata = jsonb_set(
+ jsonb_set(n.metadata, '{commentId}', to_jsonb(cm.message_id::text)),
+ '{threadId}', to_jsonb(tm.root_id::text)
+ )
+ FROM migrated_comment_id cm, migrated_comment_thread_id tm
+ WHERE n.event_item_type = 'document' AND n.event_item_id = ANY($1)
+ AND n.notification_event_type = ANY($2)
+ AND cm.document_id = n.event_item_id AND tm.document_id = n.event_item_id
+ AND n.metadata->>'commentId' = cm.comment_id::text
+ AND n.metadata->>'threadId' = tm.thread_id::text"#,
+ documents,
+ &types,
+ )
+ .execute(&mut **tx)
+ .await?;
+ let unmapped = sqlx::query_scalar!(
+ r#"SELECT count(*) AS "count!" FROM notification n
+ WHERE n.event_item_type = 'document' AND n.event_item_id = ANY($1)
+ AND n.notification_event_type = ANY($2)
+ AND (n.metadata->>'commentId' ~ '^[0-9]+$' OR n.metadata->>'threadId' ~ '^[0-9]+$')"#,
+ documents,
+ &types,
+ )
+ .fetch_one(&mut **tx)
+ .await?;
+ Ok((remapped.rows_affected(), unmapped as u64))
+}
+
+/// Saved PDF export payloads are relational metadata; their comment ids follow
+/// the mapping. Ids without a mapping stay untouched and are counted.
+async fn remap_pdf_payloads(
+ tx: &mut Transaction<'_, Postgres>,
+ documents: &[String],
+) -> Result<(u64, u64), sqlx::Error> {
+ let payloads = sqlx::query!(
+ r#"SELECT m.id, i."documentId" AS document_id, m."modificationData" AS payload
+ FROM "DocumentInstanceModificationData" m
+ JOIN "DocumentInstance" i ON i.id = m."documentInstanceId"
+ JOIN "Document" d ON d.id = i."documentId"
+ WHERE i."documentId" = ANY($1) AND d."fileType" = 'pdf'
+ AND jsonb_path_exists(m."modificationData", $2::text::jsonpath)
+ ORDER BY m.id"#,
+ documents,
+ NUMERIC_COMMENT_ID_PATH,
+ )
+ .fetch_all(&mut **tx)
+ .await?;
+ let mut remapped = 0;
+ let mut unmapped = 0;
+ let mut mappings: HashMap<String, HashMap<i64, Uuid>> = HashMap::new();
+ for row in payloads {
+ if !mappings.contains_key(&row.document_id) {
+ let rows = sqlx::query!(
+ "SELECT comment_id, message_id FROM migrated_comment_id WHERE document_id = $1",
+ row.document_id,
+ )
+ .fetch_all(&mut **tx)
+ .await?;
+ mappings.insert(
+ row.document_id.clone(),
+ rows.into_iter()
+ .map(|mapping| (mapping.comment_id, mapping.message_id))
+ .collect(),
+ );
+ }
+ let mut payload = row.payload;
+ let stats = remap_comment_ids(&mut payload, &mappings[&row.document_id]);
+ unmapped += stats.unmapped;
+ if stats.remapped > 0 {
+ sqlx::query!(
+ r#"UPDATE "DocumentInstanceModificationData" SET "modificationData" = $1 WHERE id = $2"#,
+ payload,
+ row.id,
+ )
+ .execute(&mut **tx)
+ .await?;
+ remapped += 1;
+ }
+ }
+ Ok((remapped, unmapped))
+}
+
+#[derive(Debug, Default, PartialEq, Eq)]
+struct RemapStats {
+ remapped: u64,
+ unmapped: u64,
+}
+
+/// Rewrite every `comments[].id` holding a legacy comment id, at any depth.
+fn remap_comment_ids(value: &mut serde_json::Value, mappings: &HashMap<i64, Uuid>) -> RemapStats {
+ let mut stats = RemapStats::default();
+ match value {
+ serde_json::Value::Array(items) => {
+ for item in items {
+ let inner = remap_comment_ids(item, mappings);
+ stats.remapped += inner.remapped;
+ stats.unmapped += inner.unmapped;
+ }
+ }
+ serde_json::Value::Object(fields) => {
+ for (key, child) in fields.iter_mut() {
+ if key == "comments" && child.is_array() {
+ for comment in child.as_array_mut().into_iter().flatten() {
+ if let Some(id) = comment.get("id").and_then(legacy_comment_id) {
+ match mappings.get(&id) {
+ Some(message_id) => {
+ comment["id"] =
+ serde_json::Value::String(message_id.to_string());
+ stats.remapped += 1;
+ }
+ None => stats.unmapped += 1,
+ }
+ }
+ // A comments array may nest inside a comment object.
+ let inner = remap_comment_ids(comment, mappings);
+ stats.remapped += inner.remapped;
+ stats.unmapped += inner.unmapped;
+ }
+ } else {
+ let inner = remap_comment_ids(child, mappings);
+ stats.remapped += inner.remapped;
+ stats.unmapped += inner.unmapped;
+ }
+ }
+ }
+ _ => {}
+ }
+ stats
+}
+
+fn legacy_comment_id(id: &serde_json::Value) -> Option<i64> {
+ match id {
+ serde_json::Value::Number(number) => number.as_i64(),
+ serde_json::Value::String(text) if text.bytes().all(|byte| byte.is_ascii_digit()) => {
+ text.parse().ok()
+ }
+ _ => None,
+ }
+}
+
+async fn count_invalid_mark_ids(
+ tx: &mut Transaction<'_, Postgres>,
+ documents: &[String],
+) -> Result<u64, sqlx::Error> {
+ let count = sqlx::query_scalar!(
+ r#"SELECT count(*) AS "count!" FROM "Thread" t
+ WHERE t."documentId" = ANY($1) AND t."deletedAt" IS NULL
+ AND t.metadata->>'markId' IS NOT NULL
+ AND t.metadata->>'markId' !~ $2
+ AND t.metadata->>'markId' NOT LIKE $3
+ AND NOT EXISTS (SELECT 1 FROM "PdfPlaceableCommentAnchor" p WHERE p."threadId" = t.id AND NOT p."wasDeleted")
+ AND NOT EXISTS (SELECT 1 FROM "PdfHighlightAnchor" h
+ WHERE h."threadId" = t.id AND h."deletedAt" IS NULL)"#,
+ documents,
+ UUID_PATTERN,
+ format!("{DISCUSSION_MARK_PREFIX}%"),
+ )
+ .fetch_one(&mut **tx)
+ .await?;
+ Ok(count as u64)
+}
+
+/// Roots are fixed at first import; a legacy reorder afterwards is reported, not applied.
+async fn count_root_order_drift(
+ tx: &mut Transaction<'_, Postgres>,
+ documents: &[String],
+) -> Result<u64, sqlx::Error> {
+ let count = sqlx::query_scalar!(
+ r#"SELECT count(*) AS "count!" FROM "Thread" t
+ JOIN migrated_comment_thread_id tm ON tm.thread_id = t.id
+ JOIN LATERAL (
+ SELECT cm.message_id FROM "Comment" c
+ JOIN migrated_comment_id cm ON cm.comment_id = c.id
+ WHERE c."threadId" = t.id
+ ORDER BY c."order" NULLS LAST, c."createdAt", c.id LIMIT 1
+ ) first ON true
+ WHERE t."documentId" = ANY($1) AND first.message_id <> tm.root_id"#,
+ documents,
+ )
+ .fetch_one(&mut **tx)
+ .await?;
+ Ok(count as u64)
+}
+
+#[cfg(test)]
+#[path = "test.rs"]
+mod test;
diff --git a/crates/macro_db_migrator/src/bin/comment_import/summary.rs b/crates/macro_db_migrator/src/bin/comment_import/summary.rs
new file mode 100644
index 0000000000..4e1bedd7b8
--- /dev/null
+++ b/crates/macro_db_migrator/src/bin/comment_import/summary.rs
@@ -0,0 +1,189 @@
+//! Run reports printed to the operator.
+
+use std::fmt;
+
+/// What one run wrote. Every counter is zero when the store already matched.
+#[derive(Debug, Default, Clone, PartialEq, Eq)]
+pub struct Summary {
+ pub documents: u64,
+ pub batches: u64,
+ pub comment_mappings_allocated: u64,
+ pub thread_mappings_allocated: u64,
+ pub messages_inserted: u64,
+ pub messages_updated: u64,
+ pub messages_tombstoned: u64,
+ pub threads_written: u64,
+ pub threads_tombstoned: u64,
+ pub anchors_linked: u64,
+ pub notifications_remapped: u64,
+ pub pdf_payloads_remapped: u64,
+ pub warnings: Warnings,
+}
+
+/// Legacy data the import could not represent exactly. Nothing here aborts a run.
+#[derive(Debug, Default, Clone, PartialEq, Eq)]
+pub struct Warnings {
+ /// Document comment notifications still pointing at a legacy id without a mapping.
+ pub unmapped_notifications: u64,
+ /// Numeric comment ids in saved PDF payloads without a mapping.
+ pub unmapped_pdf_comment_ids: u64,
+ /// Threads whose mark id is neither a UUID nor a `DISCUSSION:` mark; imported unanchored.
+ pub invalid_mark_ids: u64,
+ /// Threads whose first legacy comment is no longer the mapped root.
+ pub root_order_drift: u64,
+}
+
+impl Summary {
+ /// Add a committed batch to the run.
+ pub fn absorb(&mut self, batch: Self) {
+ self.documents += batch.documents;
+ self.batches += batch.batches;
+ self.comment_mappings_allocated += batch.comment_mappings_allocated;
+ self.thread_mappings_allocated += batch.thread_mappings_allocated;
+ self.messages_inserted += batch.messages_inserted;
+ self.messages_updated += batch.messages_updated;
+ self.messages_tombstoned += batch.messages_tombstoned;
+ self.threads_written += batch.threads_written;
+ self.threads_tombstoned += batch.threads_tombstoned;
+ self.anchors_linked += batch.anchors_linked;
+ self.notifications_remapped += batch.notifications_remapped;
+ self.pdf_payloads_remapped += batch.pdf_payloads_remapped;
+ self.warnings.unmapped_notifications += batch.warnings.unmapped_notifications;
+ self.warnings.unmapped_pdf_comment_ids += batch.warnings.unmapped_pdf_comment_ids;
+ self.warnings.invalid_mark_ids += batch.warnings.invalid_mark_ids;
+ self.warnings.root_order_drift += batch.warnings.root_order_drift;
+ }
+
+ /// True when the run changed nothing.
+ pub fn is_noop(&self) -> bool {
+ self.comment_mappings_allocated == 0
+ && self.thread_mappings_allocated == 0
+ && self.messages_inserted == 0
+ && self.messages_updated == 0
+ && self.messages_tombstoned == 0
+ && self.threads_written == 0
+ && self.threads_tombstoned == 0
+ && self.anchors_linked == 0
+ && self.notifications_remapped == 0
+ && self.pdf_payloads_remapped == 0
+ }
+}
+
+impl fmt::Display for Summary {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ if self.is_noop() {
+ writeln!(
+ f,
+ "Nothing to import: {} documents with legacy comments already match the message store.",
+ self.documents
+ )?;
+ } else {
+ writeln!(
+ f,
+ "Imported legacy comments across {} documents in {} batches.",
+ self.documents, self.batches
+ )?;
+ writeln!(
+ f,
+ " comment mappings allocated: {}",
+ self.comment_mappings_allocated
+ )?;
+ writeln!(
+ f,
+ " thread mappings allocated: {}",
+ self.thread_mappings_allocated
+ )?;
+ writeln!(
+ f,
+ " messages inserted: {}",
+ self.messages_inserted
+ )?;
+ writeln!(f, " messages updated: {}", self.messages_updated)?;
+ writeln!(
+ f,
+ " messages tombstoned: {}",
+ self.messages_tombstoned
+ )?;
+ writeln!(f, " thread states written: {}", self.threads_written)?;
+ writeln!(
+ f,
+ " thread states tombstoned: {}",
+ self.threads_tombstoned
+ )?;
+ writeln!(f, " PDF anchors linked: {}", self.anchors_linked)?;
+ writeln!(
+ f,
+ " notifications remapped: {}",
+ self.notifications_remapped
+ )?;
+ writeln!(
+ f,
+ " PDF payloads remapped: {}",
+ self.pdf_payloads_remapped
+ )?;
+ }
+ write!(f, "{}", self.warnings)
+ }
+}
+
+impl fmt::Display for Warnings {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ let lines = [
+ (
+ self.unmapped_notifications,
+ "document comment notifications still reference a legacy id without a mapping",
+ ),
+ (
+ self.unmapped_pdf_comment_ids,
+ "numeric comment ids in saved PDF payloads have no mapping and were left as is",
+ ),
+ (
+ self.invalid_mark_ids,
+ "threads carry a mark id that is not a UUID and were imported unanchored",
+ ),
+ (
+ self.root_order_drift,
+ "threads whose first legacy comment is no longer the mapped root",
+ ),
+ ];
+ for (count, text) in lines {
+ if count > 0 {
+ writeln!(f, " warning: {count} {text}")?;
+ }
+ }
+ Ok(())
+ }
+}
+
+/// Work still ahead of the next run, without writing anything.
+#[derive(Debug, Default, Clone, PartialEq, Eq)]
+pub struct Pending {
+ pub comments_without_mapping: i64,
+ pub threads_without_mapping: i64,
+ pub documents_with_legacy_comments: i64,
+}
+
+impl fmt::Display for Pending {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ writeln!(f, "Preflight passed.")?;
+ writeln!(
+ f,
+ " documents with legacy comments: {}",
+ self.documents_with_legacy_comments
+ )?;
+ writeln!(
+ f,
+ " comments without a mapping: {}",
+ self.comments_without_mapping
+ )?;
+ writeln!(
+ f,
+ " threads without a mapping: {}",
+ self.threads_without_mapping
+ )?;
+ write!(
+ f,
+ "Edits and deletions of already imported comments are only detected by a real run."
+ )
+ }
+}
diff --git a/crates/macro_db_migrator/src/bin/comment_import/test.rs b/crates/macro_db_migrator/src/bin/comment_import/test.rs
new file mode 100644
index 0000000000..43a2e14943
--- /dev/null
+++ b/crates/macro_db_migrator/src/bin/comment_import/test.rs
@@ -0,0 +1,804 @@
+use super::*;
+use macro_db_migrator::MACRO_DB_MIGRATIONS;
+use serde_json::{Value, json};
+use sqlx::PgPool;
+use summary::Warnings;
+
+const USER: &str = "macro|[email protected]";
+
+async fn legacy_database(pool: &PgPool) -> PgConnection {
+ let mut connection = pool.acquire().await.unwrap().detach();
+ sqlx::raw_sql(include_str!("legacy_fixture.sql"))
+ .execute(&mut connection)
+ .await
+ .unwrap();
+ connection
+}
+
+async fn mappings(connection: &mut PgConnection) -> Vec<(i64, Uuid)> {
+ sqlx::query!("SELECT comment_id, message_id FROM migrated_comment_id ORDER BY comment_id")
+ .fetch_all(connection)
+ .await
+ .unwrap()
+ .into_iter()
+ .map(|row| (row.comment_id, row.message_id))
+ .collect()
+}
+
+async fn thread_mappings(connection: &mut PgConnection) -> Vec<(i64, Uuid)> {
+ sqlx::query!("SELECT thread_id, root_id FROM migrated_comment_thread_id ORDER BY thread_id")
+ .fetch_all(connection)
+ .await
+ .unwrap()
+ .into_iter()
+ .map(|row| (row.thread_id, row.root_id))
+ .collect()
+}
+
+fn mapped(mappings: &[(i64, Uuid)], legacy_id: i64) -> Uuid {
+ mappings
+ .iter()
+ .find(|(id, _)| *id == legacy_id)
+ .map(|(_, uuid)| *uuid)
+ .unwrap()
+}
+
+#[derive(Debug, PartialEq)]
+struct StoredMessage {
+ content: String,
+ thread_id: Option<Uuid>,
+ imported_author: Option<String>,
+ deleted: bool,
+ import_order: Option<i64>,
+}
+
+async fn message(connection: &mut PgConnection, id: Uuid) -> StoredMessage {
+ sqlx::query!(
+ r#"SELECT content, thread_id, imported_author, deleted_at IS NOT NULL AS "deleted!", import_order
+ FROM comms_messages WHERE id = $1"#,
+ id
+ )
+ .map(|row| StoredMessage {
+ content: row.content,
+ thread_id: row.thread_id,
+ imported_author: row.imported_author,
+ deleted: row.deleted,
+ import_order: row.import_order,
+ })
+ .fetch_one(connection)
+ .await
+ .unwrap()
+}
+
+#[derive(Debug, PartialEq)]
+struct StoredThread {
+ resolved: bool,
+ anchor: Option<Value>,
+ deleted: bool,
+}
+
+async fn thread(connection: &mut PgConnection, root_id: Uuid) -> StoredThread {
+ sqlx::query!(
+ r#"SELECT resolved, anchor, deleted_at IS NOT NULL AS "deleted!"
+ FROM comms_message_threads WHERE root_id = $1"#,
+ root_id
+ )
+ .map(|row| StoredThread {
+ resolved: row.resolved,
+ anchor: row.anchor,
+ deleted: row.deleted,
+ })
+ .fetch_one(connection)
+ .await
+ .unwrap()
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn imports_roots_replies_empty_threads_anchors_and_remaps(pool: PgPool) {
+ let mut connection = legacy_database(&pool).await;
+ let summary = run(&mut connection, Options::default()).await.unwrap();
+ assert_eq!(
+ summary,
+ Summary {
+ documents: 2,
+ batches: 1,
+ comment_mappings_allocated: 5,
+ thread_mappings_allocated: 5,
+ // Five comments and one structural root for the empty thread.
+ messages_inserted: 6,
+ messages_updated: 0,
+ messages_tombstoned: 0,
+ threads_written: 5,
+ threads_tombstoned: 0,
+ anchors_linked: 2,
+ notifications_remapped: 3,
+ pdf_payloads_remapped: 1,
+ warnings: Warnings {
+ unmapped_notifications: 1,
+ unmapped_pdf_comment_ids: 1,
+ invalid_mark_ids: 0,
+ root_order_drift: 0,
+ },
+ }
+ );
+ assert!(!summary.is_noop());
+ assert!(
+ summary
+ .to_string()
+ .contains("messages inserted: 6")
+ );
+
+ let comments = mappings(&mut connection).await;
+ let threads = thread_mappings(&mut connection).await;
+ assert_eq!(comments.len(), 5);
+ assert_eq!(threads.len(), 5);
+ assert!(comments.iter().all(|(_, id)| id.get_version_num() == 7));
+ assert!(threads.iter().all(|(_, id)| id.get_version_num() == 7));
+ // Roots follow legacy order: the deleted root keeps its identity as the thread root.
+ assert_eq!(mapped(&threads, 1), mapped(&comments, 10));
+ assert_eq!(mapped(&threads, 3), mapped(&comments, 12));
+ assert_eq!(mapped(&threads, 4), mapped(&comments, 13));
+ assert_eq!(mapped(&threads, 5), mapped(&comments, 14));
+
+ let root = message(&mut connection, mapped(&comments, 10)).await;
+ assert_eq!(
+ root,
+ StoredMessage {
+ content: "Deleted root".into(),
+ thread_id: None,
+ imported_author: Some("Original author".into()),
+ deleted: true,
+ import_order: Some(1),
+ }
+ );
+ let reply = message(&mut connection, mapped(&comments, 11)).await;
+ assert_eq!(
+ reply,
+ StoredMessage {
+ content: "Surviving reply".into(),
+ thread_id: Some(mapped(&threads, 1)),
+ imported_author: None,
+ deleted: false,
+ import_order: Some(2),
+ }
+ );
+ let created_at: chrono::DateTime<chrono::Utc> = sqlx::query_scalar!(
+ r#"SELECT created_at FROM comms_messages WHERE id = $1"#,
+ mapped(&comments, 11)
+ )
+ .fetch_one(&mut connection)
+ .await
+ .unwrap();
+ assert_eq!(created_at.to_rfc3339(), "2020-01-02T00:00:00+00:00");
+
+ // Markdown mark, resolution, and an unanchored Discussion thread.
+ assert_eq!(
+ thread(&mut connection, mapped(&threads, 1)).await,
+ StoredThread {
+ resolved: true,
+ anchor: Some(
+ json!({"type": "markdown", "mark_id": "01990000-0000-7000-8000-000000000002"})
+ ),
+ deleted: false,
+ }
+ );
+ assert_eq!(
+ thread(&mut connection, mapped(&threads, 5)).await,
+ StoredThread {
+ resolved: false,
+ anchor: None,
+ deleted: false,
+ }
+ );
+ // The empty thread is a deleted structural root and a deleted thread.
+ let structural = message(&mut connection, mapped(&threads, 2)).await;
+ assert!(structural.deleted && structural.content.is_empty() && structural.thread_id.is_none());
+ assert!(comments.iter().all(|(_, id)| *id != mapped(&threads, 2)));
+ assert!(thread(&mut connection, mapped(&threads, 2)).await.deleted);
+
+ // Notifications point at message ids; the stale one keeps its legacy id.
+ let notices: Vec<Value> = sqlx::query_scalar!(
+ r#"SELECT metadata AS "metadata!" FROM notification ORDER BY metadata->>'commentText'"#
+ )
+ .fetch_all(&mut connection)
+ .await
+ .unwrap();
+ assert_eq!(notices.len(), 4);
+ assert_eq!(notices[0]["commentId"], 424242);
+ for notice in &notices[1..] {
+ assert_eq!(notice["commentId"], mapped(&comments, 11).to_string());
+ assert_eq!(notice["threadId"], mapped(&threads, 1).to_string());
+ assert_eq!(notice["commentText"], "Surviving reply");
+ }
+ let pdf: Value =
+ sqlx::query_scalar!(r#"SELECT "modificationData" FROM "DocumentInstanceModificationData""#)
+ .fetch_one(&mut connection)
+ .await
+ .unwrap();
+ assert_eq!(
+ pdf["pages"][0]["highlights"][0]["comments"],
+ json!([
+ {"id": mapped(&comments, 12), "text": "PDF original"},
+ {"id": "999", "text": "Stale"},
+ {"id": "external-pdf-id", "text": "External"}
+ ])
+ );
+
+ // PDF anchors keep their legacy thread id and gain the root; thread state points back.
+ let placeable = sqlx::query!(
+ r#"SELECT a."threadId" AS thread_id, a.root_id, a."xPct" AS x_pct, t.anchor
+ FROM "PdfPlaceableCommentAnchor" a JOIN comms_message_threads t ON t.root_id = a.root_id"#
+ )
+ .fetch_one(&mut connection)
+ .await
+ .unwrap();
+ assert_eq!(placeable.thread_id, Some(3));
+ assert_eq!(placeable.root_id, Some(mapped(&threads, 3)));
+ assert_eq!(placeable.x_pct, 0.1);
+ assert_eq!(
+ placeable.anchor,
+ Some(json!({"type": "pdf_placeable", "anchor_id": "01990000-0000-7000-8000-000000000003"}))
+ );
+ let highlight = sqlx::query!(
+ r#"SELECT a."threadId" AS thread_id, a.root_id, a.text, r.top, t.anchor
+ FROM "PdfHighlightAnchor" a JOIN comms_message_threads t ON t.root_id = a.root_id
+ JOIN "PdfHighlightRect" r ON r."pdfHighlightAnchorId" = a.uuid"#
+ )
+ .fetch_one(&mut connection)
+ .await
+ .unwrap();
+ assert_eq!(highlight.thread_id, Some(4));
+ assert_eq!(highlight.root_id, Some(mapped(&threads, 4)));
+ assert_eq!(highlight.text, "Highlighted text");
+ assert_eq!(highlight.top, 0.1);
+ assert_eq!(
+ highlight.anchor,
+ Some(json!({"type": "pdf_highlight", "anchor_id": "01990000-0000-7000-8000-000000000004"}))
+ );
+
+ // Legacy rows are untouched and channel messages are not affected.
+ let legacy_count: i64 = sqlx::query_scalar!(r#"SELECT count(*) AS "count!" FROM "Comment""#)
+ .fetch_one(&mut connection)
+ .await
+ .unwrap();
+ assert_eq!(legacy_count, 5);
+ let channel_messages: Vec<(String, Option<Uuid>)> = sqlx::query!(
+ "SELECT content, thread_id FROM comms_messages WHERE parent_entity_type = 'channel' ORDER BY id"
+ )
+ .fetch_all(&mut connection)
+ .await
+ .unwrap()
+ .into_iter()
+ .map(|row| (row.content, row.thread_id))
+ .collect();
+ assert_eq!(
+ channel_messages,
+ vec![
+ ("Channel root".into(), None),
+ (
+ "Channel reply".into(),
+ Some(Uuid::parse_str("01990000-0000-7000-8000-000000000006").unwrap())
+ ),
+ ]
+ );
+
+ // The imported thread accepts new replies and edits through the message store.
+ let reply_id = generate_uuid_v7();
+ sqlx::query!(
+ "INSERT INTO comms_messages(id, parent_entity_type, parent_entity_id, thread_id, sender_id, content)
+ VALUES ($1, 'document', 'legacy-md', $2, $3, 'After import')",
+ reply_id,
+ mapped(&threads, 1),
+ USER
+ )
+ .execute(&mut connection)
+ .await
+ .unwrap();
+
+ // A second run finds nothing to do and keeps every id.
+ let again = run(&mut connection, Options::default()).await.unwrap();
+ assert!(again.is_noop(), "{again:?}");
+ assert_eq!(again.documents, 2);
+ assert!(again.to_string().starts_with("Nothing to import"));
+ assert_eq!(again.warnings.unmapped_notifications, 1);
+ assert_eq!(mappings(&mut connection).await, comments);
+ assert_eq!(thread_mappings(&mut connection).await, threads);
+ assert_eq!(
+ message(&mut connection, reply_id).await.content,
+ "After import"
+ );
+
+ let pending = check(&mut connection).await.unwrap();
+ assert_eq!(
+ pending,
+ Pending {
+ comments_without_mapping: 0,
+ threads_without_mapping: 0,
+ documents_with_legacy_comments: 2,
+ }
+ );
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn delta_runs_carry_new_edited_and_deleted_legacy_comments(pool: PgPool) {
+ let mut connection = legacy_database(&pool).await;
+ run(&mut connection, Options::default()).await.unwrap();
+ let comments = mappings(&mut connection).await;
+ let threads = thread_mappings(&mut connection).await;
+
+ // Legacy writers keep going: a new reply, an edit, a comment deletion without
+ // an updatedAt bump, a thread deletion, and a comment added to the empty thread.
+ sqlx::raw_sql(
+ r#"
+ INSERT INTO "Comment"(id, "threadId", owner, text, "createdAt", "updatedAt")
+ VALUES (15, 1, 'macro|[email protected]', 'Late reply', '2020-02-01', '2020-02-01');
+ UPDATE "Thread" SET "updatedAt" = '2020-02-01' WHERE id = 1;
+ UPDATE "Comment" SET text = 'Surviving reply, edited', "updatedAt" = '2020-02-02' WHERE id = 11;
+ UPDATE "Comment" SET "deletedAt" = '2020-02-03' WHERE id = 14;
+ UPDATE "Thread" SET "deletedAt" = '2020-02-04' WHERE id = 4;
+ UPDATE "Comment" SET "deletedAt" = '2020-02-04' WHERE "threadId" = 4;
+ INSERT INTO "Comment"(id, "threadId", owner, text, "createdAt", "updatedAt")
+ VALUES (16, 2, 'macro|[email protected]', 'Revived discussion', '2020-02-05', '2020-02-05');
+ UPDATE "Thread" SET "updatedAt" = '2020-02-05' WHERE id = 2;
+ "#,
+ )
+ .execute(&mut connection)
+ .await
+ .unwrap();
+
+ let delta = run(&mut connection, Options::default()).await.unwrap();
+ assert_eq!(delta.comment_mappings_allocated, 2);
+ assert_eq!(delta.thread_mappings_allocated, 0);
+ assert_eq!(delta.messages_inserted, 2);
+ // Edited reply, deleted comment 14, deleted comment 13. The revived thread's
+ // structural root keeps its tombstone untouched.
+ assert_eq!(delta.messages_updated, 3);
+ assert_eq!(delta.messages_tombstoned, 0);
+ // Thread 1 (updated_at), thread 4 (deleted), thread 2 (revived) changed.
+ assert_eq!(delta.threads_written, 3);
+ assert_eq!(delta.warnings.root_order_drift, 1);
+
+ let after = mappings(&mut connection).await;
+ assert_eq!(after.len(), 7);
+ assert!(after.starts_with(&comments));
+ assert_eq!(thread_mappings(&mut connection).await, threads);
+
+ let late = message(&mut connection, mapped(&after, 15)).await;
+ assert_eq!(late.content, "Late reply");
+ assert_eq!(late.thread_id, Some(mapped(&threads, 1)));
+ assert_eq!(late.import_order, Some(3));
+ let edited = message(&mut connection, mapped(&after, 11)).await;
+ assert_eq!(edited.content, "Surviving reply, edited");
+ assert!(!edited.deleted);
+ let edited_at: Option<chrono::NaiveDateTime> = sqlx::query_scalar!(
+ "SELECT edited_at FROM comms_messages WHERE id = $1",
+ mapped(&after, 11)
+ )
+ .fetch_one(&mut connection)
+ .await
+ .unwrap();
+ assert_eq!(edited_at.unwrap().to_string(), "2020-02-02 00:00:00");
+ assert!(message(&mut connection, mapped(&after, 14)).await.deleted);
+ assert!(message(&mut connection, mapped(&after, 13)).await.deleted);
+ assert!(thread(&mut connection, mapped(&threads, 4)).await.deleted);
+ // The highlight keeps its root so the deleted discussion stays addressable.
+ let highlight_root: Option<Uuid> =
+ sqlx::query_scalar!(r#"SELECT root_id FROM "PdfHighlightAnchor""#)
+ .fetch_one(&mut connection)
+ .await
+ .unwrap();
+ assert_eq!(highlight_root, Some(mapped(&threads, 4)));
+
+ // The formerly empty thread is live again: structural root stays a tombstone,
+ // the new comment is a reply to it.
+ let revived = thread(&mut connection, mapped(&threads, 2)).await;
+ assert!(!revived.deleted);
+ let revived_comment = message(&mut connection, mapped(&after, 16)).await;
+ assert_eq!(revived_comment.thread_id, Some(mapped(&threads, 2)));
+ assert!(message(&mut connection, mapped(&threads, 2)).await.deleted);
+
+ let settled = run(&mut connection, Options::default()).await.unwrap();
+ assert!(settled.is_noop(), "{settled:?}");
+ assert_eq!(settled.warnings.root_order_drift, 1);
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn newer_writes_through_the_message_store_are_never_reverted(pool: PgPool) {
+ let mut connection = legacy_database(&pool).await;
+ run(&mut connection, Options::default()).await.unwrap();
+ let comments = mappings(&mut connection).await;
+ let threads = thread_mappings(&mut connection).await;
+ let reply = mapped(&comments, 11);
+
+ // The new UI edits the imported reply and resolves the Discussion thread
+ // after the legacy rows went stale.
+ sqlx::query!(
+ "UPDATE comms_messages SET content = 'Edited in the new UI', updated_at = now(), edited_at = now() WHERE id = $1",
+ reply
+ )
+ .execute(&mut connection)
+ .await
+ .unwrap();
+ sqlx::query!(
+ "UPDATE comms_message_threads SET resolved = true, updated_at = now() WHERE root_id = $1",
+ mapped(&threads, 5)
+ )
+ .execute(&mut connection)
+ .await
+ .unwrap();
+
+ let rerun = run(&mut connection, Options::default()).await.unwrap();
+ assert!(rerun.is_noop(), "{rerun:?}");
+ assert_eq!(
+ message(&mut connection, reply).await.content,
+ "Edited in the new UI"
+ );
+ assert!(thread(&mut connection, mapped(&threads, 5)).await.resolved);
+
+ // A legacy edit that is newer than the stored row still wins.
+ sqlx::query!(
+ r#"UPDATE "Comment" SET text = 'Edited again in legacy', "updatedAt" = now() + interval '1 hour' WHERE id = 11"#
+ )
+ .execute(&mut connection)
+ .await
+ .unwrap();
+ let legacy_wins = run(&mut connection, Options::default()).await.unwrap();
+ assert_eq!(legacy_wins.messages_updated, 1);
+ assert_eq!(
+ message(&mut connection, reply).await.content,
+ "Edited again in legacy"
+ );
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn removed_legacy_rows_become_tombstones(pool: PgPool) {
+ let mut connection = legacy_database(&pool).await;
+ run(&mut connection, Options::default()).await.unwrap();
+ let comments = mappings(&mut connection).await;
+ let threads = thread_mappings(&mut connection).await;
+ sqlx::raw_sql(
+ r#"
+ DELETE FROM "Comment" WHERE id = 14;
+ DELETE FROM "Thread" WHERE id = 3;
+ "#,
+ )
+ .execute(&mut connection)
+ .await
+ .unwrap();
+ let summary = run(&mut connection, Options::default()).await.unwrap();
+ // Comment 14 alone, then thread 3 with its single comment.
+ assert_eq!(summary.messages_tombstoned, 2);
+ assert_eq!(summary.threads_tombstoned, 1);
+ assert!(
+ message(&mut connection, mapped(&comments, 14))
+ .await
+ .deleted
+ );
+ assert!(
+ message(&mut connection, mapped(&comments, 12))
+ .await
+ .deleted
+ );
+ assert!(thread(&mut connection, mapped(&threads, 3)).await.deleted);
+ assert_eq!(mappings(&mut connection).await, comments);
+ assert!(
+ run(&mut connection, Options::default())
+ .await
+ .unwrap()
+ .is_noop()
+ );
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn rejects_concurrent_runner_ambiguous_anchors_and_duplicate_marks(pool: PgPool) {
+ let mut connection = legacy_database(&pool).await;
+ let mut other = pool.acquire().await.unwrap().detach();
+ sqlx::query!("SELECT pg_advisory_lock($1)", LOCK_ID)
+ .execute(&mut other)
+ .await
+ .unwrap();
+ assert!(
+ run(&mut connection, Options::default())
+ .await
+ .unwrap_err()
+ .to_string()
+ .contains("Another comment import")
+ );
+ other.close().await.unwrap();
+
+ sqlx::query!(r#"UPDATE "PdfHighlightAnchor" SET "threadId" = 3"#)
+ .execute(&mut connection)
+ .await
+ .unwrap();
+ assert!(
+ run(&mut connection, Options::default())
+ .await
+ .unwrap_err()
+ .to_string()
+ .contains("Multiple PDF anchors")
+ );
+ sqlx::query!(r#"UPDATE "PdfHighlightAnchor" SET "threadId" = 4"#)
+ .execute(&mut connection)
+ .await
+ .unwrap();
+
+ sqlx::query!(
+ r#"UPDATE "Thread" SET metadata = '{"markId":"01990000-0000-7000-8000-000000000002"}' WHERE id = 5"#
+ )
+ .execute(&mut connection)
+ .await
+ .unwrap();
+ let error = run(&mut connection, Options::default())
+ .await
+ .unwrap_err()
+ .to_string();
+ assert!(error.contains("share the same Markdown mark"), "{error}");
+ assert!(error.contains("[1, 5]"), "{error}");
+ assert!(mappings(&mut connection).await.is_empty());
+
+ // The lock is released after a failed preflight.
+ let mut probe = other_connection(&pool).await;
+ let acquired: bool =
+ sqlx::query_scalar!(r#"SELECT pg_try_advisory_lock($1) AS "acquired!""#, LOCK_ID)
+ .fetch_one(&mut probe)
+ .await
+ .unwrap();
+ assert!(acquired);
+ probe.close().await.unwrap();
+
+ sqlx::query!(r#"UPDATE "Thread" SET metadata = '{"markId":"not-a-mark"}' WHERE id = 5"#)
+ .execute(&mut connection)
+ .await
+ .unwrap();
+ let summary = run(&mut connection, Options::default()).await.unwrap();
+ assert_eq!(summary.warnings.invalid_mark_ids, 1);
+ let threads = thread_mappings(&mut connection).await;
+ assert_eq!(
+ thread(&mut connection, mapped(&threads, 5)).await.anchor,
+ None
+ );
+}
+
+async fn other_connection(pool: &PgPool) -> PgConnection {
+ pool.acquire().await.unwrap().detach()
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn failed_batch_rolls_back_and_can_retry(pool: PgPool) {
+ let mut connection = legacy_database(&pool).await;
+ // A discussion created through the message store already owns the mark of
+ // legacy thread 1, which the preflight cannot see.
+ let conflicting_root = generate_uuid_v7();
+ sqlx::query!(
+ "INSERT INTO comms_messages(id, parent_entity_type, parent_entity_id, sender_id, content)
+ VALUES ($1, 'document', 'legacy-md', $2, 'New store discussion')",
+ conflicting_root,
+ USER
+ )
+ .execute(&mut connection)
+ .await
+ .unwrap();
+ sqlx::query!(
+ r#"UPDATE comms_message_threads SET anchor = '{"type":"markdown","mark_id":"01990000-0000-7000-8000-000000000002"}' WHERE root_id = $1"#,
+ conflicting_root
+ )
+ .execute(&mut connection)
+ .await
+ .unwrap();
+
+ let error = run(&mut connection, Options::default()).await.unwrap_err();
+ assert!(
+ error
+ .to_string()
+ .contains("idx_comms_message_threads_markdown_mark"),
+ "{error}"
+ );
+ assert!(mappings(&mut connection).await.is_empty());
+ assert!(thread_mappings(&mut connection).await.is_empty());
+ let imported: i64 = sqlx::query_scalar!(
+ r#"SELECT count(*) AS "count!" FROM comms_messages WHERE parent_entity_type = 'document'"#
+ )
+ .fetch_one(&mut connection)
+ .await
+ .unwrap();
+ assert_eq!(imported, 1);
+ let legacy_count: i64 = sqlx::query_scalar!(r#"SELECT count(*) AS "count!" FROM "Comment""#)
+ .fetch_one(&mut connection)
+ .await
+ .unwrap();
+ assert_eq!(legacy_count, 5);
+
+ sqlx::query!(
+ "UPDATE comms_message_threads SET anchor = NULL WHERE root_id = $1",
+ conflicting_root
+ )
+ .execute(&mut connection)
+ .await
+ .unwrap();
+ let summary = run(&mut connection, Options::default()).await.unwrap();
+ assert_eq!(summary.comment_mappings_allocated, 5);
+ assert_eq!(summary.thread_mappings_allocated, 5);
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn mappings_cross_batch_boundaries_and_keep_ids_on_retry(pool: PgPool) {
+ let mut connection = legacy_database(&pool).await;
+ let extra = MAPPING_CHUNK as i64 + 100;
+ sqlx::query!(
+ r#"INSERT INTO "Thread"(id, owner, "documentId", resolved)
+ SELECT id, $1, 'legacy-md', false FROM generate_series(100, $2::bigint) id"#,
+ USER,
+ extra
+ )
+ .execute(&mut connection)
+ .await
+ .unwrap();
+ sqlx::query!(
+ r#"INSERT INTO "Comment"(id, "threadId", owner, text)
+ SELECT id, id, $1, 'Batch comment' FROM generate_series(100, $2::bigint) id"#,
+ USER,
+ extra
+ )
+ .execute(&mut connection)
+ .await
+ .unwrap();
+ let options = Options {
+ documents_per_batch: 1,
+ };
+ let summary = run(&mut connection, options).await.unwrap();
+ assert_eq!(summary.batches, 2);
+ assert_eq!(summary.documents, 2);
+ assert_eq!(summary.comment_mappings_allocated, MAPPING_CHUNK as u64 + 6);
+ let first = mappings(&mut connection).await;
+ assert_eq!(first.len(), MAPPING_CHUNK + 6);
+ assert!(run(&mut connection, options).await.unwrap().is_noop());
+ assert_eq!(mappings(&mut connection).await, first);
+ let count: i64 = sqlx::query_scalar!(
+ r#"SELECT count(*) AS "count!" FROM comms_message_threads WHERE parent_entity_type = 'document'"#
+ )
+ .fetch_one(&mut connection)
+ .await
+ .unwrap();
+ assert_eq!(count, MAPPING_CHUNK as i64 + 6);
+}
+
+#[test]
+fn pdf_comment_ids_are_remapped_at_any_depth_and_unmapped_ids_are_counted() {
+ let mapped_id = Uuid::from_u128(12);
+ let mappings = HashMap::from([(12_i64, mapped_id)]);
+ let mut payload = json!({
+ "placeables": [{"payload": {"comments": [
+ {"id": "12", "content": "a"},
+ {"id": "13", "content": "b"},
+ {"id": "wrapper", "comments": [{"id": 12}]}
+ ]}}],
+ "highlights": {"1": [{"thread": {"comments": [{"id": 12}, {"id": "external"}, "not an object"]}}]},
+ "comments": "not an array",
+ });
+ let stats = remap_comment_ids(&mut payload, &mappings);
+ // Two top-level 12s plus the 12 nested inside the "wrapper" comment.
+ assert_eq!(
+ stats,
+ RemapStats {
+ remapped: 3,
+ unmapped: 1,
+ }
+ );
+ assert_eq!(
+ payload["placeables"][0]["payload"]["comments"][2]["comments"][0]["id"],
+ mapped_id.to_string()
+ );
+ assert_eq!(
+ payload["placeables"][0]["payload"]["comments"][0]["id"],
+ mapped_id.to_string()
+ );
+ assert_eq!(
+ payload["placeables"][0]["payload"]["comments"][1]["id"],
+ "13"
+ );
+ assert_eq!(
+ payload["highlights"]["1"][0]["thread"]["comments"][0]["id"],
+ mapped_id.to_string()
+ );
+ assert_eq!(
+ payload["highlights"]["1"][0]["thread"]["comments"][1]["id"],
+ "external"
+ );
+ // A second pass rewrites nothing and still reports the id that has no mapping.
+ assert_eq!(
+ remap_comment_ids(&mut payload, &mappings),
+ RemapStats {
+ remapped: 0,
+ unmapped: 1,
+ }
+ );
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn channel_read_projections_keep_channel_scoped_index_plans(pool: PgPool) {
+ let mut connection = legacy_database(&pool).await;
+ run(&mut connection, Options::default()).await.unwrap();
+ // A small target channel among many unrelated channels and non-UUID documents.
+ // Never disable sequential scans: test the planner's actual chosen access path.
+ sqlx::raw_sql(
+ r#"
+ INSERT INTO comms_channels(id, name, channel_type, owner_id)
+ SELECT md5('plan-channel-' || n)::uuid, 'Plan fixture', 'private', 'macro|[email protected]'
+ FROM generate_series(1, 200) n;
+ INSERT INTO comms_messages(id, parent_entity_type, parent_entity_id, sender_id, content, created_at)
+ SELECT md5('plan-message-' || c || '-' || n)::uuid, 'channel', md5('plan-channel-' || c)::uuid::text,
+ 'macro|[email protected]', 'Unrelated channel', now() - n * interval '1 second'
+ FROM generate_series(1, 200) c CROSS JOIN generate_series(1, 100) n;
+ INSERT INTO comms_messages(id, parent_entity_type, parent_entity_id, sender_id, content)
+ SELECT md5('plan-document-' || n)::uuid, 'document', 'legacy-md',
+ 'macro|[email protected]', 'Unrelated document'
+ FROM generate_series(1, 10000) n;
+ ANALYZE comms_messages;
+ "#,
+ )
+ .execute(&mut connection)
+ .await
+ .unwrap();
+ let channel = Uuid::parse_str("01990000-0000-7000-8000-000000000005").unwrap();
+ let queries = [
+ // Timeline includes tombstone roots with surviving replies, in either direction.
+ "SELECT m.id FROM comms_messages m WHERE m.parent_entity_type = 'channel' AND m.parent_entity_id = $1::uuid::text AND m.thread_id IS NULL
+ AND (m.deleted_at IS NULL OR EXISTS (SELECT 1 FROM comms_messages r WHERE r.thread_id = m.id AND r.deleted_at IS NULL))
+ ORDER BY m.created_at DESC, m.id DESC LIMIT 50",
+ "SELECT m.id FROM comms_messages m WHERE m.parent_entity_type = 'channel' AND m.parent_entity_id = $1::uuid::text AND m.thread_id IS NULL
+ AND (m.deleted_at IS NULL OR EXISTS (SELECT 1 FROM comms_messages r WHERE r.thread_id = m.id AND r.deleted_at IS NULL))
+ ORDER BY m.created_at ASC, m.id ASC LIMIT 50",
+ // Batched latest reads use a parameterized lateral scan for each channel.
+ "SELECT latest.id FROM unnest(ARRAY[$1::uuid]) i(channel_id) LEFT JOIN LATERAL (
+ SELECT m.id FROM comms_messages m WHERE m.parent_entity_type = 'channel' AND m.parent_entity_id = i.channel_id::text AND m.deleted_at IS NULL
+ ORDER BY m.created_at DESC LIMIT 1) latest ON true",
+ "SELECT latest.id FROM unnest(ARRAY[$1::uuid]) i(channel_id) LEFT JOIN LATERAL (
+ SELECT m.id FROM comms_messages m WHERE m.parent_entity_type = 'channel' AND m.parent_entity_id = i.channel_id::text AND m.deleted_at IS NULL AND m.thread_id IS NULL
+ ORDER BY m.created_at DESC LIMIT 1) latest ON true",
+ // Unfiltered reads still need a channel index, including deleted replies.
+ "SELECT m.id FROM comms_messages m WHERE m.parent_entity_type = 'channel' AND m.parent_entity_id = $1::uuid::text ORDER BY m.created_at DESC LIMIT 50",
+ ];
+ for query in queries {
+ let plan = explain(&mut connection, query, channel).await;
+ let mut nodes = vec![];
+ collect_plan_nodes(&plan[0]["Plan"], &mut nodes);
+ assert!(
+ nodes.iter().any(|node| {
+ node["Index Name"]
+ .as_str()
+ .is_some_and(|name| name.starts_with("idx_comms_messages_parent_"))
+ && node["Index Cond"]
+ .as_str()
+ .is_some_and(|condition| condition.contains("parent_entity_id"))
+ }),
+ "Channel predicate must be an index condition, not a post-scan filter: {plan:#}"
+ );
+ assert!(
+ nodes
+ .iter()
+ .all(|node| node["Rows Removed by Filter"].as_u64().unwrap_or(0) < 100),
+ "Channel reads must not filter thousands of unrelated messages: {plan:#}"
+ );
+ }
+}
+
+#[expect(
+ clippy::disallowed_methods,
+ reason = "EXPLAIN over test-provided SQL cannot be checked at compile time"
+)]
+async fn explain(connection: &mut PgConnection, query: &str, channel: Uuid) -> Value {
+ sqlx::query_scalar(&format!("EXPLAIN (ANALYZE, FORMAT JSON) {query}"))
+ .bind(channel)
+ .fetch_one(connection)
+ .await
+ .unwrap()
+}
+
+fn collect_plan_nodes<'a>(node: &'a Value, nodes: &mut Vec<&'a Value>) {
+ nodes.push(node);
+ if let Some(children) = node["Plans"].as_array() {
+ for child in children {
+ collect_plan_nodes(child, nodes);
+ }
+ }
+}
diff --git a/crates/messages/src/outbound/pg_message_repo/test.rs b/crates/messages/src/outbound/pg_message_repo/test.rs
index caeb475b4e..6109c5864e 100644
--- a/crates/messages/src/outbound/pg_message_repo/test.rs
+++ b/crates/messages/src/outbound/pg_message_repo/test.rs
@@ -1082,3 +1082,69 @@ async fn both_parents_use_bounded_previews_and_bidirectional_windows(pool: PgPoo
);
}
}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn legacy_ids_resolve_through_the_import_mapping_tables(pool: PgPool) {
+ setup(&pool).await;
+ let repo = PgMessageRepository::new(pool.clone());
+ let parent = MessageParent::parse("document", "message-doc-a").unwrap();
+ let root = repo
+ .create(command("message-doc-a", None, "imported root"))
+ .await
+ .unwrap();
+ let reply = repo
+ .create(command("message-doc-a", Some(root.id), "imported reply"))
+ .await
+ .unwrap();
+ sqlx::query!(
+ "INSERT INTO migrated_comment_thread_id (thread_id, root_id, document_id) VALUES (1, $1, 'message-doc-a')",
+ root.id
+ )
+ .execute(&pool)
+ .await
+ .unwrap();
+ sqlx::query!(
+ "INSERT INTO migrated_comment_id (comment_id, message_id, document_id)
+ VALUES (10, $1, 'message-doc-a'), (11, $2, 'message-doc-a')",
+ root.id,
+ reply.id
+ )
+ .execute(&pool)
+ .await
+ .unwrap();
+
+ assert_eq!(
+ repo.resolve_legacy(&parent, 11, false).await.unwrap(),
+ Some(reply.id)
+ );
+ assert_eq!(
+ repo.resolve_legacy(&parent, 10, false).await.unwrap(),
+ Some(root.id)
+ );
+ assert_eq!(
+ repo.resolve_legacy(&parent, 1, true).await.unwrap(),
+ Some(root.id)
+ );
+ // A comment id is not a thread id, and a mapping only resolves under its own document.
+ assert_eq!(repo.resolve_legacy(&parent, 11, true).await.unwrap(), None);
+ let other = MessageParent::parse("document", "message-doc-b").unwrap();
+ assert_eq!(repo.resolve_legacy(&other, 11, false).await.unwrap(), None);
+ let channel = MessageParent::parse("channel", &Uuid::from_u128(7).to_string()).unwrap();
+ assert_eq!(
+ repo.resolve_legacy(&channel, 11, false).await.unwrap(),
+ None
+ );
+
+ // Mappings cannot point at messages that were never written.
+ let dangling = sqlx::query!(
+ "INSERT INTO migrated_comment_id (comment_id, message_id, document_id) VALUES (12, $1, 'message-doc-a')",
+ macro_uuid::generate_uuid_v7()
+ )
+ .execute(&pool)
+ .await
+ .unwrap_err();
+ assert_eq!(
+ dangling.as_database_error().unwrap().code().as_deref(),
+ Some("23503")
+ );
+}
diff --git a/docs/legacy-comment-import.md b/docs/legacy-comment-import.md
new file mode 100644
index 0000000000..02b2b8eac2
--- /dev/null
+++ b/docs/legacy-comment-import.md
@@ -0,0 +1,138 @@
+# Legacy comment import
+
+Moves the historical document comments in `"Comment"` / `"Thread"` into the
+shared message store (`comms_messages`, `comms_message_threads`) while the
+application keeps running. Nothing is dropped; the legacy tables and endpoints
+go away in the last PR of the channel and comment unification stack.
+
+## Operating model
+
+The importer is a projection: each run recomputes what every legacy thread and
+comment should look like in the message store and writes only the rows whose
+projection differs. That gives three properties operators rely on:
+
+- **Re-runnable.** A second run without legacy changes prints
+ `Nothing to import` and writes nothing. Ids never change between runs: every
+ comment and thread gets one UUIDv7 in `migrated_comment_id` /
+ `migrated_comment_thread_id` on its first sighting and keeps it.
+- **Delta-aware.** Comments created, edited, or deleted (soft or hard) through
+ the legacy endpoints after a run are picked up by the next run. New comments
+ get mappings; changed ones are compared by text, timestamps, and deletion.
+- **Live.** Work happens in one transaction per batch of documents (250 by
+ default). Each batch takes a `SHARE ROW EXCLUSIVE` lock on `"Comment"` and
+ `"Thread"` for the duration of that transaction, so legacy comment writes
+ wait for at most one batch; they are never rejected. Reads are unaffected.
+ The lock waits at most five seconds for in-flight writers and the batch is
+ retried. Nothing takes `ACCESS EXCLUSIVE`.
+
+Two runs cannot overlap: a session advisory lock makes the second exit with
+`Another comment import is running`.
+
+### Sequence
+
+1. Run the importer as often as you like while legacy writes continue.
+ Each run reports what it wrote and what it could not represent.
+2. Before enabling the new document discussion UI, freeze legacy comment
+ writes: set `LEGACY_COMMENT_WRITES_ENABLED=false` on the document storage
+ service (Doppler project `cloud-storage-service`) and roll it out. Comment
+ create, edit, and delete and anchor delete then answer `503` with
+ `Document comments are read-only while they move to the new message store. Refresh the app in a few minutes and try again.`.
+ Reads, highlight creation, and anchor edits keep working.
+3. Run the importer once more. It must end with `Nothing to import` on a
+ second invocation.
+4. Enable the new UI. Do not run the importer again after that unless legacy
+ writes are frozen; an edit made through the new API is never reverted by a
+ stale legacy row (newer `updated_at` wins), but a legacy write and a new
+ store write on the same thread while both are live is a race.
+
+## Running it
+
+Build from the repository root; the binary is feature gated so it never ships
+with a service:
+
+```sh
+cargo build -p macro_db_migrator --features comment-import --bin comment_import
+```
+
+Point `DATABASE_URL` at the target. For RDS use the existing helper and never
+the repository's local database recipes:
+
+```sh
+export DATABASE_URL="$(tooling/scripts/rds_database_url.sh <rds_identifier> <secret_name> <db_name>)"
+cargo run -p macro_db_migrator --features comment-import --bin comment_import -- --check
+cargo run -p macro_db_migrator --features comment-import --bin comment_import
+cargo run -p macro_db_migrator --features comment-import --bin comment_import -- --documents-per-batch 50
+```
+
+`--check` runs the preflight and prints how many comments and threads still
+lack a mapping without writing. Edits and deletions of already imported
+comments are only detected by a real run.
+
+## What a run writes
+
+- `migrated_comment_id` / `migrated_comment_thread_id`: one row per legacy
+ comment and thread. Roots follow legacy display `order`, then `createdAt`,
+ then id, including deleted roots. An empty thread gets a fresh root id.
+- `comms_messages`: one row per comment with `parent_entity_type = 'document'`,
+ the thread's root as `thread_id`, `sender_id` from `owner`, `imported_author`
+ from `sender`, `import_metadata` from the comment metadata, and
+ `import_order` from the legacy order. A deleted root stays a tombstone with
+ its replies intact. An empty thread gets an empty, deleted structural root.
+- `comms_message_threads`: resolution, legacy metadata, and the anchor:
+ `pdf_placeable` when a placeable anchor points at the thread, else
+ `pdf_highlight` for a live highlight, else `markdown` when the thread's
+ `markId` is a UUID. `DISCUSSION:` marks, missing marks, and marks that are
+ not UUIDs land unanchored. Threads with no comments are marked deleted, so
+ the new UI never shows an empty "This message was deleted." discussion.
+- `"PdfPlaceableCommentAnchor".root_id` / `"PdfHighlightAnchor".root_id` from
+ the thread mapping; the legacy `"threadId"` stays until the contract PR.
+- `notification.metadata` `commentId` / `threadId` for
+ `commented_on_document`, `replied_to_document_comment_thread`, and
+ `mentioned_in_document_comment` rows on the batch's documents.
+- Numeric `comments[].id` values inside saved PDF
+ `"DocumentInstanceModificationData"` payloads.
+
+Rows written through the message store after import are protected: a legacy
+row only overwrites a message or thread state when it is at least as new as the
+stored `updated_at`. Legacy deletions do not bump `updatedAt`, so they still
+apply.
+
+## Preflight failures
+
+The run stops before writing when the legacy data cannot be represented:
+
+- A live thread with more than one PDF anchor. Detach or delete one.
+- Two live legacy threads with comments sharing the same Markdown mark on one
+ document. Delete the duplicate; the message store allows one active
+ discussion per mark.
+
+A batch that fails for another reason (for example a discussion created through
+the new API already owns a legacy thread's mark) rolls back that batch alone.
+Earlier batches stay committed and the failed one is retried on the next run.
+
+## Warnings
+
+Warnings are reported and never abort a run:
+
+- notifications whose `commentId` or `threadId` has no mapping (the comment was
+ removed before the import existed); they keep their legacy ids.
+- numeric PDF comment ids without a mapping; they are left as they are.
+- threads whose mark is neither a UUID nor a `DISCUSSION:` mark; imported
+ unanchored.
+- threads whose first legacy comment is no longer the mapped root; roots are
+ fixed at first import, later comments are replies.
+
+## Tests
+
+From the repository root with `SQLX_OFFLINE` unset and local Postgres running:
+
+```sh
+cargo test -p macro_db_migrator --features comment-import
+```
+
+The suite seeds the legacy tables on the current schema and covers roots,
+replies, deleted roots, empty threads, Markdown and PDF anchors, notification
+and PDF payload remaps, the no-op second run, delta runs, protection of newer
+message store writes, tombstones for removed legacy rows, concurrent runners,
+preflight aborts, rollback of a failed batch, mapping batches beyond 1,000 rows,
+and the parent-scoped index plans for channel reads.
diff --git a/services/document_storage_service/src/api/annotations.rs b/services/document_storage_service/src/api/annotations.rs
index e602d571b4..6fda73daff 100644
--- a/services/document_storage_service/src/api/annotations.rs
+++ b/services/document_storage_service/src/api/annotations.rs
@@ -5,6 +5,7 @@ pub mod delete_comment;
pub mod edit_anchor;
pub mod edit_comment;
pub mod get;
+pub mod write_switch;
use std::collections::HashSet;
@@ -27,6 +28,12 @@ use notification::domain::models::SendNotificationRequestBuilder;
use tower::ServiceBuilder;
pub fn router(state: ApiContext) -> Router<ApiContext> {
+ let write_switch = axum::middleware::from_fn_with_state(
+ write_switch::LegacyCommentWrites {
+ enabled: state.config.legacy_comment_writes_enabled,
+ },
+ write_switch::handler,
+ );
Router::new()
.route(
"/comments/document/{document_id}",
@@ -39,22 +46,27 @@ pub fn router(state: ApiContext) -> Router<ApiContext> {
)
.route(
"/comments/document/{document_id}",
- post(create_comment::create_comment_handler).layer(ServiceBuilder::new().layer(
- axum::middleware::from_fn_with_state(
- state.clone(),
- macro_middleware::cloud_storage::document::ensure_document_exists::handler,
+ post(create_comment::create_comment_handler).layer(
+ ServiceBuilder::new().layer(write_switch.clone()).layer(
+ axum::middleware::from_fn_with_state(
+ state.clone(),
+ macro_middleware::cloud_storage::document::ensure_document_exists::handler,
+ ),
),
- )),
+ ),
)
.route(
"/comments/comment/{comment_id}",
- delete(delete_comment::delete_comment_handler),
+ delete(delete_comment::delete_comment_handler).layer(write_switch.clone()),
+ )
+ .route(
+ "/anchors",
+ delete(delete_anchor::delete_anchor_handler).layer(write_switch.clone()),
)
- .route("/anchors", delete(delete_anchor::delete_anchor_handler))
.route("/anchors", patch(edit_anchor::edit_anchor_handler))
.route(
"/comments/comment/{comment_id}",
- patch(edit_comment::edit_comment_handler),
+ patch(edit_comment::edit_comment_handler).layer(write_switch),
)
.route(
"/anchors/document/{document_id}",
diff --git a/services/document_storage_service/src/api/annotations/write_switch.rs b/services/document_storage_service/src/api/annotations/write_switch.rs
new file mode 100644
index 0000000000..fe3e3fc89f
--- /dev/null
+++ b/services/document_storage_service/src/api/annotations/write_switch.rs
@@ -0,0 +1,39 @@
+use axum::{
+ Json,
+ extract::{Request, State},
+ http::StatusCode,
+ middleware::Next,
+ response::{IntoResponse, Response},
+};
+use model::response::ErrorResponse;
+
+/// Whether the legacy comment tables still accept writes through this service.
+#[derive(Debug, Clone, Copy)]
+pub struct LegacyCommentWrites {
+ pub enabled: bool,
+}
+
+/// Shown to clients that still write through the legacy comment endpoints
+/// while the final import into the shared message store runs.
+pub const DISABLED_MESSAGE: &str = "Document comments are read-only while they move to the new message store. Refresh the app in a few minutes and try again.";
+
+/// Answers 503 for legacy comment writes once the operator has frozen them.
+pub async fn handler(
+ State(switch): State<LegacyCommentWrites>,
+ request: Request,
+ next: Next,
+) -> Response {
+ if switch.enabled {
+ return next.run(request).await;
+ }
+ (
+ StatusCode::SERVICE_UNAVAILABLE,
+ Json(ErrorResponse {
+ message: DISABLED_MESSAGE.into(),
+ }),
+ )
+ .into_response()
+}
+
+#[cfg(test)]
+mod test;
diff --git a/services/document_storage_service/src/api/annotations/write_switch/test.rs b/services/document_storage_service/src/api/annotations/write_switch/test.rs
new file mode 100644
index 0000000000..79beece009
--- /dev/null
+++ b/services/document_storage_service/src/api/annotations/write_switch/test.rs
@@ -0,0 +1,37 @@
+use super::*;
+use axum::{Router, body::Body, http::Request, routing::post};
+use tower::ServiceExt as _;
+
+fn router(enabled: bool) -> Router {
+ Router::new()
+ .route("/comments", post(|| async { StatusCode::OK }))
+ .layer(axum::middleware::from_fn_with_state(
+ LegacyCommentWrites { enabled },
+ handler,
+ ))
+}
+
+fn write_request() -> Request<Body> {
+ Request::builder()
+ .method("POST")
+ .uri("/comments")
+ .body(Body::empty())
+ .unwrap()
+}
+
+#[tokio::test]
+async fn enabled_switch_passes_writes_through() {
+ let response = router(true).oneshot(write_request()).await.unwrap();
+ assert_eq!(response.status(), StatusCode::OK);
+}
+
+#[tokio::test]
+async fn disabled_switch_answers_service_unavailable_with_the_reason() {
+ let response = router(false).oneshot(write_request()).await.unwrap();
+ assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
+ let body = axum::body::to_bytes(response.into_body(), usize::MAX)
+ .await
+ .unwrap();
+ let error: ErrorResponse = serde_json::from_slice(&body).unwrap();
+ assert_eq!(error.message, DISABLED_MESSAGE);
+}
diff --git a/services/document_storage_service/src/config.rs b/services/document_storage_service/src/config.rs
index 5125e64102..b4e4768745 100644
--- a/services/document_storage_service/src/config.rs
+++ b/services/document_storage_service/src/config.rs
@@ -157,6 +157,14 @@ pub struct Config {
#[macro_config_default(false)]
pub calendar_reminder_dispatch_enabled: bool,
+ /// Master switch for the legacy document comment writers: comment create,
+ /// edit and delete, and anchor delete, which also deletes the thread.
+ /// Set to `false` for the final pass of the legacy comment importer, before
+ /// the new document discussion UI is enabled; those handlers then answer
+ /// 503 and the importer works from a frozen source.
+ #[macro_config_default(true)]
+ pub legacy_comment_writes_enabled: bool,
+
/// The number of seconds a signed document or call recording URL is valid for.
#[macro_config_default(DEFAULT_PRESIGNED_URL_EXPIRY_SECONDS)]
pub document_storage_service_presigned_url_expiry_seconds: u64,