Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
feat(messages): idempotent import of legacy document comments (#6437)
gbirman43cb2efaec2e
diff --git a/.github/workspace-dep-closures.json b/.github/workspace-dep-closures.jsonindex 431d81b864..25e5a992d9 100644--- a/.github/workspace-dep-closures.json+++ b/.github/workspace-dep-closures.json@@ -5690,7 +5690,12 @@"crates/workspace-hack"],"macro_db_migrator": [+ "crates/database_env_vars","crates/macro_db_migrator",+ "crates/macro_entrypoint",+ "crates/macro_env",+ "crates/macro_env_var",+ "crates/macro_uuid","crates/workspace-hack"],"macro_entrypoint": [diff --git a/.sqlx/query-038db5b91138c574e5927816834bdedef8ac3d00b378999c7370d1b6f8ec22be.json b/.sqlx/query-038db5b91138c574e5927816834bdedef8ac3d00b378999c7370d1b6f8ec22be.jsonnew file mode 100644index 0000000000..94d118fc91--- /dev/null+++ b/.sqlx/query-038db5b91138c574e5927816834bdedef8ac3d00b378999c7370d1b6f8ec22be.json@@ -0,0 +1,22 @@+{+ "db_name": "PostgreSQL",+ "query": "INSERT INTO comms_messages (\n id, parent_entity_type, parent_entity_id, sender_id, content,\n created_at, updated_at, deleted_at\n )\n SELECT tm.root_id, 'document', t.\"documentId\", t.owner, '',\n t.\"createdAt\" AT TIME ZONE 'UTC', t.\"updatedAt\" AT TIME ZONE 'UTC',\n COALESCE(t.\"deletedAt\", t.\"updatedAt\")\n FROM \"Thread\" t\n JOIN migrated_comment_thread_id tm ON tm.thread_id = t.id\n WHERE t.\"documentId\" = ANY($1)\n AND NOT EXISTS (SELECT 1 FROM \"Comment\" c WHERE c.\"threadId\" = t.id)\n ON CONFLICT (id) DO UPDATE SET\n updated_at = EXCLUDED.updated_at,\n deleted_at = EXCLUDED.deleted_at\n WHERE comms_messages.updated_at <= EXCLUDED.updated_at\n AND (comms_messages.updated_at, comms_messages.deleted_at)\n IS DISTINCT FROM (EXCLUDED.updated_at, EXCLUDED.deleted_at)\n RETURNING (xmax = 0) AS \"inserted!\"",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "inserted!",+ "type_info": "Bool"+ }+ ],+ "parameters": {+ "Left": [+ "TextArray"+ ]+ },+ "nullable": [+ null+ ]+ },+ "hash": "038db5b91138c574e5927816834bdedef8ac3d00b378999c7370d1b6f8ec22be"+}diff --git a/.sqlx/query-099de7baaa6a5bbb26a23719ae554f5e71c055fd4a36eba60c158d663d8d65b6.json b/.sqlx/query-099de7baaa6a5bbb26a23719ae554f5e71c055fd4a36eba60c158d663d8d65b6.jsonnew file mode 100644index 0000000000..605567a034--- /dev/null+++ b/.sqlx/query-099de7baaa6a5bbb26a23719ae554f5e71c055fd4a36eba60c158d663d8d65b6.json@@ -0,0 +1,16 @@+{+ "db_name": "PostgreSQL",+ "query": "INSERT INTO migrated_comment_thread_id (thread_id, root_id, document_id)\n SELECT * FROM UNNEST($1::bigint[], $2::uuid[], $3::text[])",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": [+ "Int8Array",+ "UuidArray",+ "TextArray"+ ]+ },+ "nullable": []+ },+ "hash": "099de7baaa6a5bbb26a23719ae554f5e71c055fd4a36eba60c158d663d8d65b6"+}diff --git a/.sqlx/query-12eba8d33668ec91d11bf8787de9c90499802d0ebe46f7a5b2e5d0f958c12364.json b/.sqlx/query-12eba8d33668ec91d11bf8787de9c90499802d0ebe46f7a5b2e5d0f958c12364.jsonnew file mode 100644index 0000000000..c16520dedf--- /dev/null+++ b/.sqlx/query-12eba8d33668ec91d11bf8787de9c90499802d0ebe46f7a5b2e5d0f958c12364.json@@ -0,0 +1,28 @@+{+ "db_name": "PostgreSQL",+ "query": "SELECT comment_id, message_id FROM migrated_comment_id WHERE document_id = $1",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "comment_id",+ "type_info": "Int8"+ },+ {+ "ordinal": 1,+ "name": "message_id",+ "type_info": "Uuid"+ }+ ],+ "parameters": {+ "Left": [+ "Text"+ ]+ },+ "nullable": [+ false,+ false+ ]+ },+ "hash": "12eba8d33668ec91d11bf8787de9c90499802d0ebe46f7a5b2e5d0f958c12364"+}diff --git a/.sqlx/query-180a9c059064bca1eab390de5c76bf41bc489612d22ae37c8bf52989ba8e062a.json b/.sqlx/query-180a9c059064bca1eab390de5c76bf41bc489612d22ae37c8bf52989ba8e062a.jsonnew file mode 100644index 0000000000..261ad9c71a--- /dev/null+++ b/.sqlx/query-180a9c059064bca1eab390de5c76bf41bc489612d22ae37c8bf52989ba8e062a.json@@ -0,0 +1,14 @@+{+ "db_name": "PostgreSQL",+ "query": "UPDATE \"PdfPlaceableCommentAnchor\" a SET root_id = tm.root_id\n FROM migrated_comment_thread_id tm\n WHERE a.\"threadId\" = tm.thread_id AND a.\"documentId\" = ANY($1)\n AND a.root_id IS DISTINCT FROM tm.root_id",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": [+ "TextArray"+ ]+ },+ "nullable": []+ },+ "hash": "180a9c059064bca1eab390de5c76bf41bc489612d22ae37c8bf52989ba8e062a"+}diff --git a/.sqlx/query-27dba71899c33b28e6f3daf31030b8e9f609ee7e6b0a62bba74ac168ac76f8cd.json b/.sqlx/query-27dba71899c33b28e6f3daf31030b8e9f609ee7e6b0a62bba74ac168ac76f8cd.jsonnew file mode 100644index 0000000000..9cf00258a3--- /dev/null+++ b/.sqlx/query-27dba71899c33b28e6f3daf31030b8e9f609ee7e6b0a62bba74ac168ac76f8cd.json@@ -0,0 +1,23 @@+{+ "db_name": "PostgreSQL",+ "query": "SELECT count(*) AS \"count!\" FROM notification n\n WHERE n.event_item_type = 'document' AND n.event_item_id = ANY($1)\n AND n.notification_event_type = ANY($2)\n AND (n.metadata->>'commentId' ~ '^[0-9]+$' OR n.metadata->>'threadId' ~ '^[0-9]+$')",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "count!",+ "type_info": "Int8"+ }+ ],+ "parameters": {+ "Left": [+ "TextArray",+ "TextArray"+ ]+ },+ "nullable": [+ null+ ]+ },+ "hash": "27dba71899c33b28e6f3daf31030b8e9f609ee7e6b0a62bba74ac168ac76f8cd"+}diff --git a/.sqlx/query-3d92901e27e9ca5ff9f78b9f09e02c76b9e44ee49efec6c4afc9ea620cb27f2f.json b/.sqlx/query-3d92901e27e9ca5ff9f78b9f09e02c76b9e44ee49efec6c4afc9ea620cb27f2f.jsonnew file mode 100644index 0000000000..7fbf5c1431--- /dev/null+++ b/.sqlx/query-3d92901e27e9ca5ff9f78b9f09e02c76b9e44ee49efec6c4afc9ea620cb27f2f.json@@ -0,0 +1,35 @@+{+ "db_name": "PostgreSQL",+ "query": "SELECT m.id, i.\"documentId\" AS document_id, m.\"modificationData\" AS payload\n FROM \"DocumentInstanceModificationData\" m\n JOIN \"DocumentInstance\" i ON i.id = m.\"documentInstanceId\"\n JOIN \"Document\" d ON d.id = i.\"documentId\"\n WHERE i.\"documentId\" = ANY($1) AND d.\"fileType\" = 'pdf'\n AND jsonb_path_exists(m.\"modificationData\", $2::text::jsonpath)\n ORDER BY m.id",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "id",+ "type_info": "Int8"+ },+ {+ "ordinal": 1,+ "name": "document_id",+ "type_info": "Text"+ },+ {+ "ordinal": 2,+ "name": "payload",+ "type_info": "Jsonb"+ }+ ],+ "parameters": {+ "Left": [+ "TextArray",+ "Text"+ ]+ },+ "nullable": [+ false,+ false,+ false+ ]+ },+ "hash": "3d92901e27e9ca5ff9f78b9f09e02c76b9e44ee49efec6c4afc9ea620cb27f2f"+}diff --git a/.sqlx/query-4027823662c88d28cb2a95792b615423ead933fb94bbd3dad5e6e5e7ff59ff53.json b/.sqlx/query-4027823662c88d28cb2a95792b615423ead933fb94bbd3dad5e6e5e7ff59ff53.jsonnew file mode 100644index 0000000000..733b7ba987--- /dev/null+++ b/.sqlx/query-4027823662c88d28cb2a95792b615423ead933fb94bbd3dad5e6e5e7ff59ff53.json@@ -0,0 +1,22 @@+{+ "db_name": "PostgreSQL",+ "query": "SELECT pg_advisory_unlock($1) AS \"released!\"",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "released!",+ "type_info": "Bool"+ }+ ],+ "parameters": {+ "Left": [+ "Int8"+ ]+ },+ "nullable": [+ null+ ]+ },+ "hash": "4027823662c88d28cb2a95792b615423ead933fb94bbd3dad5e6e5e7ff59ff53"+}diff --git a/.sqlx/query-48fbddcebd915244c16d53a6f2c8b68ad4ffd6735cd44817ee2cec34aa6bbe9c.json b/.sqlx/query-48fbddcebd915244c16d53a6f2c8b68ad4ffd6735cd44817ee2cec34aa6bbe9c.jsonnew file mode 100644index 0000000000..f94c37e726--- /dev/null+++ b/.sqlx/query-48fbddcebd915244c16d53a6f2c8b68ad4ffd6735cd44817ee2cec34aa6bbe9c.json@@ -0,0 +1,22 @@+{+ "db_name": "PostgreSQL",+ "query": "SELECT t.id FROM \"Thread\" t\n LEFT JOIN \"PdfPlaceableCommentAnchor\" p ON p.\"threadId\" = t.id AND NOT p.\"wasDeleted\"\n LEFT JOIN \"PdfHighlightAnchor\" h ON h.\"threadId\" = t.id AND h.\"deletedAt\" IS NULL\n WHERE t.\"deletedAt\" IS NULL\n GROUP BY t.id HAVING count(DISTINCT p.uuid) + count(DISTINCT h.uuid) > 1\n ORDER BY t.id LIMIT $1",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "id",+ "type_info": "Int8"+ }+ ],+ "parameters": {+ "Left": [+ "Int8"+ ]+ },+ "nullable": [+ false+ ]+ },+ "hash": "48fbddcebd915244c16d53a6f2c8b68ad4ffd6735cd44817ee2cec34aa6bbe9c"+}diff --git a/.sqlx/query-576c0d1c1c043d4418c8f137138c5a0cd749cc32100c3d95c0cc6792e697e8a9.json b/.sqlx/query-576c0d1c1c043d4418c8f137138c5a0cd749cc32100c3d95c0cc6792e697e8a9.jsonnew file mode 100644index 0000000000..38304c51cb--- /dev/null+++ b/.sqlx/query-576c0d1c1c043d4418c8f137138c5a0cd749cc32100c3d95c0cc6792e697e8a9.json@@ -0,0 +1,22 @@+{+ "db_name": "PostgreSQL",+ "query": "SELECT count(*) AS \"count!\" FROM \"Thread\" t\n JOIN migrated_comment_thread_id tm ON tm.thread_id = t.id\n JOIN LATERAL (\n SELECT cm.message_id FROM \"Comment\" c\n JOIN migrated_comment_id cm ON cm.comment_id = c.id\n WHERE c.\"threadId\" = t.id\n ORDER BY c.\"order\" NULLS LAST, c.\"createdAt\", c.id LIMIT 1\n ) first ON true\n WHERE t.\"documentId\" = ANY($1) AND first.message_id <> tm.root_id",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "count!",+ "type_info": "Int8"+ }+ ],+ "parameters": {+ "Left": [+ "TextArray"+ ]+ },+ "nullable": [+ null+ ]+ },+ "hash": "576c0d1c1c043d4418c8f137138c5a0cd749cc32100c3d95c0cc6792e697e8a9"+}diff --git a/.sqlx/query-624ac87b24d542ae21dabeb58d067643329cd92029864abda17d5376dc999739.json b/.sqlx/query-624ac87b24d542ae21dabeb58d067643329cd92029864abda17d5376dc999739.jsonnew file mode 100644index 0000000000..a0d957a6c9--- /dev/null+++ b/.sqlx/query-624ac87b24d542ae21dabeb58d067643329cd92029864abda17d5376dc999739.json@@ -0,0 +1,15 @@+{+ "db_name": "PostgreSQL",+ "query": "UPDATE \"DocumentInstanceModificationData\" SET \"modificationData\" = $1 WHERE id = $2",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": [+ "Jsonb",+ "Int8"+ ]+ },+ "nullable": []+ },+ "hash": "624ac87b24d542ae21dabeb58d067643329cd92029864abda17d5376dc999739"+}diff --git a/.sqlx/query-72c6260a4c0cdfb61d8273040c5d79d6c615c3e1dcbb4bbfabc9a61e2cead4cd.json b/.sqlx/query-72c6260a4c0cdfb61d8273040c5d79d6c615c3e1dcbb4bbfabc9a61e2cead4cd.jsonnew file mode 100644index 0000000000..57c0fe1328--- /dev/null+++ b/.sqlx/query-72c6260a4c0cdfb61d8273040c5d79d6c615c3e1dcbb4bbfabc9a61e2cead4cd.json@@ -0,0 +1,20 @@+{+ "db_name": "PostgreSQL",+ "query": "SELECT count(*) AS \"count!\" FROM \"Comment\" c\n WHERE NOT EXISTS (SELECT 1 FROM migrated_comment_id m WHERE m.comment_id = c.id)",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "count!",+ "type_info": "Int8"+ }+ ],+ "parameters": {+ "Left": []+ },+ "nullable": [+ null+ ]+ },+ "hash": "72c6260a4c0cdfb61d8273040c5d79d6c615c3e1dcbb4bbfabc9a61e2cead4cd"+}diff --git a/.sqlx/query-74275afbf7e9f3348efbbd7d8c7eb565faee20148c38b8ca42582c02628af0e7.json b/.sqlx/query-74275afbf7e9f3348efbbd7d8c7eb565faee20148c38b8ca42582c02628af0e7.jsonnew file mode 100644index 0000000000..c2e34c72c1--- /dev/null+++ b/.sqlx/query-74275afbf7e9f3348efbbd7d8c7eb565faee20148c38b8ca42582c02628af0e7.json@@ -0,0 +1,16 @@+{+ "db_name": "PostgreSQL",+ "query": "INSERT INTO migrated_comment_id (comment_id, message_id, document_id)\n SELECT * FROM UNNEST($1::bigint[], $2::uuid[], $3::text[])",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": [+ "Int8Array",+ "UuidArray",+ "TextArray"+ ]+ },+ "nullable": []+ },+ "hash": "74275afbf7e9f3348efbbd7d8c7eb565faee20148c38b8ca42582c02628af0e7"+}diff --git a/.sqlx/query-74842f5645a90d4bdefd392a080062648d83a046b36d7d83ed963c3842657a41.json b/.sqlx/query-74842f5645a90d4bdefd392a080062648d83a046b36d7d83ed963c3842657a41.jsonnew file mode 100644index 0000000000..27ff206084--- /dev/null+++ b/.sqlx/query-74842f5645a90d4bdefd392a080062648d83a046b36d7d83ed963c3842657a41.json@@ -0,0 +1,35 @@+{+ "db_name": "PostgreSQL",+ "query": "SELECT t.\"documentId\" AS document_id, t.metadata->>'markId' AS \"mark_id!\",\n array_agg(t.id ORDER BY t.id) AS \"thread_ids!\"\n FROM \"Thread\" t\n WHERE t.\"deletedAt\" IS NULL\n AND t.metadata->>'markId' ~ $1\n AND EXISTS (SELECT 1 FROM \"Comment\" c WHERE c.\"threadId\" = t.id)\n AND NOT EXISTS (SELECT 1 FROM \"PdfPlaceableCommentAnchor\" p WHERE p.\"threadId\" = t.id AND NOT p.\"wasDeleted\")\n AND NOT EXISTS (SELECT 1 FROM \"PdfHighlightAnchor\" h\n WHERE h.\"threadId\" = t.id AND h.\"deletedAt\" IS NULL)\n GROUP BY 1, 2 HAVING count(*) > 1\n ORDER BY 1, 2 LIMIT $2",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "document_id",+ "type_info": "Text"+ },+ {+ "ordinal": 1,+ "name": "mark_id!",+ "type_info": "Text"+ },+ {+ "ordinal": 2,+ "name": "thread_ids!",+ "type_info": "Int8Array"+ }+ ],+ "parameters": {+ "Left": [+ "Text",+ "Int8"+ ]+ },+ "nullable": [+ false,+ null,+ null+ ]+ },+ "hash": "74842f5645a90d4bdefd392a080062648d83a046b36d7d83ed963c3842657a41"+}diff --git a/.sqlx/query-76f3e71eaea836800a3e5857a00621edab609f4eaba0ce9a4da11743cc2b1482.json b/.sqlx/query-76f3e71eaea836800a3e5857a00621edab609f4eaba0ce9a4da11743cc2b1482.jsonnew file mode 100644index 0000000000..fb2702be6c--- /dev/null+++ b/.sqlx/query-76f3e71eaea836800a3e5857a00621edab609f4eaba0ce9a4da11743cc2b1482.json@@ -0,0 +1,15 @@+{+ "db_name": "PostgreSQL",+ "query": "UPDATE notification n SET metadata = jsonb_set(\n jsonb_set(n.metadata, '{commentId}', to_jsonb(cm.message_id::text)),\n '{threadId}', to_jsonb(tm.root_id::text)\n )\n FROM migrated_comment_id cm, migrated_comment_thread_id tm\n WHERE n.event_item_type = 'document' AND n.event_item_id = ANY($1)\n AND n.notification_event_type = ANY($2)\n AND cm.document_id = n.event_item_id AND tm.document_id = n.event_item_id\n AND n.metadata->>'commentId' = cm.comment_id::text\n AND n.metadata->>'threadId' = tm.thread_id::text",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": [+ "TextArray",+ "TextArray"+ ]+ },+ "nullable": []+ },+ "hash": "76f3e71eaea836800a3e5857a00621edab609f4eaba0ce9a4da11743cc2b1482"+}diff --git a/.sqlx/query-7a590cadb532aa035907f3c2889166dd7c5cea5d61ab9e7fedf9f8a5186295e7.json b/.sqlx/query-7a590cadb532aa035907f3c2889166dd7c5cea5d61ab9e7fedf9f8a5186295e7.jsonnew file mode 100644index 0000000000..3ef0c29241--- /dev/null+++ b/.sqlx/query-7a590cadb532aa035907f3c2889166dd7c5cea5d61ab9e7fedf9f8a5186295e7.json@@ -0,0 +1,14 @@+{+ "db_name": "PostgreSQL",+ "query": "UPDATE \"PdfHighlightAnchor\" a SET root_id = tm.root_id\n FROM migrated_comment_thread_id tm\n WHERE a.\"threadId\" = tm.thread_id AND a.\"documentId\" = ANY($1)\n AND a.root_id IS DISTINCT FROM tm.root_id",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": [+ "TextArray"+ ]+ },+ "nullable": []+ },+ "hash": "7a590cadb532aa035907f3c2889166dd7c5cea5d61ab9e7fedf9f8a5186295e7"+}diff --git a/.sqlx/query-8da419734f41296de7dd848d4b2659623a2e31379ba795b68a366b2d6439a516.json b/.sqlx/query-8da419734f41296de7dd848d4b2659623a2e31379ba795b68a366b2d6439a516.jsonnew file mode 100644index 0000000000..c7196a52c7--- /dev/null+++ b/.sqlx/query-8da419734f41296de7dd848d4b2659623a2e31379ba795b68a366b2d6439a516.json@@ -0,0 +1,22 @@+{+ "db_name": "PostgreSQL",+ "query": "SELECT pg_try_advisory_lock($1) AS \"acquired!\"",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "acquired!",+ "type_info": "Bool"+ }+ ],+ "parameters": {+ "Left": [+ "Int8"+ ]+ },+ "nullable": [+ null+ ]+ },+ "hash": "8da419734f41296de7dd848d4b2659623a2e31379ba795b68a366b2d6439a516"+}diff --git a/.sqlx/query-99086fa5c096f71daeb129691382932436bf3ce8a810b8a868655f399e0fc3f5.json b/.sqlx/query-99086fa5c096f71daeb129691382932436bf3ce8a810b8a868655f399e0fc3f5.jsonnew file mode 100644index 0000000000..baa475902d--- /dev/null+++ b/.sqlx/query-99086fa5c096f71daeb129691382932436bf3ce8a810b8a868655f399e0fc3f5.json@@ -0,0 +1,22 @@+{+ "db_name": "PostgreSQL",+ "query": "UPDATE comms_message_threads s SET deleted_at = now(), updated_at = now()\n FROM migrated_comment_thread_id tm\n WHERE tm.root_id = s.root_id AND tm.document_id = ANY($1) AND s.deleted_at IS NULL\n AND NOT EXISTS (SELECT 1 FROM \"Thread\" t WHERE t.id = tm.thread_id)\n RETURNING s.root_id",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "root_id",+ "type_info": "Uuid"+ }+ ],+ "parameters": {+ "Left": [+ "TextArray"+ ]+ },+ "nullable": [+ false+ ]+ },+ "hash": "99086fa5c096f71daeb129691382932436bf3ce8a810b8a868655f399e0fc3f5"+}diff --git a/.sqlx/query-a65e5f3ae0b0506fb5204aa83ea49fa70745c460e53ec316be184a309f492754.json b/.sqlx/query-a65e5f3ae0b0506fb5204aa83ea49fa70745c460e53ec316be184a309f492754.jsonnew file mode 100644index 0000000000..102b4ff269--- /dev/null+++ b/.sqlx/query-a65e5f3ae0b0506fb5204aa83ea49fa70745c460e53ec316be184a309f492754.json@@ -0,0 +1,14 @@+{+ "db_name": "PostgreSQL",+ "query": "UPDATE comms_messages SET deleted_at = now(), updated_at = now()\n WHERE (id = ANY($1) OR thread_id = ANY($1)) AND deleted_at IS NULL",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": [+ "UuidArray"+ ]+ },+ "nullable": []+ },+ "hash": "a65e5f3ae0b0506fb5204aa83ea49fa70745c460e53ec316be184a309f492754"+}diff --git a/.sqlx/query-a6fe5e93cc6b7ed25934216cfa647b8591213be9dc848d874b3b3f2a78226669.json b/.sqlx/query-a6fe5e93cc6b7ed25934216cfa647b8591213be9dc848d874b3b3f2a78226669.jsonnew file mode 100644index 0000000000..5dd86cd9e5--- /dev/null+++ b/.sqlx/query-a6fe5e93cc6b7ed25934216cfa647b8591213be9dc848d874b3b3f2a78226669.json@@ -0,0 +1,22 @@+{+ "db_name": "PostgreSQL",+ "query": "INSERT INTO comms_messages (\n id, parent_entity_type, parent_entity_id, thread_id, sender_id, imported_author,\n content, created_at, updated_at, edited_at, deleted_at, import_metadata, import_order\n )\n SELECT cm.message_id, 'document', t.\"documentId\",\n CASE WHEN cm.message_id = tm.root_id THEN NULL ELSE tm.root_id END,\n c.owner, c.sender, c.text,\n c.\"createdAt\" AT TIME ZONE 'UTC', c.\"updatedAt\" AT TIME ZONE 'UTC',\n CASE WHEN c.\"updatedAt\" > c.\"createdAt\" THEN c.\"updatedAt\" END,\n COALESCE(c.\"deletedAt\", t.\"deletedAt\"), c.metadata,\n row_number() OVER (PARTITION BY t.id ORDER BY c.\"order\" NULLS LAST, c.\"createdAt\", c.id)\n FROM \"Comment\" c\n JOIN \"Thread\" t ON t.id = c.\"threadId\"\n JOIN migrated_comment_id cm ON cm.comment_id = c.id\n JOIN migrated_comment_thread_id tm ON tm.thread_id = t.id\n WHERE t.\"documentId\" = ANY($1)\n ON CONFLICT (id) DO UPDATE SET\n content = EXCLUDED.content,\n updated_at = EXCLUDED.updated_at,\n edited_at = EXCLUDED.edited_at,\n deleted_at = EXCLUDED.deleted_at,\n imported_author = EXCLUDED.imported_author,\n import_metadata = EXCLUDED.import_metadata,\n import_order = EXCLUDED.import_order\n WHERE comms_messages.updated_at <= EXCLUDED.updated_at\n AND (comms_messages.content, comms_messages.updated_at, comms_messages.edited_at,\n comms_messages.deleted_at, comms_messages.imported_author,\n comms_messages.import_metadata, comms_messages.import_order)\n IS DISTINCT FROM\n (EXCLUDED.content, EXCLUDED.updated_at, EXCLUDED.edited_at, EXCLUDED.deleted_at,\n EXCLUDED.imported_author, EXCLUDED.import_metadata, EXCLUDED.import_order)\n RETURNING (xmax = 0) AS \"inserted!\"",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "inserted!",+ "type_info": "Bool"+ }+ ],+ "parameters": {+ "Left": [+ "TextArray"+ ]+ },+ "nullable": [+ null+ ]+ },+ "hash": "a6fe5e93cc6b7ed25934216cfa647b8591213be9dc848d874b3b3f2a78226669"+}diff --git a/.sqlx/query-b44f233a1a834c1ae81dbfda9c248a7b7daf2813c73521fbb8691d4f34ca28ea.json b/.sqlx/query-b44f233a1a834c1ae81dbfda9c248a7b7daf2813c73521fbb8691d4f34ca28ea.jsonnew file mode 100644index 0000000000..bae7d68881--- /dev/null+++ b/.sqlx/query-b44f233a1a834c1ae81dbfda9c248a7b7daf2813c73521fbb8691d4f34ca28ea.json@@ -0,0 +1,20 @@+{+ "db_name": "PostgreSQL",+ "query": "SELECT count(*) AS \"count!\" FROM \"Thread\" t\n WHERE NOT EXISTS (SELECT 1 FROM migrated_comment_thread_id m WHERE m.thread_id = t.id)",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "count!",+ "type_info": "Int8"+ }+ ],+ "parameters": {+ "Left": []+ },+ "nullable": [+ null+ ]+ },+ "hash": "b44f233a1a834c1ae81dbfda9c248a7b7daf2813c73521fbb8691d4f34ca28ea"+}diff --git a/.sqlx/query-bc996dcd91b86d30e494387bacf11fc307ac363e4d1701439b0bc6ad59e29d16.json b/.sqlx/query-bc996dcd91b86d30e494387bacf11fc307ac363e4d1701439b0bc6ad59e29d16.jsonnew file mode 100644index 0000000000..05bb70f569--- /dev/null+++ b/.sqlx/query-bc996dcd91b86d30e494387bacf11fc307ac363e4d1701439b0bc6ad59e29d16.json@@ -0,0 +1,23 @@+{+ "db_name": "PostgreSQL",+ "query": "SELECT d.id FROM \"Document\" d\n WHERE ($1::text IS NULL OR d.id > $1)\n AND (EXISTS (SELECT 1 FROM \"Thread\" t WHERE t.\"documentId\" = d.id)\n OR EXISTS (SELECT 1 FROM migrated_comment_thread_id m WHERE m.document_id = d.id))\n ORDER BY d.id LIMIT $2",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "id",+ "type_info": "Text"+ }+ ],+ "parameters": {+ "Left": [+ "Text",+ "Int8"+ ]+ },+ "nullable": [+ false+ ]+ },+ "hash": "bc996dcd91b86d30e494387bacf11fc307ac363e4d1701439b0bc6ad59e29d16"+}diff --git a/.sqlx/query-c32423d596dca1eb14bef93e22d6b2a37d52cfbf3cf650d1965df042056ad9ce.json b/.sqlx/query-c32423d596dca1eb14bef93e22d6b2a37d52cfbf3cf650d1965df042056ad9ce.jsonnew file mode 100644index 0000000000..62ee47591e--- /dev/null+++ b/.sqlx/query-c32423d596dca1eb14bef93e22d6b2a37d52cfbf3cf650d1965df042056ad9ce.json@@ -0,0 +1,24 @@+{+ "db_name": "PostgreSQL",+ "query": "SELECT count(*) AS \"count!\" FROM \"Thread\" t\n WHERE t.\"documentId\" = ANY($1) AND t.\"deletedAt\" IS NULL\n AND t.metadata->>'markId' IS NOT NULL\n AND t.metadata->>'markId' !~ $2\n AND t.metadata->>'markId' NOT LIKE $3\n AND NOT EXISTS (SELECT 1 FROM \"PdfPlaceableCommentAnchor\" p WHERE p.\"threadId\" = t.id AND NOT p.\"wasDeleted\")\n AND NOT EXISTS (SELECT 1 FROM \"PdfHighlightAnchor\" h\n WHERE h.\"threadId\" = t.id AND h.\"deletedAt\" IS NULL)",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "count!",+ "type_info": "Int8"+ }+ ],+ "parameters": {+ "Left": [+ "TextArray",+ "Text",+ "Text"+ ]+ },+ "nullable": [+ null+ ]+ },+ "hash": "c32423d596dca1eb14bef93e22d6b2a37d52cfbf3cf650d1965df042056ad9ce"+}diff --git a/.sqlx/query-c5e01d489c1b1316ab38953bf81a99b5d98b5f73d5a4ce47c5499d9bf958ce6e.json b/.sqlx/query-c5e01d489c1b1316ab38953bf81a99b5d98b5f73d5a4ce47c5499d9bf958ce6e.jsonnew file mode 100644index 0000000000..e0b1610c7f--- /dev/null+++ b/.sqlx/query-c5e01d489c1b1316ab38953bf81a99b5d98b5f73d5a4ce47c5499d9bf958ce6e.json@@ -0,0 +1,16 @@+{+ "db_name": "PostgreSQL",+ "query": "INSERT INTO comms_message_threads (\n root_id, parent_entity_type, parent_entity_id, user_id, resolved, anchor,\n import_metadata, created_at, updated_at, deleted_at\n )\n SELECT tm.root_id, 'document', t.\"documentId\", t.owner, t.resolved,\n CASE\n WHEN pa.uuid IS NOT NULL\n THEN jsonb_build_object('type', 'pdf_placeable', 'anchor_id', pa.uuid)\n WHEN ph.uuid IS NOT NULL\n THEN jsonb_build_object('type', 'pdf_highlight', 'anchor_id', ph.uuid)\n WHEN t.metadata->>'markId' ~ $3\n THEN jsonb_build_object('type', 'markdown', 'mark_id', (t.metadata->>'markId')::uuid)\n END,\n t.metadata, t.\"createdAt\" AT TIME ZONE 'UTC', t.\"updatedAt\" AT TIME ZONE 'UTC',\n CASE\n WHEN t.\"deletedAt\" IS NOT NULL THEN t.\"deletedAt\" AT TIME ZONE 'UTC'\n WHEN NOT EXISTS (SELECT 1 FROM \"Comment\" c WHERE c.\"threadId\" = t.id)\n THEN t.\"updatedAt\" AT TIME ZONE 'UTC'\n END\n FROM \"Thread\" t\n JOIN migrated_comment_thread_id tm ON tm.thread_id = t.id\n LEFT JOIN LATERAL (\n SELECT a.uuid FROM \"PdfPlaceableCommentAnchor\" a\n WHERE a.\"threadId\" = t.id AND NOT a.\"wasDeleted\"\n ORDER BY a.uuid LIMIT 1\n ) pa ON true\n LEFT JOIN LATERAL (\n SELECT a.uuid FROM \"PdfHighlightAnchor\" a\n WHERE a.\"threadId\" = t.id AND a.\"deletedAt\" IS NULL\n ORDER BY a.uuid LIMIT 1\n ) ph ON true\n WHERE t.\"documentId\" = ANY($1)\n ON CONFLICT (root_id) DO UPDATE SET\n user_id = EXCLUDED.user_id,\n resolved = EXCLUDED.resolved,\n anchor = EXCLUDED.anchor,\n import_metadata = EXCLUDED.import_metadata,\n created_at = EXCLUDED.created_at,\n updated_at = EXCLUDED.updated_at,\n deleted_at = EXCLUDED.deleted_at\n WHERE (comms_message_threads.root_id = ANY($2)\n OR comms_message_threads.updated_at <= EXCLUDED.updated_at)\n AND (comms_message_threads.user_id, comms_message_threads.resolved,\n comms_message_threads.anchor, comms_message_threads.import_metadata,\n comms_message_threads.created_at, comms_message_threads.updated_at,\n comms_message_threads.deleted_at)\n IS DISTINCT FROM\n (EXCLUDED.user_id, EXCLUDED.resolved, EXCLUDED.anchor, EXCLUDED.import_metadata,\n EXCLUDED.created_at, EXCLUDED.updated_at, EXCLUDED.deleted_at)",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": [+ "TextArray",+ "UuidArray",+ "Text"+ ]+ },+ "nullable": []+ },+ "hash": "c5e01d489c1b1316ab38953bf81a99b5d98b5f73d5a4ce47c5499d9bf958ce6e"+}diff --git a/.sqlx/query-cad533fdf9dff2cb374956bb810b6672179e9e5afc7bc6e059b3ba14f4cd0dd0.json b/.sqlx/query-cad533fdf9dff2cb374956bb810b6672179e9e5afc7bc6e059b3ba14f4cd0dd0.jsonnew file mode 100644index 0000000000..63e2c7761a--- /dev/null+++ b/.sqlx/query-cad533fdf9dff2cb374956bb810b6672179e9e5afc7bc6e059b3ba14f4cd0dd0.json@@ -0,0 +1,34 @@+{+ "db_name": "PostgreSQL",+ "query": "SELECT t.id, t.\"documentId\" AS document_id, first.message_id AS \"root_id?\"\n FROM \"Thread\" t\n LEFT JOIN LATERAL (\n SELECT m.message_id FROM \"Comment\" c\n JOIN migrated_comment_id m ON m.comment_id = c.id\n WHERE c.\"threadId\" = t.id\n ORDER BY c.\"order\" NULLS LAST, c.\"createdAt\", c.id LIMIT 1\n ) first ON true\n WHERE t.\"documentId\" = ANY($1)\n AND NOT EXISTS (SELECT 1 FROM migrated_comment_thread_id m WHERE m.thread_id = t.id)\n ORDER BY t.id",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "id",+ "type_info": "Int8"+ },+ {+ "ordinal": 1,+ "name": "document_id",+ "type_info": "Text"+ },+ {+ "ordinal": 2,+ "name": "root_id?",+ "type_info": "Uuid"+ }+ ],+ "parameters": {+ "Left": [+ "TextArray"+ ]+ },+ "nullable": [+ false,+ false,+ false+ ]+ },+ "hash": "cad533fdf9dff2cb374956bb810b6672179e9e5afc7bc6e059b3ba14f4cd0dd0"+}diff --git a/.sqlx/query-e558ac1741835623b34dea5518c43f58913909a51b4f5f6ef057f8cccdd8eab7.json b/.sqlx/query-e558ac1741835623b34dea5518c43f58913909a51b4f5f6ef057f8cccdd8eab7.jsonnew file mode 100644index 0000000000..c831efb930--- /dev/null+++ b/.sqlx/query-e558ac1741835623b34dea5518c43f58913909a51b4f5f6ef057f8cccdd8eab7.json@@ -0,0 +1,28 @@+{+ "db_name": "PostgreSQL",+ "query": "SELECT c.id, t.\"documentId\" AS document_id FROM \"Comment\" c\n JOIN \"Thread\" t ON t.id = c.\"threadId\"\n WHERE t.\"documentId\" = ANY($1)\n AND NOT EXISTS (SELECT 1 FROM migrated_comment_id m WHERE m.comment_id = c.id)\n ORDER BY c.id",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "id",+ "type_info": "Int8"+ },+ {+ "ordinal": 1,+ "name": "document_id",+ "type_info": "Text"+ }+ ],+ "parameters": {+ "Left": [+ "TextArray"+ ]+ },+ "nullable": [+ false,+ false+ ]+ },+ "hash": "e558ac1741835623b34dea5518c43f58913909a51b4f5f6ef057f8cccdd8eab7"+}diff --git a/.sqlx/query-e9f33d6edca87d88c7403b13d19fe3aa835c20b17993e9e8cdabfe20ed5efa34.json b/.sqlx/query-e9f33d6edca87d88c7403b13d19fe3aa835c20b17993e9e8cdabfe20ed5efa34.jsonnew file mode 100644index 0000000000..d4071cbc7a--- /dev/null+++ b/.sqlx/query-e9f33d6edca87d88c7403b13d19fe3aa835c20b17993e9e8cdabfe20ed5efa34.json@@ -0,0 +1,14 @@+{+ "db_name": "PostgreSQL",+ "query": "UPDATE comms_messages m SET deleted_at = now(), updated_at = now()\n FROM migrated_comment_id cm\n WHERE cm.message_id = m.id AND cm.document_id = ANY($1) AND m.deleted_at IS NULL\n AND NOT EXISTS (SELECT 1 FROM \"Comment\" c WHERE c.id = cm.comment_id)",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": [+ "TextArray"+ ]+ },+ "nullable": []+ },+ "hash": "e9f33d6edca87d88c7403b13d19fe3aa835c20b17993e9e8cdabfe20ed5efa34"+}diff --git a/.sqlx/query-f4387247669703803329aa0d090c0b6427a7b17676d1fdf8fdfd950dbf685dd0.json b/.sqlx/query-f4387247669703803329aa0d090c0b6427a7b17676d1fdf8fdfd950dbf685dd0.jsonnew file mode 100644index 0000000000..e0e7f4d8d0--- /dev/null+++ b/.sqlx/query-f4387247669703803329aa0d090c0b6427a7b17676d1fdf8fdfd950dbf685dd0.json@@ -0,0 +1,20 @@+{+ "db_name": "PostgreSQL",+ "query": "SELECT count(DISTINCT \"documentId\") AS \"count!\" FROM \"Thread\"",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "count!",+ "type_info": "Int8"+ }+ ],+ "parameters": {+ "Left": []+ },+ "nullable": [+ null+ ]+ },+ "hash": "f4387247669703803329aa0d090c0b6427a7b17676d1fdf8fdfd950dbf685dd0"+}diff --git a/Cargo.lock b/Cargo.lockindex c35d080dfa..e53cb6ac5e 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -10134,7 +10134,17 @@ dependencies = [name = "macro_db_migrator"version = "0.1.0"dependencies = [+ "chrono",+ "clap",+ "database_env_vars",+ "macro_entrypoint",+ "macro_env",+ "macro_uuid",+ "rootcause",+ "serde_json","sqlx",+ "tokio",+ "tracing","workspace-hack",]diff --git a/crates/macro_db_client/migrations/20260917190902_messages_mapping_foreign_keys.sql b/crates/macro_db_client/migrations/20260917190902_messages_mapping_foreign_keys.sqlnew file mode 100644index 0000000000..770566ff01--- /dev/null+++ b/crates/macro_db_client/migrations/20260917190902_messages_mapping_foreign_keys.sql@@ -0,0 +1,20 @@+-- The legacy comment importer allocates a mapping row and writes the message+-- it points at inside the same transaction, so these references hold at every+-- commit boundary. Deferred so the mapping can be inserted before the message.+ALTER TABLE migrated_comment_id+ ADD CONSTRAINT migrated_comment_id_message_id_fkey+ FOREIGN KEY (message_id) REFERENCES comms_messages (id)+ ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;++ALTER TABLE migrated_comment_thread_id+ ADD CONSTRAINT migrated_comment_thread_id_root_id_fkey+ FOREIGN KEY (root_id) REFERENCES comms_message_threads (root_id)+ ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;++-- The importer works one batch of documents at a time and the Document FK+-- cascades by document; neither had an index on the referencing column.+CREATE INDEX idx_migrated_comment_id_document+ ON migrated_comment_id (document_id);++CREATE INDEX idx_migrated_comment_thread_id_document+ ON migrated_comment_thread_id (document_id);diff --git a/crates/macro_db_migrator/Cargo.toml b/crates/macro_db_migrator/Cargo.tomlindex 15efd1049b..6d827198e5 100644--- a/crates/macro_db_migrator/Cargo.toml+++ b/crates/macro_db_migrator/Cargo.toml@@ -4,6 +4,35 @@ name = "macro_db_migrator"publish = falseversion = "0.1.0"+[features]+comment-import = [+ "dep:clap",+ "dep:database_env_vars",+ "dep:macro_entrypoint",+ "dep:macro_env",+ "dep:macro_uuid",+ "dep:rootcause",+ "dep:serde_json",+ "dep:tokio",+ "dep:tracing",+]++[[bin]]+name = "comment_import"+required-features = ["comment-import"]+[dependencies]+clap = { workspace = true, optional = true }+database_env_vars = { path = "../database_env_vars", optional = true }+macro_entrypoint = { path = "../macro_entrypoint", optional = true }+macro_env = { path = "../macro_env", optional = true }+macro_uuid = { path = "../macro_uuid", optional = true }+rootcause = { workspace = true, optional = true }+serde_json = { workspace = true, optional = true }sqlx = { workspace = true, features = ["migrate"] }+tokio = { workspace = true, features = ["macros", "rt-multi-thread", "time"], optional = true }+tracing = { workspace = true, optional = true }workspace-hack = { version = "0.1", path = "../workspace-hack" }++[dev-dependencies]+chrono = { workspace = true }diff --git a/crates/macro_db_migrator/src/bin/comment_import.rs b/crates/macro_db_migrator/src/bin/comment_import.rsnew file mode 100644index 0000000000..6496b370b1--- /dev/null+++ b/crates/macro_db_migrator/src/bin/comment_import.rs@@ -0,0 +1,45 @@+//! Online, idempotent import of legacy document comments into the shared+//! message store. Safe to rerun while the application keeps serving requests.++#[path = "comment_import/runner.rs"]+mod runner;++use clap::Parser;+use database_env_vars::DatabaseUrl;+use macro_entrypoint::MacroEntrypoint;+use macro_env::Environment;+use sqlx::{Connection, PgConnection};++#[derive(Debug, Parser)]+#[command(+ about = "Import legacy document comments into comms_messages while the application stays live"+)]+struct Args {+ /// Documents imported per transaction. Smaller batches hold the legacy+ /// table lock for less time; larger batches finish sooner.+ #[arg(long, default_value_t = runner::DEFAULT_DOCUMENTS_PER_BATCH)]+ documents_per_batch: i64,+ /// Run the preflight and report pending work without writing anything.+ #[arg(long)]+ check: bool,+}++#[tokio::main]+async fn main() -> Result<(), rootcause::Report> {+ let args = Args::parse();+ let entrypoint = MacroEntrypoint::new(Environment::Local).init();+ let database_url = DatabaseUrl::new()?;+ let mut connection = PgConnection::connect(database_url.as_ref()).await?;+ let report = if args.check {+ runner::check(&mut connection).await?.to_string()+ } else {+ let options = runner::Options {+ documents_per_batch: args.documents_per_batch,+ };+ runner::run(&mut connection, options).await?.to_string()+ };+ println!("{report}");+ connection.close().await?;+ entrypoint.shutdown();+ Ok(())+}diff --git a/crates/macro_db_migrator/src/bin/comment_import/legacy_fixture.sql b/crates/macro_db_migrator/src/bin/comment_import/legacy_fixture.sqlnew file mode 100644index 0000000000..d404905602--- /dev/null+++ b/crates/macro_db_migrator/src/bin/comment_import/legacy_fixture.sql@@ -0,0 +1,46 @@+INSERT INTO macro_user(id, username, email, stripe_customer_id)+VALUES ('01990000-0000-7000-8000-000000000001', 'migration', '[email protected]', 'migration');+INSERT INTO "User"(id, email, macro_user_id)+VALUES ('macro|[email protected]', '[email protected]', '01990000-0000-7000-8000-000000000001');+INSERT INTO comms_channels(id, name, channel_type, owner_id)+VALUES ('01990000-0000-7000-8000-000000000005', 'Existing channel', 'private', 'macro|[email protected]');+INSERT INTO comms_messages(id, channel_id, thread_id, sender_id, content) VALUES+ ('01990000-0000-7000-8000-000000000006', '01990000-0000-7000-8000-000000000005', null, 'macro|[email protected]', 'Channel root'),+ ('01990000-0000-7000-8000-000000000007', '01990000-0000-7000-8000-000000000005', '01990000-0000-7000-8000-000000000006', 'macro|[email protected]', 'Channel reply');+INSERT INTO "Document"(id, name, owner, "fileType") VALUES+ ('legacy-md', 'Legacy', 'macro|[email protected]', 'md'),+ ('legacy-pdf', 'Legacy PDF', 'macro|[email protected]', 'pdf');+INSERT INTO "Thread"(id, owner, "documentId", resolved, metadata, "createdAt", "updatedAt") VALUES+ (1, 'macro|[email protected]', 'legacy-md', true, '{"markId":"01990000-0000-7000-8000-000000000002"}', '2020-01-01', '2020-01-02'),+ (2, 'macro|[email protected]', 'legacy-md', false, '{"markId":"DISCUSSION:legacy"}', '2020-01-05', '2020-01-06'),+ (3, 'macro|[email protected]', 'legacy-pdf', false, '{}', '2020-01-01', '2020-01-01'),+ (4, 'macro|[email protected]', 'legacy-pdf', false, '{}', '2020-01-01', '2020-01-01'),+ (5, 'macro|[email protected]', 'legacy-md', false, '{"markId":"DISCUSSION:01990000-0000-7000-8000-000000000009"}', '2020-01-07', '2020-01-07');+INSERT INTO "Comment"(id, "threadId", owner, sender, text, "order", "createdAt", "updatedAt", "deletedAt") VALUES+ (10, 1, 'macro|[email protected]', 'Original author', 'Deleted root', 1, '2020-01-01', '2020-01-01', '2020-01-03'),+ (11, 1, 'macro|[email protected]', null, 'Surviving reply', 2, '2020-01-02', '2020-01-02', null),+ (12, 3, 'macro|[email protected]', 'PDF Author', 'PDF original', 1, '2020-01-01', '2020-01-01', null),+ (13, 4, 'macro|[email protected]', null, 'Highlight comment', 1, '2020-01-01', '2020-01-01', null),+ (14, 5, 'macro|[email protected]', null, 'Discussion root', null, '2020-01-07', '2020-01-08', null);+INSERT INTO "PdfPlaceableCommentAnchor"(+ uuid, "documentId", owner, page, "wasEdited", "wasDeleted", "shouldLockOnSave",+ "originalPage", "originalIndex", "xPct", "yPct", "widthPct", "heightPct", rotation, "threadId"+) VALUES ('01990000-0000-7000-8000-000000000003', 'legacy-pdf', 'macro|[email protected]',+ 2, false, false, false, 2, 0, 0.1, 0.2, 0.3, 0.4, 0, 3);+INSERT INTO "PdfHighlightAnchor"(+ uuid, "documentId", owner, page, red, green, blue, alpha, type, text,+ "pageViewportWidth", "pageViewportHeight", "threadId"+) VALUES ('01990000-0000-7000-8000-000000000004', 'legacy-pdf', 'macro|[email protected]',+ 1, 255, 255, 0, 0.5, 0, 'Highlighted text', 600, 800, 4);+INSERT INTO "PdfHighlightRect"(top, "left", width, height, "pdfHighlightAnchorId")+VALUES (0.1, 0.2, 0.3, 0.4, '01990000-0000-7000-8000-000000000004');+INSERT INTO "DocumentInstance"(id, "documentId", sha) VALUES (1, 'legacy-pdf', 'fixture');+INSERT INTO "DocumentInstanceModificationData"("documentInstanceId", "modificationData")+VALUES (1, '{"pages":[{"highlights":[{"comments":[{"id":12,"text":"PDF original"},{"id":"999","text":"Stale"},{"id":"external-pdf-id","text":"External"}]}]}]}');+INSERT INTO notification(id, notification_event_type, event_item_id, event_item_type, service_sender, metadata)+SELECT gen_random_uuid(), kind, 'legacy-md', 'document', 'dss',+ '{"commentId":11,"threadId":1,"commentText":"Surviving reply"}'::jsonb+FROM unnest(ARRAY['commented_on_document', 'replied_to_document_comment_thread', 'mentioned_in_document_comment']) kind;+INSERT INTO notification(id, notification_event_type, event_item_id, event_item_type, service_sender, metadata)+VALUES (gen_random_uuid(), 'commented_on_document', 'legacy-md', 'document', 'dss',+ '{"commentId":424242,"threadId":1,"commentText":"Removed long ago"}'::jsonb);diff --git a/crates/macro_db_migrator/src/bin/comment_import/runner.rs b/crates/macro_db_migrator/src/bin/comment_import/runner.rsnew file mode 100644index 0000000000..9b7c11394c--- /dev/null+++ b/crates/macro_db_migrator/src/bin/comment_import/runner.rs@@ -0,0 +1,781 @@+//! Database-only operator tooling. No application request or authorization path.+//!+//! Every run projects the legacy `"Comment"` / `"Thread"` rows onto the shared+//! message tables one batch of documents at a time. Rows whose projection+//! already matches are left alone, so a rerun is a no-op and a run after new+//! legacy writes carries only the delta.++#[path = "summary.rs"]+mod summary;++use std::collections::HashMap;+use std::time::Duration;++use macro_uuid::{Uuid, generate_uuid_v7};+use sqlx::{Connection, PgConnection, Postgres, Transaction};+pub(super) use summary::{Pending, Summary};++const LOCK_ID: i64 = 732_847_301;+/// Documents handled per transaction when the operator gives no override.+pub(super) const DEFAULT_DOCUMENTS_PER_BATCH: i64 = 250;+const MAPPING_CHUNK: usize = 1_000;+const LOCK_RETRIES: u32 = 5;+const LOCK_RETRY_DELAY: Duration = Duration::from_secs(2);+const LOCK_NOT_AVAILABLE: &str = "55P03";+const DEADLOCK_DETECTED: &str = "40P01";+const PREFLIGHT_SAMPLE: i64 = 20;+const UUID_PATTERN: &str =+ "^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$";+const DISCUSSION_MARK_PREFIX: &str = "DISCUSSION:";+const COMMENT_NOTIFICATION_TYPES: [&str; 3] = [+ "commented_on_document",+ "replied_to_document_comment_thread",+ "mentioned_in_document_comment",+];+/// Saved PDF export payloads whose `comments[].id` still carries a legacy id.+const NUMERIC_COMMENT_ID_PATH: &str =+ r#"$.**.comments[*].id ? (@.type() == "number" || @ like_regex "^[0-9]+$")"#;++#[derive(Debug, Clone, Copy)]+pub(super) struct Options {+ pub documents_per_batch: i64,+}++impl Default for Options {+ fn default() -> Self {+ Self {+ documents_per_batch: DEFAULT_DOCUMENTS_PER_BATCH,+ }+ }+}++/// The caller owns this dedicated connection; dropping it also releases the lock.+pub(super) async fn run(+ connection: &mut PgConnection,+ options: Options,+) -> Result<Summary, rootcause::Report> {+ if options.documents_per_batch < 1 {+ rootcause::bail!("documents_per_batch must be at least 1");+ }+ with_lock(connection, |connection| {+ Box::pin(run_locked(connection, options))+ })+ .await+}++/// Preflight and pending counts; never writes.+pub(super) async fn check(connection: &mut PgConnection) -> Result<Pending, rootcause::Report> {+ with_lock(connection, |connection| Box::pin(check_locked(connection))).await+}++type Locked<'a, T> =+ std::pin::Pin<Box<dyn Future<Output = Result<T, rootcause::Report>> + Send + 'a>>;++async fn with_lock<T>(+ connection: &mut PgConnection,+ body: impl for<'a> FnOnce(&'a mut PgConnection) -> Locked<'a, T>,+) -> Result<T, rootcause::Report> {+ let acquired =+ sqlx::query_scalar!(r#"SELECT pg_try_advisory_lock($1) AS "acquired!""#, LOCK_ID)+ .fetch_one(&mut *connection)+ .await?;+ if !acquired {+ rootcause::bail!("Another comment import is running");+ }+ let result = body(connection).await;+ let unlock = sqlx::query_scalar!(r#"SELECT pg_advisory_unlock($1) AS "released!""#, LOCK_ID)+ .fetch_one(&mut *connection)+ .await;+ let outcome = result?;+ unlock?;+ Ok(outcome)+}++async fn check_locked(connection: &mut PgConnection) -> Result<Pending, rootcause::Report> {+ preflight(connection).await?;+ let comments_without_mapping = sqlx::query_scalar!(+ r#"SELECT count(*) AS "count!" FROM "Comment" c+ WHERE NOT EXISTS (SELECT 1 FROM migrated_comment_id m WHERE m.comment_id = c.id)"#+ )+ .fetch_one(&mut *connection)+ .await?;+ let threads_without_mapping = sqlx::query_scalar!(+ r#"SELECT count(*) AS "count!" FROM "Thread" t+ WHERE NOT EXISTS (SELECT 1 FROM migrated_comment_thread_id m WHERE m.thread_id = t.id)"#+ )+ .fetch_one(&mut *connection)+ .await?;+ let documents_with_legacy_comments =+ sqlx::query_scalar!(r#"SELECT count(DISTINCT "documentId") AS "count!" FROM "Thread""#)+ .fetch_one(&mut *connection)+ .await?;+ Ok(Pending {+ comments_without_mapping,+ threads_without_mapping,+ documents_with_legacy_comments,+ })+}++async fn run_locked(+ connection: &mut PgConnection,+ options: Options,+) -> Result<Summary, rootcause::Report> {+ preflight(connection).await?;+ let mut summary = Summary::default();+ let mut cursor: Option<String> = None;+ loop {+ let documents = sqlx::query_scalar!(+ r#"SELECT d.id FROM "Document" d+ WHERE ($1::text IS NULL OR d.id > $1)+ AND (EXISTS (SELECT 1 FROM "Thread" t WHERE t."documentId" = d.id)+ OR EXISTS (SELECT 1 FROM migrated_comment_thread_id m WHERE m.document_id = d.id))+ ORDER BY d.id LIMIT $2"#,+ cursor,+ options.documents_per_batch,+ )+ .fetch_all(&mut *connection)+ .await?;+ let Some(last) = documents.last() else { break };+ cursor = Some(last.clone());+ import_batch_with_retry(connection, &documents, &mut summary).await?;+ summary.documents += documents.len() as u64;+ summary.batches += 1;+ }+ Ok(summary)+}++/// Legacy data that cannot be represented and must be resolved by hand first.+async fn preflight(connection: &mut PgConnection) -> Result<(), rootcause::Report> {+ let ambiguous = sqlx::query_scalar!(+ r#"SELECT t.id FROM "Thread" t+ LEFT JOIN "PdfPlaceableCommentAnchor" p ON p."threadId" = t.id AND NOT p."wasDeleted"+ LEFT JOIN "PdfHighlightAnchor" h ON h."threadId" = t.id AND h."deletedAt" IS NULL+ WHERE t."deletedAt" IS NULL+ GROUP BY t.id HAVING count(DISTINCT p.uuid) + count(DISTINCT h.uuid) > 1+ ORDER BY t.id LIMIT $1"#,+ PREFLIGHT_SAMPLE,+ )+ .fetch_all(&mut *connection)+ .await?;+ if !ambiguous.is_empty() {+ rootcause::bail!(+ "Multiple PDF anchors on a legacy thread; resolve before importing (thread ids {ambiguous:?})"+ );+ }+ let duplicates = sqlx::query!(+ r#"SELECT t."documentId" AS document_id, t.metadata->>'markId' AS "mark_id!",+ array_agg(t.id ORDER BY t.id) AS "thread_ids!"+ FROM "Thread" t+ WHERE t."deletedAt" IS NULL+ AND t.metadata->>'markId' ~ $1+ AND EXISTS (SELECT 1 FROM "Comment" c WHERE c."threadId" = t.id)+ AND NOT EXISTS (SELECT 1 FROM "PdfPlaceableCommentAnchor" p WHERE p."threadId" = t.id AND NOT p."wasDeleted")+ AND NOT EXISTS (SELECT 1 FROM "PdfHighlightAnchor" h+ WHERE h."threadId" = t.id AND h."deletedAt" IS NULL)+ GROUP BY 1, 2 HAVING count(*) > 1+ ORDER BY 1, 2 LIMIT $2"#,+ UUID_PATTERN,+ PREFLIGHT_SAMPLE,+ )+ .fetch_all(&mut *connection)+ .await?;+ if let Some(first) = duplicates.first() {+ rootcause::bail!(+ "Multiple active legacy threads share the same Markdown mark; delete the duplicates before importing (document {} mark {} threads {:?}, {} conflicts sampled)",+ first.document_id,+ first.mark_id,+ first.thread_ids,+ duplicates.len()+ );+ }+ Ok(())+}++async fn import_batch_with_retry(+ connection: &mut PgConnection,+ documents: &[String],+ summary: &mut Summary,+) -> Result<(), rootcause::Report> {+ let mut attempt = 0;+ loop {+ match import_batch(connection, documents, summary).await {+ Ok(()) => return Ok(()),+ Err(error) if attempt < LOCK_RETRIES && is_retryable(&error) => {+ attempt += 1;+ tracing::warn!(+ error = %error,+ attempt,+ documents = documents.len(),+ "legacy tables busy; retrying batch"+ );+ tokio::time::sleep(LOCK_RETRY_DELAY).await;+ }+ Err(error) => return Err(error.into()),+ }+ }+}++fn is_retryable(error: &sqlx::Error) -> bool {+ error+ .as_database_error()+ .and_then(|e| e.code())+ .is_some_and(|code| code == LOCK_NOT_AVAILABLE || code == DEADLOCK_DETECTED)+}++/// One transaction per batch: mapping allocation and the rows those mappings+/// point at commit together, so the mapping foreign keys hold at every boundary.+/// Counts join the run summary only once the batch has committed, so a retried+/// batch is counted once.+async fn import_batch(+ connection: &mut PgConnection,+ documents: &[String],+ summary: &mut Summary,+) -> Result<(), sqlx::Error> {+ let mut tx = connection.begin().await?;+ let mut batch = Summary::default();+ lock_legacy_tables(&mut tx).await?;+ batch.comment_mappings_allocated = allocate_comment_mappings(&mut tx, documents).await?;+ let new_roots = allocate_thread_mappings(&mut tx, documents).await?;+ batch.thread_mappings_allocated = new_roots.len() as u64;++ let (inserted, updated) = upsert_messages(&mut tx, documents).await?;+ batch.messages_inserted += inserted;+ batch.messages_updated += updated;+ let (inserted, updated) = upsert_structural_roots(&mut tx, documents).await?;+ batch.messages_inserted += inserted;+ batch.messages_updated += updated;+ batch.messages_tombstoned += tombstone_orphan_messages(&mut tx, documents).await?;++ batch.threads_written = upsert_thread_state(&mut tx, documents, &new_roots).await?;+ let (threads, messages) = tombstone_orphan_threads(&mut tx, documents).await?;+ batch.threads_tombstoned = threads;+ batch.messages_tombstoned += messages;++ batch.anchors_linked = link_anchors(&mut tx, documents).await?;+ let (remapped, unmapped) = remap_notifications(&mut tx, documents).await?;+ batch.notifications_remapped = remapped;+ batch.warnings.unmapped_notifications = unmapped;+ let (remapped, unmapped) = remap_pdf_payloads(&mut tx, documents).await?;+ batch.pdf_payloads_remapped = remapped;+ batch.warnings.unmapped_pdf_comment_ids = unmapped;++ batch.warnings.invalid_mark_ids = count_invalid_mark_ids(&mut tx, documents).await?;+ batch.warnings.root_order_drift = count_root_order_drift(&mut tx, documents).await?;+ tx.commit().await?;+ summary.absorb(batch);+ Ok(())+}++#[expect(+ clippy::disallowed_methods,+ reason = "utility statements carry no result shape for SQLx to check"+)]+async fn lock_legacy_tables(tx: &mut Transaction<'_, Postgres>) -> Result<(), sqlx::Error> {+ sqlx::query("SET LOCAL lock_timeout = '5s'")+ .execute(&mut **tx)+ .await?;+ sqlx::query(r#"LOCK TABLE "Comment", "Thread" IN SHARE ROW EXCLUSIVE MODE"#)+ .execute(&mut **tx)+ .await?;+ // The 5s timeout bounds only the table-lock wait. Clear it so later batch+ // statements wait normally for unrelated row locks instead of aborting.+ sqlx::query("SET LOCAL lock_timeout = 0")+ .execute(&mut **tx)+ .await?;+ Ok(())+}++async fn allocate_comment_mappings(+ tx: &mut Transaction<'_, Postgres>,+ documents: &[String],+) -> Result<u64, sqlx::Error> {+ let missing = sqlx::query!(+ r#"SELECT c.id, t."documentId" AS document_id FROM "Comment" c+ JOIN "Thread" t ON t.id = c."threadId"+ WHERE t."documentId" = ANY($1)+ AND NOT EXISTS (SELECT 1 FROM migrated_comment_id m WHERE m.comment_id = c.id)+ ORDER BY c.id"#,+ documents,+ )+ .fetch_all(&mut **tx)+ .await?;+ for chunk in missing.chunks(MAPPING_CHUNK) {+ let ids: Vec<i64> = chunk.iter().map(|row| row.id).collect();+ let uuids: Vec<Uuid> = chunk.iter().map(|_| generate_uuid_v7()).collect();+ let document_ids: Vec<String> = chunk.iter().map(|row| row.document_id.clone()).collect();+ sqlx::query!(+ "INSERT INTO migrated_comment_id (comment_id, message_id, document_id)+ SELECT * FROM UNNEST($1::bigint[], $2::uuid[], $3::text[])",+ &ids,+ &uuids,+ &document_ids,+ )+ .execute(&mut **tx)+ .await?;+ }+ Ok(missing.len() as u64)+}++/// Roots follow legacy display order, then creation time, then id, including+/// deleted comments. Empty threads receive a fresh root id for a structural tombstone.+async fn allocate_thread_mappings(+ tx: &mut Transaction<'_, Postgres>,+ documents: &[String],+) -> Result<Vec<Uuid>, sqlx::Error> {+ let missing = sqlx::query!(+ r#"SELECT t.id, t."documentId" AS document_id, first.message_id AS "root_id?"+ FROM "Thread" t+ LEFT JOIN LATERAL (+ SELECT m.message_id FROM "Comment" c+ JOIN migrated_comment_id m ON m.comment_id = c.id+ WHERE c."threadId" = t.id+ ORDER BY c."order" NULLS LAST, c."createdAt", c.id LIMIT 1+ ) first ON true+ WHERE t."documentId" = ANY($1)+ AND NOT EXISTS (SELECT 1 FROM migrated_comment_thread_id m WHERE m.thread_id = t.id)+ ORDER BY t.id"#,+ documents,+ )+ .fetch_all(&mut **tx)+ .await?;+ let mut roots = Vec::with_capacity(missing.len());+ for chunk in missing.chunks(MAPPING_CHUNK) {+ let ids: Vec<i64> = chunk.iter().map(|row| row.id).collect();+ let chunk_roots: Vec<Uuid> = chunk+ .iter()+ .map(|row| row.root_id.unwrap_or_else(generate_uuid_v7))+ .collect();+ let document_ids: Vec<String> = chunk.iter().map(|row| row.document_id.clone()).collect();+ sqlx::query!(+ "INSERT INTO migrated_comment_thread_id (thread_id, root_id, document_id)+ SELECT * FROM UNNEST($1::bigint[], $2::uuid[], $3::text[])",+ &ids,+ &chunk_roots,+ &document_ids,+ )+ .execute(&mut **tx)+ .await?;+ roots.extend(chunk_roots);+ }+ Ok(roots)+}++fn count_inserts(rows: &[bool]) -> (u64, u64) {+ let inserted = rows.iter().filter(|inserted| **inserted).count() as u64;+ (inserted, rows.len() as u64 - inserted)+}++/// Every mapped comment. Existing rows change only when the legacy row differs+/// and is not older than the stored row, so an edit made through the new API+/// after a legacy row went stale is never reverted.+async fn upsert_messages(+ tx: &mut Transaction<'_, Postgres>,+ documents: &[String],+) -> Result<(u64, u64), sqlx::Error> {+ let rows = sqlx::query_scalar!(+ r#"INSERT INTO comms_messages (+ id, parent_entity_type, parent_entity_id, thread_id, sender_id, imported_author,+ content, created_at, updated_at, edited_at, deleted_at, import_metadata, import_order+ )+ SELECT cm.message_id, 'document', t."documentId",+ CASE WHEN cm.message_id = tm.root_id THEN NULL ELSE tm.root_id END,+ c.owner, c.sender, c.text,+ c."createdAt" AT TIME ZONE 'UTC', c."updatedAt" AT TIME ZONE 'UTC',+ CASE WHEN c."updatedAt" > c."createdAt" THEN c."updatedAt" END,+ COALESCE(c."deletedAt", t."deletedAt"), c.metadata,+ row_number() OVER (PARTITION BY t.id ORDER BY c."order" NULLS LAST, c."createdAt", c.id)+ FROM "Comment" c+ JOIN "Thread" t ON t.id = c."threadId"+ JOIN migrated_comment_id cm ON cm.comment_id = c.id+ JOIN migrated_comment_thread_id tm ON tm.thread_id = t.id+ WHERE t."documentId" = ANY($1)+ ON CONFLICT (id) DO UPDATE SET+ content = EXCLUDED.content,+ updated_at = EXCLUDED.updated_at,+ edited_at = EXCLUDED.edited_at,+ deleted_at = EXCLUDED.deleted_at,+ imported_author = EXCLUDED.imported_author,+ import_metadata = EXCLUDED.import_metadata,+ import_order = EXCLUDED.import_order+ WHERE comms_messages.updated_at <= EXCLUDED.updated_at+ AND (comms_messages.content, comms_messages.updated_at, comms_messages.edited_at,+ comms_messages.deleted_at, comms_messages.imported_author,+ comms_messages.import_metadata, comms_messages.import_order)+ IS DISTINCT FROM+ (EXCLUDED.content, EXCLUDED.updated_at, EXCLUDED.edited_at, EXCLUDED.deleted_at,+ EXCLUDED.imported_author, EXCLUDED.import_metadata, EXCLUDED.import_order)+ RETURNING (xmax = 0) AS "inserted!""#,+ documents,+ )+ .fetch_all(&mut **tx)+ .await?;+ Ok(count_inserts(&rows))+}++/// Empty legacy threads keep a deleted root so their identity, anchors, and+/// external links stay representable without rendering as a discussion.+async fn upsert_structural_roots(+ tx: &mut Transaction<'_, Postgres>,+ documents: &[String],+) -> Result<(u64, u64), sqlx::Error> {+ let rows = sqlx::query_scalar!(+ r#"INSERT INTO comms_messages (+ id, parent_entity_type, parent_entity_id, sender_id, content,+ created_at, updated_at, deleted_at+ )+ SELECT tm.root_id, 'document', t."documentId", t.owner, '',+ t."createdAt" AT TIME ZONE 'UTC', t."updatedAt" AT TIME ZONE 'UTC',+ COALESCE(t."deletedAt", t."updatedAt")+ FROM "Thread" t+ JOIN migrated_comment_thread_id tm ON tm.thread_id = t.id+ WHERE t."documentId" = ANY($1)+ AND NOT EXISTS (SELECT 1 FROM "Comment" c WHERE c."threadId" = t.id)+ ON CONFLICT (id) DO UPDATE SET+ updated_at = EXCLUDED.updated_at,+ deleted_at = EXCLUDED.deleted_at+ WHERE comms_messages.updated_at <= EXCLUDED.updated_at+ AND (comms_messages.updated_at, comms_messages.deleted_at)+ IS DISTINCT FROM (EXCLUDED.updated_at, EXCLUDED.deleted_at)+ RETURNING (xmax = 0) AS "inserted!""#,+ documents,+ )+ .fetch_all(&mut **tx)+ .await?;+ Ok(count_inserts(&rows))+}++/// A mapped comment whose legacy row was removed outright reads as deleted.+async fn tombstone_orphan_messages(+ tx: &mut Transaction<'_, Postgres>,+ documents: &[String],+) -> Result<u64, sqlx::Error> {+ let result = sqlx::query!(+ r#"UPDATE comms_messages m SET deleted_at = now(), updated_at = now()+ FROM migrated_comment_id cm+ WHERE cm.message_id = m.id AND cm.document_id = ANY($1) AND m.deleted_at IS NULL+ AND NOT EXISTS (SELECT 1 FROM "Comment" c WHERE c.id = cm.comment_id)"#,+ documents,+ )+ .execute(&mut **tx)+ .await?;+ Ok(result.rows_affected())+}++/// Thread state for every mapped thread. The root insert trigger has already+/// created a placeholder row for roots written in this batch; those are always+/// overwritten. Older rows follow the same newer-wins rule as messages.+async fn upsert_thread_state(+ tx: &mut Transaction<'_, Postgres>,+ documents: &[String],+ new_roots: &[Uuid],+) -> Result<u64, sqlx::Error> {+ let result = sqlx::query!(+ r#"INSERT INTO comms_message_threads (+ root_id, parent_entity_type, parent_entity_id, user_id, resolved, anchor,+ import_metadata, created_at, updated_at, deleted_at+ )+ SELECT tm.root_id, 'document', t."documentId", t.owner, t.resolved,+ CASE+ WHEN pa.uuid IS NOT NULL+ THEN jsonb_build_object('type', 'pdf_placeable', 'anchor_id', pa.uuid)+ WHEN ph.uuid IS NOT NULL+ THEN jsonb_build_object('type', 'pdf_highlight', 'anchor_id', ph.uuid)+ WHEN t.metadata->>'markId' ~ $3+ THEN jsonb_build_object('type', 'markdown', 'mark_id', (t.metadata->>'markId')::uuid)+ END,+ t.metadata, t."createdAt" AT TIME ZONE 'UTC', t."updatedAt" AT TIME ZONE 'UTC',+ CASE+ WHEN t."deletedAt" IS NOT NULL THEN t."deletedAt" AT TIME ZONE 'UTC'+ WHEN NOT EXISTS (SELECT 1 FROM "Comment" c WHERE c."threadId" = t.id)+ THEN t."updatedAt" AT TIME ZONE 'UTC'+ END+ FROM "Thread" t+ JOIN migrated_comment_thread_id tm ON tm.thread_id = t.id+ LEFT JOIN LATERAL (+ SELECT a.uuid FROM "PdfPlaceableCommentAnchor" a+ WHERE a."threadId" = t.id AND NOT a."wasDeleted"+ ORDER BY a.uuid LIMIT 1+ ) pa ON true+ LEFT JOIN LATERAL (+ SELECT a.uuid FROM "PdfHighlightAnchor" a+ WHERE a."threadId" = t.id AND a."deletedAt" IS NULL+ ORDER BY a.uuid LIMIT 1+ ) ph ON true+ WHERE t."documentId" = ANY($1)+ ON CONFLICT (root_id) DO UPDATE SET+ user_id = EXCLUDED.user_id,+ resolved = EXCLUDED.resolved,+ anchor = EXCLUDED.anchor,+ import_metadata = EXCLUDED.import_metadata,+ created_at = EXCLUDED.created_at,+ updated_at = EXCLUDED.updated_at,+ deleted_at = EXCLUDED.deleted_at+ WHERE (comms_message_threads.root_id = ANY($2)+ OR comms_message_threads.updated_at <= EXCLUDED.updated_at)+ AND (comms_message_threads.user_id, comms_message_threads.resolved,+ comms_message_threads.anchor, comms_message_threads.import_metadata,+ comms_message_threads.created_at, comms_message_threads.updated_at,+ comms_message_threads.deleted_at)+ IS DISTINCT FROM+ (EXCLUDED.user_id, EXCLUDED.resolved, EXCLUDED.anchor, EXCLUDED.import_metadata,+ EXCLUDED.created_at, EXCLUDED.updated_at, EXCLUDED.deleted_at)"#,+ documents,+ new_roots,+ UUID_PATTERN,+ )+ .execute(&mut **tx)+ .await?;+ Ok(result.rows_affected())+}++/// A mapped thread whose legacy row was removed outright reads as deleted, with its messages.+async fn tombstone_orphan_threads(+ tx: &mut Transaction<'_, Postgres>,+ documents: &[String],+) -> Result<(u64, u64), sqlx::Error> {+ let roots = sqlx::query_scalar!(+ r#"UPDATE comms_message_threads s SET deleted_at = now(), updated_at = now()+ FROM migrated_comment_thread_id tm+ WHERE tm.root_id = s.root_id AND tm.document_id = ANY($1) AND s.deleted_at IS NULL+ AND NOT EXISTS (SELECT 1 FROM "Thread" t WHERE t.id = tm.thread_id)+ RETURNING s.root_id"#,+ documents,+ )+ .fetch_all(&mut **tx)+ .await?;+ if roots.is_empty() {+ return Ok((0, 0));+ }+ let messages = sqlx::query!(+ r#"UPDATE comms_messages SET deleted_at = now(), updated_at = now()+ WHERE (id = ANY($1) OR thread_id = ANY($1)) AND deleted_at IS NULL"#,+ &roots,+ )+ .execute(&mut **tx)+ .await?;+ Ok((roots.len() as u64, messages.rows_affected()))+}++async fn link_anchors(+ tx: &mut Transaction<'_, Postgres>,+ documents: &[String],+) -> Result<u64, sqlx::Error> {+ let placeables = sqlx::query!(+ r#"UPDATE "PdfPlaceableCommentAnchor" a SET root_id = tm.root_id+ FROM migrated_comment_thread_id tm+ WHERE a."threadId" = tm.thread_id AND a."documentId" = ANY($1)+ AND a.root_id IS DISTINCT FROM tm.root_id"#,+ documents,+ )+ .execute(&mut **tx)+ .await?;+ let highlights = sqlx::query!(+ r#"UPDATE "PdfHighlightAnchor" a SET root_id = tm.root_id+ FROM migrated_comment_thread_id tm+ WHERE a."threadId" = tm.thread_id AND a."documentId" = ANY($1)+ AND a.root_id IS DISTINCT FROM tm.root_id"#,+ documents,+ )+ .execute(&mut **tx)+ .await?;+ Ok(placeables.rows_affected() + highlights.rows_affected())+}++/// Historical document comment notifications point at message ids once both+/// the comment and its thread have mappings. Rewritten ids never match again.+async fn remap_notifications(+ tx: &mut Transaction<'_, Postgres>,+ documents: &[String],+) -> Result<(u64, u64), sqlx::Error> {+ let types: Vec<String> = COMMENT_NOTIFICATION_TYPES+ .iter()+ .map(|kind| (*kind).to_owned())+ .collect();+ let remapped = sqlx::query!(+ r#"UPDATE notification n SET metadata = jsonb_set(+ jsonb_set(n.metadata, '{commentId}', to_jsonb(cm.message_id::text)),+ '{threadId}', to_jsonb(tm.root_id::text)+ )+ FROM migrated_comment_id cm, migrated_comment_thread_id tm+ WHERE n.event_item_type = 'document' AND n.event_item_id = ANY($1)+ AND n.notification_event_type = ANY($2)+ AND cm.document_id = n.event_item_id AND tm.document_id = n.event_item_id+ AND n.metadata->>'commentId' = cm.comment_id::text+ AND n.metadata->>'threadId' = tm.thread_id::text"#,+ documents,+ &types,+ )+ .execute(&mut **tx)+ .await?;+ let unmapped = sqlx::query_scalar!(+ r#"SELECT count(*) AS "count!" FROM notification n+ WHERE n.event_item_type = 'document' AND n.event_item_id = ANY($1)+ AND n.notification_event_type = ANY($2)+ AND (n.metadata->>'commentId' ~ '^[0-9]+$' OR n.metadata->>'threadId' ~ '^[0-9]+$')"#,+ documents,+ &types,+ )+ .fetch_one(&mut **tx)+ .await?;+ Ok((remapped.rows_affected(), unmapped as u64))+}++/// Saved PDF export payloads are relational metadata; their comment ids follow+/// the mapping. Ids without a mapping stay untouched and are counted.+async fn remap_pdf_payloads(+ tx: &mut Transaction<'_, Postgres>,+ documents: &[String],+) -> Result<(u64, u64), sqlx::Error> {+ let payloads = sqlx::query!(+ r#"SELECT m.id, i."documentId" AS document_id, m."modificationData" AS payload+ FROM "DocumentInstanceModificationData" m+ JOIN "DocumentInstance" i ON i.id = m."documentInstanceId"+ JOIN "Document" d ON d.id = i."documentId"+ WHERE i."documentId" = ANY($1) AND d."fileType" = 'pdf'+ AND jsonb_path_exists(m."modificationData", $2::text::jsonpath)+ ORDER BY m.id"#,+ documents,+ NUMERIC_COMMENT_ID_PATH,+ )+ .fetch_all(&mut **tx)+ .await?;+ let mut remapped = 0;+ let mut unmapped = 0;+ let mut mappings: HashMap<String, HashMap<i64, Uuid>> = HashMap::new();+ for row in payloads {+ if !mappings.contains_key(&row.document_id) {+ let rows = sqlx::query!(+ "SELECT comment_id, message_id FROM migrated_comment_id WHERE document_id = $1",+ row.document_id,+ )+ .fetch_all(&mut **tx)+ .await?;+ mappings.insert(+ row.document_id.clone(),+ rows.into_iter()+ .map(|mapping| (mapping.comment_id, mapping.message_id))+ .collect(),+ );+ }+ let mut payload = row.payload;+ let stats = remap_comment_ids(&mut payload, &mappings[&row.document_id]);+ unmapped += stats.unmapped;+ if stats.remapped > 0 {+ sqlx::query!(+ r#"UPDATE "DocumentInstanceModificationData" SET "modificationData" = $1 WHERE id = $2"#,+ payload,+ row.id,+ )+ .execute(&mut **tx)+ .await?;+ remapped += 1;+ }+ }+ Ok((remapped, unmapped))+}++#[derive(Debug, Default, PartialEq, Eq)]+struct RemapStats {+ remapped: u64,+ unmapped: u64,+}++/// Rewrite every `comments[].id` holding a legacy comment id, at any depth.+fn remap_comment_ids(value: &mut serde_json::Value, mappings: &HashMap<i64, Uuid>) -> RemapStats {+ let mut stats = RemapStats::default();+ match value {+ serde_json::Value::Array(items) => {+ for item in items {+ let inner = remap_comment_ids(item, mappings);+ stats.remapped += inner.remapped;+ stats.unmapped += inner.unmapped;+ }+ }+ serde_json::Value::Object(fields) => {+ for (key, child) in fields.iter_mut() {+ if key == "comments" && child.is_array() {+ for comment in child.as_array_mut().into_iter().flatten() {+ if let Some(id) = comment.get("id").and_then(legacy_comment_id) {+ match mappings.get(&id) {+ Some(message_id) => {+ comment["id"] =+ serde_json::Value::String(message_id.to_string());+ stats.remapped += 1;+ }+ None => stats.unmapped += 1,+ }+ }+ // A comments array may nest inside a comment object.+ let inner = remap_comment_ids(comment, mappings);+ stats.remapped += inner.remapped;+ stats.unmapped += inner.unmapped;+ }+ } else {+ let inner = remap_comment_ids(child, mappings);+ stats.remapped += inner.remapped;+ stats.unmapped += inner.unmapped;+ }+ }+ }+ _ => {}+ }+ stats+}++fn legacy_comment_id(id: &serde_json::Value) -> Option<i64> {+ match id {+ serde_json::Value::Number(number) => number.as_i64(),+ serde_json::Value::String(text) if text.bytes().all(|byte| byte.is_ascii_digit()) => {+ text.parse().ok()+ }+ _ => None,+ }+}++async fn count_invalid_mark_ids(+ tx: &mut Transaction<'_, Postgres>,+ documents: &[String],+) -> Result<u64, sqlx::Error> {+ let count = sqlx::query_scalar!(+ r#"SELECT count(*) AS "count!" FROM "Thread" t+ WHERE t."documentId" = ANY($1) AND t."deletedAt" IS NULL+ AND t.metadata->>'markId' IS NOT NULL+ AND t.metadata->>'markId' !~ $2+ AND t.metadata->>'markId' NOT LIKE $3+ AND NOT EXISTS (SELECT 1 FROM "PdfPlaceableCommentAnchor" p WHERE p."threadId" = t.id AND NOT p."wasDeleted")+ AND NOT EXISTS (SELECT 1 FROM "PdfHighlightAnchor" h+ WHERE h."threadId" = t.id AND h."deletedAt" IS NULL)"#,+ documents,+ UUID_PATTERN,+ format!("{DISCUSSION_MARK_PREFIX}%"),+ )+ .fetch_one(&mut **tx)+ .await?;+ Ok(count as u64)+}++/// Roots are fixed at first import; a legacy reorder afterwards is reported, not applied.+async fn count_root_order_drift(+ tx: &mut Transaction<'_, Postgres>,+ documents: &[String],+) -> Result<u64, sqlx::Error> {+ let count = sqlx::query_scalar!(+ r#"SELECT count(*) AS "count!" FROM "Thread" t+ JOIN migrated_comment_thread_id tm ON tm.thread_id = t.id+ JOIN LATERAL (+ SELECT cm.message_id FROM "Comment" c+ JOIN migrated_comment_id cm ON cm.comment_id = c.id+ WHERE c."threadId" = t.id+ ORDER BY c."order" NULLS LAST, c."createdAt", c.id LIMIT 1+ ) first ON true+ WHERE t."documentId" = ANY($1) AND first.message_id <> tm.root_id"#,+ documents,+ )+ .fetch_one(&mut **tx)+ .await?;+ Ok(count as u64)+}++#[cfg(test)]+#[path = "test.rs"]+mod test;diff --git a/crates/macro_db_migrator/src/bin/comment_import/summary.rs b/crates/macro_db_migrator/src/bin/comment_import/summary.rsnew file mode 100644index 0000000000..4e1bedd7b8--- /dev/null+++ b/crates/macro_db_migrator/src/bin/comment_import/summary.rs@@ -0,0 +1,189 @@+//! Run reports printed to the operator.++use std::fmt;++/// What one run wrote. Every counter is zero when the store already matched.+#[derive(Debug, Default, Clone, PartialEq, Eq)]+pub struct Summary {+ pub documents: u64,+ pub batches: u64,+ pub comment_mappings_allocated: u64,+ pub thread_mappings_allocated: u64,+ pub messages_inserted: u64,+ pub messages_updated: u64,+ pub messages_tombstoned: u64,+ pub threads_written: u64,+ pub threads_tombstoned: u64,+ pub anchors_linked: u64,+ pub notifications_remapped: u64,+ pub pdf_payloads_remapped: u64,+ pub warnings: Warnings,+}++/// Legacy data the import could not represent exactly. Nothing here aborts a run.+#[derive(Debug, Default, Clone, PartialEq, Eq)]+pub struct Warnings {+ /// Document comment notifications still pointing at a legacy id without a mapping.+ pub unmapped_notifications: u64,+ /// Numeric comment ids in saved PDF payloads without a mapping.+ pub unmapped_pdf_comment_ids: u64,+ /// Threads whose mark id is neither a UUID nor a `DISCUSSION:` mark; imported unanchored.+ pub invalid_mark_ids: u64,+ /// Threads whose first legacy comment is no longer the mapped root.+ pub root_order_drift: u64,+}++impl Summary {+ /// Add a committed batch to the run.+ pub fn absorb(&mut self, batch: Self) {+ self.documents += batch.documents;+ self.batches += batch.batches;+ self.comment_mappings_allocated += batch.comment_mappings_allocated;+ self.thread_mappings_allocated += batch.thread_mappings_allocated;+ self.messages_inserted += batch.messages_inserted;+ self.messages_updated += batch.messages_updated;+ self.messages_tombstoned += batch.messages_tombstoned;+ self.threads_written += batch.threads_written;+ self.threads_tombstoned += batch.threads_tombstoned;+ self.anchors_linked += batch.anchors_linked;+ self.notifications_remapped += batch.notifications_remapped;+ self.pdf_payloads_remapped += batch.pdf_payloads_remapped;+ self.warnings.unmapped_notifications += batch.warnings.unmapped_notifications;+ self.warnings.unmapped_pdf_comment_ids += batch.warnings.unmapped_pdf_comment_ids;+ self.warnings.invalid_mark_ids += batch.warnings.invalid_mark_ids;+ self.warnings.root_order_drift += batch.warnings.root_order_drift;+ }++ /// True when the run changed nothing.+ pub fn is_noop(&self) -> bool {+ self.comment_mappings_allocated == 0+ && self.thread_mappings_allocated == 0+ && self.messages_inserted == 0+ && self.messages_updated == 0+ && self.messages_tombstoned == 0+ && self.threads_written == 0+ && self.threads_tombstoned == 0+ && self.anchors_linked == 0+ && self.notifications_remapped == 0+ && self.pdf_payloads_remapped == 0+ }+}++impl fmt::Display for Summary {+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {+ if self.is_noop() {+ writeln!(+ f,+ "Nothing to import: {} documents with legacy comments already match the message store.",+ self.documents+ )?;+ } else {+ writeln!(+ f,+ "Imported legacy comments across {} documents in {} batches.",+ self.documents, self.batches+ )?;+ writeln!(+ f,+ " comment mappings allocated: {}",+ self.comment_mappings_allocated+ )?;+ writeln!(+ f,+ " thread mappings allocated: {}",+ self.thread_mappings_allocated+ )?;+ writeln!(+ f,+ " messages inserted: {}",+ self.messages_inserted+ )?;+ writeln!(f, " messages updated: {}", self.messages_updated)?;+ writeln!(+ f,+ " messages tombstoned: {}",+ self.messages_tombstoned+ )?;+ writeln!(f, " thread states written: {}", self.threads_written)?;+ writeln!(+ f,+ " thread states tombstoned: {}",+ self.threads_tombstoned+ )?;+ writeln!(f, " PDF anchors linked: {}", self.anchors_linked)?;+ writeln!(+ f,+ " notifications remapped: {}",+ self.notifications_remapped+ )?;+ writeln!(+ f,+ " PDF payloads remapped: {}",+ self.pdf_payloads_remapped+ )?;+ }+ write!(f, "{}", self.warnings)+ }+}++impl fmt::Display for Warnings {+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {+ let lines = [+ (+ self.unmapped_notifications,+ "document comment notifications still reference a legacy id without a mapping",+ ),+ (+ self.unmapped_pdf_comment_ids,+ "numeric comment ids in saved PDF payloads have no mapping and were left as is",+ ),+ (+ self.invalid_mark_ids,+ "threads carry a mark id that is not a UUID and were imported unanchored",+ ),+ (+ self.root_order_drift,+ "threads whose first legacy comment is no longer the mapped root",+ ),+ ];+ for (count, text) in lines {+ if count > 0 {+ writeln!(f, " warning: {count} {text}")?;+ }+ }+ Ok(())+ }+}++/// Work still ahead of the next run, without writing anything.+#[derive(Debug, Default, Clone, PartialEq, Eq)]+pub struct Pending {+ pub comments_without_mapping: i64,+ pub threads_without_mapping: i64,+ pub documents_with_legacy_comments: i64,+}++impl fmt::Display for Pending {+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {+ writeln!(f, "Preflight passed.")?;+ writeln!(+ f,+ " documents with legacy comments: {}",+ self.documents_with_legacy_comments+ )?;+ writeln!(+ f,+ " comments without a mapping: {}",+ self.comments_without_mapping+ )?;+ writeln!(+ f,+ " threads without a mapping: {}",+ self.threads_without_mapping+ )?;+ write!(+ f,+ "Edits and deletions of already imported comments are only detected by a real run."+ )+ }+}diff --git a/crates/macro_db_migrator/src/bin/comment_import/test.rs b/crates/macro_db_migrator/src/bin/comment_import/test.rsnew file mode 100644index 0000000000..43a2e14943--- /dev/null+++ b/crates/macro_db_migrator/src/bin/comment_import/test.rs@@ -0,0 +1,804 @@+use super::*;+use macro_db_migrator::MACRO_DB_MIGRATIONS;+use serde_json::{Value, json};+use sqlx::PgPool;+use summary::Warnings;++const USER: &str = "macro|[email protected]";++async fn legacy_database(pool: &PgPool) -> PgConnection {+ let mut connection = pool.acquire().await.unwrap().detach();+ sqlx::raw_sql(include_str!("legacy_fixture.sql"))+ .execute(&mut connection)+ .await+ .unwrap();+ connection+}++async fn mappings(connection: &mut PgConnection) -> Vec<(i64, Uuid)> {+ sqlx::query!("SELECT comment_id, message_id FROM migrated_comment_id ORDER BY comment_id")+ .fetch_all(connection)+ .await+ .unwrap()+ .into_iter()+ .map(|row| (row.comment_id, row.message_id))+ .collect()+}++async fn thread_mappings(connection: &mut PgConnection) -> Vec<(i64, Uuid)> {+ sqlx::query!("SELECT thread_id, root_id FROM migrated_comment_thread_id ORDER BY thread_id")+ .fetch_all(connection)+ .await+ .unwrap()+ .into_iter()+ .map(|row| (row.thread_id, row.root_id))+ .collect()+}++fn mapped(mappings: &[(i64, Uuid)], legacy_id: i64) -> Uuid {+ mappings+ .iter()+ .find(|(id, _)| *id == legacy_id)+ .map(|(_, uuid)| *uuid)+ .unwrap()+}++#[derive(Debug, PartialEq)]+struct StoredMessage {+ content: String,+ thread_id: Option<Uuid>,+ imported_author: Option<String>,+ deleted: bool,+ import_order: Option<i64>,+}++async fn message(connection: &mut PgConnection, id: Uuid) -> StoredMessage {+ sqlx::query!(+ r#"SELECT content, thread_id, imported_author, deleted_at IS NOT NULL AS "deleted!", import_order+ FROM comms_messages WHERE id = $1"#,+ id+ )+ .map(|row| StoredMessage {+ content: row.content,+ thread_id: row.thread_id,+ imported_author: row.imported_author,+ deleted: row.deleted,+ import_order: row.import_order,+ })+ .fetch_one(connection)+ .await+ .unwrap()+}++#[derive(Debug, PartialEq)]+struct StoredThread {+ resolved: bool,+ anchor: Option<Value>,+ deleted: bool,+}++async fn thread(connection: &mut PgConnection, root_id: Uuid) -> StoredThread {+ sqlx::query!(+ r#"SELECT resolved, anchor, deleted_at IS NOT NULL AS "deleted!"+ FROM comms_message_threads WHERE root_id = $1"#,+ root_id+ )+ .map(|row| StoredThread {+ resolved: row.resolved,+ anchor: row.anchor,+ deleted: row.deleted,+ })+ .fetch_one(connection)+ .await+ .unwrap()+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn imports_roots_replies_empty_threads_anchors_and_remaps(pool: PgPool) {+ let mut connection = legacy_database(&pool).await;+ let summary = run(&mut connection, Options::default()).await.unwrap();+ assert_eq!(+ summary,+ Summary {+ documents: 2,+ batches: 1,+ comment_mappings_allocated: 5,+ thread_mappings_allocated: 5,+ // Five comments and one structural root for the empty thread.+ messages_inserted: 6,+ messages_updated: 0,+ messages_tombstoned: 0,+ threads_written: 5,+ threads_tombstoned: 0,+ anchors_linked: 2,+ notifications_remapped: 3,+ pdf_payloads_remapped: 1,+ warnings: Warnings {+ unmapped_notifications: 1,+ unmapped_pdf_comment_ids: 1,+ invalid_mark_ids: 0,+ root_order_drift: 0,+ },+ }+ );+ assert!(!summary.is_noop());+ assert!(+ summary+ .to_string()+ .contains("messages inserted: 6")+ );++ let comments = mappings(&mut connection).await;+ let threads = thread_mappings(&mut connection).await;+ assert_eq!(comments.len(), 5);+ assert_eq!(threads.len(), 5);+ assert!(comments.iter().all(|(_, id)| id.get_version_num() == 7));+ assert!(threads.iter().all(|(_, id)| id.get_version_num() == 7));+ // Roots follow legacy order: the deleted root keeps its identity as the thread root.+ assert_eq!(mapped(&threads, 1), mapped(&comments, 10));+ assert_eq!(mapped(&threads, 3), mapped(&comments, 12));+ assert_eq!(mapped(&threads, 4), mapped(&comments, 13));+ assert_eq!(mapped(&threads, 5), mapped(&comments, 14));++ let root = message(&mut connection, mapped(&comments, 10)).await;+ assert_eq!(+ root,+ StoredMessage {+ content: "Deleted root".into(),+ thread_id: None,+ imported_author: Some("Original author".into()),+ deleted: true,+ import_order: Some(1),+ }+ );+ let reply = message(&mut connection, mapped(&comments, 11)).await;+ assert_eq!(+ reply,+ StoredMessage {+ content: "Surviving reply".into(),+ thread_id: Some(mapped(&threads, 1)),+ imported_author: None,+ deleted: false,+ import_order: Some(2),+ }+ );+ let created_at: chrono::DateTime<chrono::Utc> = sqlx::query_scalar!(+ r#"SELECT created_at FROM comms_messages WHERE id = $1"#,+ mapped(&comments, 11)+ )+ .fetch_one(&mut connection)+ .await+ .unwrap();+ assert_eq!(created_at.to_rfc3339(), "2020-01-02T00:00:00+00:00");++ // Markdown mark, resolution, and an unanchored Discussion thread.+ assert_eq!(+ thread(&mut connection, mapped(&threads, 1)).await,+ StoredThread {+ resolved: true,+ anchor: Some(+ json!({"type": "markdown", "mark_id": "01990000-0000-7000-8000-000000000002"})+ ),+ deleted: false,+ }+ );+ assert_eq!(+ thread(&mut connection, mapped(&threads, 5)).await,+ StoredThread {+ resolved: false,+ anchor: None,+ deleted: false,+ }+ );+ // The empty thread is a deleted structural root and a deleted thread.+ let structural = message(&mut connection, mapped(&threads, 2)).await;+ assert!(structural.deleted && structural.content.is_empty() && structural.thread_id.is_none());+ assert!(comments.iter().all(|(_, id)| *id != mapped(&threads, 2)));+ assert!(thread(&mut connection, mapped(&threads, 2)).await.deleted);++ // Notifications point at message ids; the stale one keeps its legacy id.+ let notices: Vec<Value> = sqlx::query_scalar!(+ r#"SELECT metadata AS "metadata!" FROM notification ORDER BY metadata->>'commentText'"#+ )+ .fetch_all(&mut connection)+ .await+ .unwrap();+ assert_eq!(notices.len(), 4);+ assert_eq!(notices[0]["commentId"], 424242);+ for notice in ¬ices[1..] {+ assert_eq!(notice["commentId"], mapped(&comments, 11).to_string());+ assert_eq!(notice["threadId"], mapped(&threads, 1).to_string());+ assert_eq!(notice["commentText"], "Surviving reply");+ }+ let pdf: Value =+ sqlx::query_scalar!(r#"SELECT "modificationData" FROM "DocumentInstanceModificationData""#)+ .fetch_one(&mut connection)+ .await+ .unwrap();+ assert_eq!(+ pdf["pages"][0]["highlights"][0]["comments"],+ json!([+ {"id": mapped(&comments, 12), "text": "PDF original"},+ {"id": "999", "text": "Stale"},+ {"id": "external-pdf-id", "text": "External"}+ ])+ );++ // PDF anchors keep their legacy thread id and gain the root; thread state points back.+ let placeable = sqlx::query!(+ r#"SELECT a."threadId" AS thread_id, a.root_id, a."xPct" AS x_pct, t.anchor+ FROM "PdfPlaceableCommentAnchor" a JOIN comms_message_threads t ON t.root_id = a.root_id"#+ )+ .fetch_one(&mut connection)+ .await+ .unwrap();+ assert_eq!(placeable.thread_id, Some(3));+ assert_eq!(placeable.root_id, Some(mapped(&threads, 3)));+ assert_eq!(placeable.x_pct, 0.1);+ assert_eq!(+ placeable.anchor,+ Some(json!({"type": "pdf_placeable", "anchor_id": "01990000-0000-7000-8000-000000000003"}))+ );+ let highlight = sqlx::query!(+ r#"SELECT a."threadId" AS thread_id, a.root_id, a.text, r.top, t.anchor+ FROM "PdfHighlightAnchor" a JOIN comms_message_threads t ON t.root_id = a.root_id+ JOIN "PdfHighlightRect" r ON r."pdfHighlightAnchorId" = a.uuid"#+ )+ .fetch_one(&mut connection)+ .await+ .unwrap();+ assert_eq!(highlight.thread_id, Some(4));+ assert_eq!(highlight.root_id, Some(mapped(&threads, 4)));+ assert_eq!(highlight.text, "Highlighted text");+ assert_eq!(highlight.top, 0.1);+ assert_eq!(+ highlight.anchor,+ Some(json!({"type": "pdf_highlight", "anchor_id": "01990000-0000-7000-8000-000000000004"}))+ );++ // Legacy rows are untouched and channel messages are not affected.+ let legacy_count: i64 = sqlx::query_scalar!(r#"SELECT count(*) AS "count!" FROM "Comment""#)+ .fetch_one(&mut connection)+ .await+ .unwrap();+ assert_eq!(legacy_count, 5);+ let channel_messages: Vec<(String, Option<Uuid>)> = sqlx::query!(+ "SELECT content, thread_id FROM comms_messages WHERE parent_entity_type = 'channel' ORDER BY id"+ )+ .fetch_all(&mut connection)+ .await+ .unwrap()+ .into_iter()+ .map(|row| (row.content, row.thread_id))+ .collect();+ assert_eq!(+ channel_messages,+ vec![+ ("Channel root".into(), None),+ (+ "Channel reply".into(),+ Some(Uuid::parse_str("01990000-0000-7000-8000-000000000006").unwrap())+ ),+ ]+ );++ // The imported thread accepts new replies and edits through the message store.+ let reply_id = generate_uuid_v7();+ sqlx::query!(+ "INSERT INTO comms_messages(id, parent_entity_type, parent_entity_id, thread_id, sender_id, content)+ VALUES ($1, 'document', 'legacy-md', $2, $3, 'After import')",+ reply_id,+ mapped(&threads, 1),+ USER+ )+ .execute(&mut connection)+ .await+ .unwrap();++ // A second run finds nothing to do and keeps every id.+ let again = run(&mut connection, Options::default()).await.unwrap();+ assert!(again.is_noop(), "{again:?}");+ assert_eq!(again.documents, 2);+ assert!(again.to_string().starts_with("Nothing to import"));+ assert_eq!(again.warnings.unmapped_notifications, 1);+ assert_eq!(mappings(&mut connection).await, comments);+ assert_eq!(thread_mappings(&mut connection).await, threads);+ assert_eq!(+ message(&mut connection, reply_id).await.content,+ "After import"+ );++ let pending = check(&mut connection).await.unwrap();+ assert_eq!(+ pending,+ Pending {+ comments_without_mapping: 0,+ threads_without_mapping: 0,+ documents_with_legacy_comments: 2,+ }+ );+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn delta_runs_carry_new_edited_and_deleted_legacy_comments(pool: PgPool) {+ let mut connection = legacy_database(&pool).await;+ run(&mut connection, Options::default()).await.unwrap();+ let comments = mappings(&mut connection).await;+ let threads = thread_mappings(&mut connection).await;++ // Legacy writers keep going: a new reply, an edit, a comment deletion without+ // an updatedAt bump, a thread deletion, and a comment added to the empty thread.+ sqlx::raw_sql(+ r#"+ INSERT INTO "Comment"(id, "threadId", owner, text, "createdAt", "updatedAt")+ VALUES (15, 1, 'macro|[email protected]', 'Late reply', '2020-02-01', '2020-02-01');+ UPDATE "Thread" SET "updatedAt" = '2020-02-01' WHERE id = 1;+ UPDATE "Comment" SET text = 'Surviving reply, edited', "updatedAt" = '2020-02-02' WHERE id = 11;+ UPDATE "Comment" SET "deletedAt" = '2020-02-03' WHERE id = 14;+ UPDATE "Thread" SET "deletedAt" = '2020-02-04' WHERE id = 4;+ UPDATE "Comment" SET "deletedAt" = '2020-02-04' WHERE "threadId" = 4;+ INSERT INTO "Comment"(id, "threadId", owner, text, "createdAt", "updatedAt")+ VALUES (16, 2, 'macro|[email protected]', 'Revived discussion', '2020-02-05', '2020-02-05');+ UPDATE "Thread" SET "updatedAt" = '2020-02-05' WHERE id = 2;+ "#,+ )+ .execute(&mut connection)+ .await+ .unwrap();++ let delta = run(&mut connection, Options::default()).await.unwrap();+ assert_eq!(delta.comment_mappings_allocated, 2);+ assert_eq!(delta.thread_mappings_allocated, 0);+ assert_eq!(delta.messages_inserted, 2);+ // Edited reply, deleted comment 14, deleted comment 13. The revived thread's+ // structural root keeps its tombstone untouched.+ assert_eq!(delta.messages_updated, 3);+ assert_eq!(delta.messages_tombstoned, 0);+ // Thread 1 (updated_at), thread 4 (deleted), thread 2 (revived) changed.+ assert_eq!(delta.threads_written, 3);+ assert_eq!(delta.warnings.root_order_drift, 1);++ let after = mappings(&mut connection).await;+ assert_eq!(after.len(), 7);+ assert!(after.starts_with(&comments));+ assert_eq!(thread_mappings(&mut connection).await, threads);++ let late = message(&mut connection, mapped(&after, 15)).await;+ assert_eq!(late.content, "Late reply");+ assert_eq!(late.thread_id, Some(mapped(&threads, 1)));+ assert_eq!(late.import_order, Some(3));+ let edited = message(&mut connection, mapped(&after, 11)).await;+ assert_eq!(edited.content, "Surviving reply, edited");+ assert!(!edited.deleted);+ let edited_at: Option<chrono::NaiveDateTime> = sqlx::query_scalar!(+ "SELECT edited_at FROM comms_messages WHERE id = $1",+ mapped(&after, 11)+ )+ .fetch_one(&mut connection)+ .await+ .unwrap();+ assert_eq!(edited_at.unwrap().to_string(), "2020-02-02 00:00:00");+ assert!(message(&mut connection, mapped(&after, 14)).await.deleted);+ assert!(message(&mut connection, mapped(&after, 13)).await.deleted);+ assert!(thread(&mut connection, mapped(&threads, 4)).await.deleted);+ // The highlight keeps its root so the deleted discussion stays addressable.+ let highlight_root: Option<Uuid> =+ sqlx::query_scalar!(r#"SELECT root_id FROM "PdfHighlightAnchor""#)+ .fetch_one(&mut connection)+ .await+ .unwrap();+ assert_eq!(highlight_root, Some(mapped(&threads, 4)));++ // The formerly empty thread is live again: structural root stays a tombstone,+ // the new comment is a reply to it.+ let revived = thread(&mut connection, mapped(&threads, 2)).await;+ assert!(!revived.deleted);+ let revived_comment = message(&mut connection, mapped(&after, 16)).await;+ assert_eq!(revived_comment.thread_id, Some(mapped(&threads, 2)));+ assert!(message(&mut connection, mapped(&threads, 2)).await.deleted);++ let settled = run(&mut connection, Options::default()).await.unwrap();+ assert!(settled.is_noop(), "{settled:?}");+ assert_eq!(settled.warnings.root_order_drift, 1);+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn newer_writes_through_the_message_store_are_never_reverted(pool: PgPool) {+ let mut connection = legacy_database(&pool).await;+ run(&mut connection, Options::default()).await.unwrap();+ let comments = mappings(&mut connection).await;+ let threads = thread_mappings(&mut connection).await;+ let reply = mapped(&comments, 11);++ // The new UI edits the imported reply and resolves the Discussion thread+ // after the legacy rows went stale.+ sqlx::query!(+ "UPDATE comms_messages SET content = 'Edited in the new UI', updated_at = now(), edited_at = now() WHERE id = $1",+ reply+ )+ .execute(&mut connection)+ .await+ .unwrap();+ sqlx::query!(+ "UPDATE comms_message_threads SET resolved = true, updated_at = now() WHERE root_id = $1",+ mapped(&threads, 5)+ )+ .execute(&mut connection)+ .await+ .unwrap();++ let rerun = run(&mut connection, Options::default()).await.unwrap();+ assert!(rerun.is_noop(), "{rerun:?}");+ assert_eq!(+ message(&mut connection, reply).await.content,+ "Edited in the new UI"+ );+ assert!(thread(&mut connection, mapped(&threads, 5)).await.resolved);++ // A legacy edit that is newer than the stored row still wins.+ sqlx::query!(+ r#"UPDATE "Comment" SET text = 'Edited again in legacy', "updatedAt" = now() + interval '1 hour' WHERE id = 11"#+ )+ .execute(&mut connection)+ .await+ .unwrap();+ let legacy_wins = run(&mut connection, Options::default()).await.unwrap();+ assert_eq!(legacy_wins.messages_updated, 1);+ assert_eq!(+ message(&mut connection, reply).await.content,+ "Edited again in legacy"+ );+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn removed_legacy_rows_become_tombstones(pool: PgPool) {+ let mut connection = legacy_database(&pool).await;+ run(&mut connection, Options::default()).await.unwrap();+ let comments = mappings(&mut connection).await;+ let threads = thread_mappings(&mut connection).await;+ sqlx::raw_sql(+ r#"+ DELETE FROM "Comment" WHERE id = 14;+ DELETE FROM "Thread" WHERE id = 3;+ "#,+ )+ .execute(&mut connection)+ .await+ .unwrap();+ let summary = run(&mut connection, Options::default()).await.unwrap();+ // Comment 14 alone, then thread 3 with its single comment.+ assert_eq!(summary.messages_tombstoned, 2);+ assert_eq!(summary.threads_tombstoned, 1);+ assert!(+ message(&mut connection, mapped(&comments, 14))+ .await+ .deleted+ );+ assert!(+ message(&mut connection, mapped(&comments, 12))+ .await+ .deleted+ );+ assert!(thread(&mut connection, mapped(&threads, 3)).await.deleted);+ assert_eq!(mappings(&mut connection).await, comments);+ assert!(+ run(&mut connection, Options::default())+ .await+ .unwrap()+ .is_noop()+ );+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn rejects_concurrent_runner_ambiguous_anchors_and_duplicate_marks(pool: PgPool) {+ let mut connection = legacy_database(&pool).await;+ let mut other = pool.acquire().await.unwrap().detach();+ sqlx::query!("SELECT pg_advisory_lock($1)", LOCK_ID)+ .execute(&mut other)+ .await+ .unwrap();+ assert!(+ run(&mut connection, Options::default())+ .await+ .unwrap_err()+ .to_string()+ .contains("Another comment import")+ );+ other.close().await.unwrap();++ sqlx::query!(r#"UPDATE "PdfHighlightAnchor" SET "threadId" = 3"#)+ .execute(&mut connection)+ .await+ .unwrap();+ assert!(+ run(&mut connection, Options::default())+ .await+ .unwrap_err()+ .to_string()+ .contains("Multiple PDF anchors")+ );+ sqlx::query!(r#"UPDATE "PdfHighlightAnchor" SET "threadId" = 4"#)+ .execute(&mut connection)+ .await+ .unwrap();++ sqlx::query!(+ r#"UPDATE "Thread" SET metadata = '{"markId":"01990000-0000-7000-8000-000000000002"}' WHERE id = 5"#+ )+ .execute(&mut connection)+ .await+ .unwrap();+ let error = run(&mut connection, Options::default())+ .await+ .unwrap_err()+ .to_string();+ assert!(error.contains("share the same Markdown mark"), "{error}");+ assert!(error.contains("[1, 5]"), "{error}");+ assert!(mappings(&mut connection).await.is_empty());++ // The lock is released after a failed preflight.+ let mut probe = other_connection(&pool).await;+ let acquired: bool =+ sqlx::query_scalar!(r#"SELECT pg_try_advisory_lock($1) AS "acquired!""#, LOCK_ID)+ .fetch_one(&mut probe)+ .await+ .unwrap();+ assert!(acquired);+ probe.close().await.unwrap();++ sqlx::query!(r#"UPDATE "Thread" SET metadata = '{"markId":"not-a-mark"}' WHERE id = 5"#)+ .execute(&mut connection)+ .await+ .unwrap();+ let summary = run(&mut connection, Options::default()).await.unwrap();+ assert_eq!(summary.warnings.invalid_mark_ids, 1);+ let threads = thread_mappings(&mut connection).await;+ assert_eq!(+ thread(&mut connection, mapped(&threads, 5)).await.anchor,+ None+ );+}++async fn other_connection(pool: &PgPool) -> PgConnection {+ pool.acquire().await.unwrap().detach()+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn failed_batch_rolls_back_and_can_retry(pool: PgPool) {+ let mut connection = legacy_database(&pool).await;+ // A discussion created through the message store already owns the mark of+ // legacy thread 1, which the preflight cannot see.+ let conflicting_root = generate_uuid_v7();+ sqlx::query!(+ "INSERT INTO comms_messages(id, parent_entity_type, parent_entity_id, sender_id, content)+ VALUES ($1, 'document', 'legacy-md', $2, 'New store discussion')",+ conflicting_root,+ USER+ )+ .execute(&mut connection)+ .await+ .unwrap();+ sqlx::query!(+ r#"UPDATE comms_message_threads SET anchor = '{"type":"markdown","mark_id":"01990000-0000-7000-8000-000000000002"}' WHERE root_id = $1"#,+ conflicting_root+ )+ .execute(&mut connection)+ .await+ .unwrap();++ let error = run(&mut connection, Options::default()).await.unwrap_err();+ assert!(+ error+ .to_string()+ .contains("idx_comms_message_threads_markdown_mark"),+ "{error}"+ );+ assert!(mappings(&mut connection).await.is_empty());+ assert!(thread_mappings(&mut connection).await.is_empty());+ let imported: i64 = sqlx::query_scalar!(+ r#"SELECT count(*) AS "count!" FROM comms_messages WHERE parent_entity_type = 'document'"#+ )+ .fetch_one(&mut connection)+ .await+ .unwrap();+ assert_eq!(imported, 1);+ let legacy_count: i64 = sqlx::query_scalar!(r#"SELECT count(*) AS "count!" FROM "Comment""#)+ .fetch_one(&mut connection)+ .await+ .unwrap();+ assert_eq!(legacy_count, 5);++ sqlx::query!(+ "UPDATE comms_message_threads SET anchor = NULL WHERE root_id = $1",+ conflicting_root+ )+ .execute(&mut connection)+ .await+ .unwrap();+ let summary = run(&mut connection, Options::default()).await.unwrap();+ assert_eq!(summary.comment_mappings_allocated, 5);+ assert_eq!(summary.thread_mappings_allocated, 5);+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn mappings_cross_batch_boundaries_and_keep_ids_on_retry(pool: PgPool) {+ let mut connection = legacy_database(&pool).await;+ let extra = MAPPING_CHUNK as i64 + 100;+ sqlx::query!(+ r#"INSERT INTO "Thread"(id, owner, "documentId", resolved)+ SELECT id, $1, 'legacy-md', false FROM generate_series(100, $2::bigint) id"#,+ USER,+ extra+ )+ .execute(&mut connection)+ .await+ .unwrap();+ sqlx::query!(+ r#"INSERT INTO "Comment"(id, "threadId", owner, text)+ SELECT id, id, $1, 'Batch comment' FROM generate_series(100, $2::bigint) id"#,+ USER,+ extra+ )+ .execute(&mut connection)+ .await+ .unwrap();+ let options = Options {+ documents_per_batch: 1,+ };+ let summary = run(&mut connection, options).await.unwrap();+ assert_eq!(summary.batches, 2);+ assert_eq!(summary.documents, 2);+ assert_eq!(summary.comment_mappings_allocated, MAPPING_CHUNK as u64 + 6);+ let first = mappings(&mut connection).await;+ assert_eq!(first.len(), MAPPING_CHUNK + 6);+ assert!(run(&mut connection, options).await.unwrap().is_noop());+ assert_eq!(mappings(&mut connection).await, first);+ let count: i64 = sqlx::query_scalar!(+ r#"SELECT count(*) AS "count!" FROM comms_message_threads WHERE parent_entity_type = 'document'"#+ )+ .fetch_one(&mut connection)+ .await+ .unwrap();+ assert_eq!(count, MAPPING_CHUNK as i64 + 6);+}++#[test]+fn pdf_comment_ids_are_remapped_at_any_depth_and_unmapped_ids_are_counted() {+ let mapped_id = Uuid::from_u128(12);+ let mappings = HashMap::from([(12_i64, mapped_id)]);+ let mut payload = json!({+ "placeables": [{"payload": {"comments": [+ {"id": "12", "content": "a"},+ {"id": "13", "content": "b"},+ {"id": "wrapper", "comments": [{"id": 12}]}+ ]}}],+ "highlights": {"1": [{"thread": {"comments": [{"id": 12}, {"id": "external"}, "not an object"]}}]},+ "comments": "not an array",+ });+ let stats = remap_comment_ids(&mut payload, &mappings);+ // Two top-level 12s plus the 12 nested inside the "wrapper" comment.+ assert_eq!(+ stats,+ RemapStats {+ remapped: 3,+ unmapped: 1,+ }+ );+ assert_eq!(+ payload["placeables"][0]["payload"]["comments"][2]["comments"][0]["id"],+ mapped_id.to_string()+ );+ assert_eq!(+ payload["placeables"][0]["payload"]["comments"][0]["id"],+ mapped_id.to_string()+ );+ assert_eq!(+ payload["placeables"][0]["payload"]["comments"][1]["id"],+ "13"+ );+ assert_eq!(+ payload["highlights"]["1"][0]["thread"]["comments"][0]["id"],+ mapped_id.to_string()+ );+ assert_eq!(+ payload["highlights"]["1"][0]["thread"]["comments"][1]["id"],+ "external"+ );+ // A second pass rewrites nothing and still reports the id that has no mapping.+ assert_eq!(+ remap_comment_ids(&mut payload, &mappings),+ RemapStats {+ remapped: 0,+ unmapped: 1,+ }+ );+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn channel_read_projections_keep_channel_scoped_index_plans(pool: PgPool) {+ let mut connection = legacy_database(&pool).await;+ run(&mut connection, Options::default()).await.unwrap();+ // A small target channel among many unrelated channels and non-UUID documents.+ // Never disable sequential scans: test the planner's actual chosen access path.+ sqlx::raw_sql(+ r#"+ INSERT INTO comms_channels(id, name, channel_type, owner_id)+ SELECT md5('plan-channel-' || n)::uuid, 'Plan fixture', 'private', 'macro|[email protected]'+ FROM generate_series(1, 200) n;+ INSERT INTO comms_messages(id, parent_entity_type, parent_entity_id, sender_id, content, created_at)+ SELECT md5('plan-message-' || c || '-' || n)::uuid, 'channel', md5('plan-channel-' || c)::uuid::text,+ 'macro|[email protected]', 'Unrelated channel', now() - n * interval '1 second'+ FROM generate_series(1, 200) c CROSS JOIN generate_series(1, 100) n;+ INSERT INTO comms_messages(id, parent_entity_type, parent_entity_id, sender_id, content)+ SELECT md5('plan-document-' || n)::uuid, 'document', 'legacy-md',+ 'macro|[email protected]', 'Unrelated document'+ FROM generate_series(1, 10000) n;+ ANALYZE comms_messages;+ "#,+ )+ .execute(&mut connection)+ .await+ .unwrap();+ let channel = Uuid::parse_str("01990000-0000-7000-8000-000000000005").unwrap();+ let queries = [+ // Timeline includes tombstone roots with surviving replies, in either direction.+ "SELECT m.id FROM comms_messages m WHERE m.parent_entity_type = 'channel' AND m.parent_entity_id = $1::uuid::text AND m.thread_id IS NULL+ AND (m.deleted_at IS NULL OR EXISTS (SELECT 1 FROM comms_messages r WHERE r.thread_id = m.id AND r.deleted_at IS NULL))+ ORDER BY m.created_at DESC, m.id DESC LIMIT 50",+ "SELECT m.id FROM comms_messages m WHERE m.parent_entity_type = 'channel' AND m.parent_entity_id = $1::uuid::text AND m.thread_id IS NULL+ AND (m.deleted_at IS NULL OR EXISTS (SELECT 1 FROM comms_messages r WHERE r.thread_id = m.id AND r.deleted_at IS NULL))+ ORDER BY m.created_at ASC, m.id ASC LIMIT 50",+ // Batched latest reads use a parameterized lateral scan for each channel.+ "SELECT latest.id FROM unnest(ARRAY[$1::uuid]) i(channel_id) LEFT JOIN LATERAL (+ SELECT m.id FROM comms_messages m WHERE m.parent_entity_type = 'channel' AND m.parent_entity_id = i.channel_id::text AND m.deleted_at IS NULL+ ORDER BY m.created_at DESC LIMIT 1) latest ON true",+ "SELECT latest.id FROM unnest(ARRAY[$1::uuid]) i(channel_id) LEFT JOIN LATERAL (+ SELECT m.id FROM comms_messages m WHERE m.parent_entity_type = 'channel' AND m.parent_entity_id = i.channel_id::text AND m.deleted_at IS NULL AND m.thread_id IS NULL+ ORDER BY m.created_at DESC LIMIT 1) latest ON true",+ // Unfiltered reads still need a channel index, including deleted replies.+ "SELECT m.id FROM comms_messages m WHERE m.parent_entity_type = 'channel' AND m.parent_entity_id = $1::uuid::text ORDER BY m.created_at DESC LIMIT 50",+ ];+ for query in queries {+ let plan = explain(&mut connection, query, channel).await;+ let mut nodes = vec![];+ collect_plan_nodes(&plan[0]["Plan"], &mut nodes);+ assert!(+ nodes.iter().any(|node| {+ node["Index Name"]+ .as_str()+ .is_some_and(|name| name.starts_with("idx_comms_messages_parent_"))+ && node["Index Cond"]+ .as_str()+ .is_some_and(|condition| condition.contains("parent_entity_id"))+ }),+ "Channel predicate must be an index condition, not a post-scan filter: {plan:#}"+ );+ assert!(+ nodes+ .iter()+ .all(|node| node["Rows Removed by Filter"].as_u64().unwrap_or(0) < 100),+ "Channel reads must not filter thousands of unrelated messages: {plan:#}"+ );+ }+}++#[expect(+ clippy::disallowed_methods,+ reason = "EXPLAIN over test-provided SQL cannot be checked at compile time"+)]+async fn explain(connection: &mut PgConnection, query: &str, channel: Uuid) -> Value {+ sqlx::query_scalar(&format!("EXPLAIN (ANALYZE, FORMAT JSON) {query}"))+ .bind(channel)+ .fetch_one(connection)+ .await+ .unwrap()+}++fn collect_plan_nodes<'a>(node: &'a Value, nodes: &mut Vec<&'a Value>) {+ nodes.push(node);+ if let Some(children) = node["Plans"].as_array() {+ for child in children {+ collect_plan_nodes(child, nodes);+ }+ }+}diff --git a/crates/messages/src/outbound/pg_message_repo/test.rs b/crates/messages/src/outbound/pg_message_repo/test.rsindex caeb475b4e..6109c5864e 100644--- a/crates/messages/src/outbound/pg_message_repo/test.rs+++ b/crates/messages/src/outbound/pg_message_repo/test.rs@@ -1082,3 +1082,69 @@ async fn both_parents_use_bounded_previews_and_bidirectional_windows(pool: PgPoo);}}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn legacy_ids_resolve_through_the_import_mapping_tables(pool: PgPool) {+ setup(&pool).await;+ let repo = PgMessageRepository::new(pool.clone());+ let parent = MessageParent::parse("document", "message-doc-a").unwrap();+ let root = repo+ .create(command("message-doc-a", None, "imported root"))+ .await+ .unwrap();+ let reply = repo+ .create(command("message-doc-a", Some(root.id), "imported reply"))+ .await+ .unwrap();+ sqlx::query!(+ "INSERT INTO migrated_comment_thread_id (thread_id, root_id, document_id) VALUES (1, $1, 'message-doc-a')",+ root.id+ )+ .execute(&pool)+ .await+ .unwrap();+ sqlx::query!(+ "INSERT INTO migrated_comment_id (comment_id, message_id, document_id)+ VALUES (10, $1, 'message-doc-a'), (11, $2, 'message-doc-a')",+ root.id,+ reply.id+ )+ .execute(&pool)+ .await+ .unwrap();++ assert_eq!(+ repo.resolve_legacy(&parent, 11, false).await.unwrap(),+ Some(reply.id)+ );+ assert_eq!(+ repo.resolve_legacy(&parent, 10, false).await.unwrap(),+ Some(root.id)+ );+ assert_eq!(+ repo.resolve_legacy(&parent, 1, true).await.unwrap(),+ Some(root.id)+ );+ // A comment id is not a thread id, and a mapping only resolves under its own document.+ assert_eq!(repo.resolve_legacy(&parent, 11, true).await.unwrap(), None);+ let other = MessageParent::parse("document", "message-doc-b").unwrap();+ assert_eq!(repo.resolve_legacy(&other, 11, false).await.unwrap(), None);+ let channel = MessageParent::parse("channel", &Uuid::from_u128(7).to_string()).unwrap();+ assert_eq!(+ repo.resolve_legacy(&channel, 11, false).await.unwrap(),+ None+ );++ // Mappings cannot point at messages that were never written.+ let dangling = sqlx::query!(+ "INSERT INTO migrated_comment_id (comment_id, message_id, document_id) VALUES (12, $1, 'message-doc-a')",+ macro_uuid::generate_uuid_v7()+ )+ .execute(&pool)+ .await+ .unwrap_err();+ assert_eq!(+ dangling.as_database_error().unwrap().code().as_deref(),+ Some("23503")+ );+}diff --git a/docs/legacy-comment-import.md b/docs/legacy-comment-import.mdnew file mode 100644index 0000000000..02b2b8eac2--- /dev/null+++ b/docs/legacy-comment-import.md@@ -0,0 +1,138 @@+# Legacy comment import++Moves the historical document comments in `"Comment"` / `"Thread"` into the+shared message store (`comms_messages`, `comms_message_threads`) while the+application keeps running. Nothing is dropped; the legacy tables and endpoints+go away in the last PR of the channel and comment unification stack.++## Operating model++The importer is a projection: each run recomputes what every legacy thread and+comment should look like in the message store and writes only the rows whose+projection differs. That gives three properties operators rely on:++- **Re-runnable.** A second run without legacy changes prints+ `Nothing to import` and writes nothing. Ids never change between runs: every+ comment and thread gets one UUIDv7 in `migrated_comment_id` /+ `migrated_comment_thread_id` on its first sighting and keeps it.+- **Delta-aware.** Comments created, edited, or deleted (soft or hard) through+ the legacy endpoints after a run are picked up by the next run. New comments+ get mappings; changed ones are compared by text, timestamps, and deletion.+- **Live.** Work happens in one transaction per batch of documents (250 by+ default). Each batch takes a `SHARE ROW EXCLUSIVE` lock on `"Comment"` and+ `"Thread"` for the duration of that transaction, so legacy comment writes+ wait for at most one batch; they are never rejected. Reads are unaffected.+ The lock waits at most five seconds for in-flight writers and the batch is+ retried. Nothing takes `ACCESS EXCLUSIVE`.++Two runs cannot overlap: a session advisory lock makes the second exit with+`Another comment import is running`.++### Sequence++1. Run the importer as often as you like while legacy writes continue.+ Each run reports what it wrote and what it could not represent.+2. Before enabling the new document discussion UI, freeze legacy comment+ writes: set `LEGACY_COMMENT_WRITES_ENABLED=false` on the document storage+ service (Doppler project `cloud-storage-service`) and roll it out. Comment+ create, edit, and delete and anchor delete then answer `503` with+ `Document comments are read-only while they move to the new message store. Refresh the app in a few minutes and try again.`.+ Reads, highlight creation, and anchor edits keep working.+3. Run the importer once more. It must end with `Nothing to import` on a+ second invocation.+4. Enable the new UI. Do not run the importer again after that unless legacy+ writes are frozen; an edit made through the new API is never reverted by a+ stale legacy row (newer `updated_at` wins), but a legacy write and a new+ store write on the same thread while both are live is a race.++## Running it++Build from the repository root; the binary is feature gated so it never ships+with a service:++```sh+cargo build -p macro_db_migrator --features comment-import --bin comment_import+```++Point `DATABASE_URL` at the target. For RDS use the existing helper and never+the repository's local database recipes:++```sh+export DATABASE_URL="$(tooling/scripts/rds_database_url.sh <rds_identifier> <secret_name> <db_name>)"+cargo run -p macro_db_migrator --features comment-import --bin comment_import -- --check+cargo run -p macro_db_migrator --features comment-import --bin comment_import+cargo run -p macro_db_migrator --features comment-import --bin comment_import -- --documents-per-batch 50+```++`--check` runs the preflight and prints how many comments and threads still+lack a mapping without writing. Edits and deletions of already imported+comments are only detected by a real run.++## What a run writes++- `migrated_comment_id` / `migrated_comment_thread_id`: one row per legacy+ comment and thread. Roots follow legacy display `order`, then `createdAt`,+ then id, including deleted roots. An empty thread gets a fresh root id.+- `comms_messages`: one row per comment with `parent_entity_type = 'document'`,+ the thread's root as `thread_id`, `sender_id` from `owner`, `imported_author`+ from `sender`, `import_metadata` from the comment metadata, and+ `import_order` from the legacy order. A deleted root stays a tombstone with+ its replies intact. An empty thread gets an empty, deleted structural root.+- `comms_message_threads`: resolution, legacy metadata, and the anchor:+ `pdf_placeable` when a placeable anchor points at the thread, else+ `pdf_highlight` for a live highlight, else `markdown` when the thread's+ `markId` is a UUID. `DISCUSSION:` marks, missing marks, and marks that are+ not UUIDs land unanchored. Threads with no comments are marked deleted, so+ the new UI never shows an empty "This message was deleted." discussion.+- `"PdfPlaceableCommentAnchor".root_id` / `"PdfHighlightAnchor".root_id` from+ the thread mapping; the legacy `"threadId"` stays until the contract PR.+- `notification.metadata` `commentId` / `threadId` for+ `commented_on_document`, `replied_to_document_comment_thread`, and+ `mentioned_in_document_comment` rows on the batch's documents.+- Numeric `comments[].id` values inside saved PDF+ `"DocumentInstanceModificationData"` payloads.++Rows written through the message store after import are protected: a legacy+row only overwrites a message or thread state when it is at least as new as the+stored `updated_at`. Legacy deletions do not bump `updatedAt`, so they still+apply.++## Preflight failures++The run stops before writing when the legacy data cannot be represented:++- A live thread with more than one PDF anchor. Detach or delete one.+- Two live legacy threads with comments sharing the same Markdown mark on one+ document. Delete the duplicate; the message store allows one active+ discussion per mark.++A batch that fails for another reason (for example a discussion created through+the new API already owns a legacy thread's mark) rolls back that batch alone.+Earlier batches stay committed and the failed one is retried on the next run.++## Warnings++Warnings are reported and never abort a run:++- notifications whose `commentId` or `threadId` has no mapping (the comment was+ removed before the import existed); they keep their legacy ids.+- numeric PDF comment ids without a mapping; they are left as they are.+- threads whose mark is neither a UUID nor a `DISCUSSION:` mark; imported+ unanchored.+- threads whose first legacy comment is no longer the mapped root; roots are+ fixed at first import, later comments are replies.++## Tests++From the repository root with `SQLX_OFFLINE` unset and local Postgres running:++```sh+cargo test -p macro_db_migrator --features comment-import+```++The suite seeds the legacy tables on the current schema and covers roots,+replies, deleted roots, empty threads, Markdown and PDF anchors, notification+and PDF payload remaps, the no-op second run, delta runs, protection of newer+message store writes, tombstones for removed legacy rows, concurrent runners,+preflight aborts, rollback of a failed batch, mapping batches beyond 1,000 rows,+and the parent-scoped index plans for channel reads.diff --git a/services/document_storage_service/src/api/annotations.rs b/services/document_storage_service/src/api/annotations.rsindex e602d571b4..6fda73daff 100644--- a/services/document_storage_service/src/api/annotations.rs+++ b/services/document_storage_service/src/api/annotations.rs@@ -5,6 +5,7 @@ pub mod delete_comment;pub mod edit_anchor;pub mod edit_comment;pub mod get;+pub mod write_switch;use std::collections::HashSet;@@ -27,6 +28,12 @@ use notification::domain::models::SendNotificationRequestBuilder;use tower::ServiceBuilder;pub fn router(state: ApiContext) -> Router<ApiContext> {+ let write_switch = axum::middleware::from_fn_with_state(+ write_switch::LegacyCommentWrites {+ enabled: state.config.legacy_comment_writes_enabled,+ },+ write_switch::handler,+ );Router::new().route("/comments/document/{document_id}",@@ -39,22 +46,27 @@ pub fn router(state: ApiContext) -> Router<ApiContext> {).route("/comments/document/{document_id}",- post(create_comment::create_comment_handler).layer(ServiceBuilder::new().layer(- axum::middleware::from_fn_with_state(- state.clone(),- macro_middleware::cloud_storage::document::ensure_document_exists::handler,+ post(create_comment::create_comment_handler).layer(+ ServiceBuilder::new().layer(write_switch.clone()).layer(+ axum::middleware::from_fn_with_state(+ state.clone(),+ macro_middleware::cloud_storage::document::ensure_document_exists::handler,+ ),),- )),+ ),).route("/comments/comment/{comment_id}",- delete(delete_comment::delete_comment_handler),+ delete(delete_comment::delete_comment_handler).layer(write_switch.clone()),+ )+ .route(+ "/anchors",+ delete(delete_anchor::delete_anchor_handler).layer(write_switch.clone()),)- .route("/anchors", delete(delete_anchor::delete_anchor_handler)).route("/anchors", patch(edit_anchor::edit_anchor_handler)).route("/comments/comment/{comment_id}",- patch(edit_comment::edit_comment_handler),+ patch(edit_comment::edit_comment_handler).layer(write_switch),).route("/anchors/document/{document_id}",diff --git a/services/document_storage_service/src/api/annotations/write_switch.rs b/services/document_storage_service/src/api/annotations/write_switch.rsnew file mode 100644index 0000000000..fe3e3fc89f--- /dev/null+++ b/services/document_storage_service/src/api/annotations/write_switch.rs@@ -0,0 +1,39 @@+use axum::{+ Json,+ extract::{Request, State},+ http::StatusCode,+ middleware::Next,+ response::{IntoResponse, Response},+};+use model::response::ErrorResponse;++/// Whether the legacy comment tables still accept writes through this service.+#[derive(Debug, Clone, Copy)]+pub struct LegacyCommentWrites {+ pub enabled: bool,+}++/// Shown to clients that still write through the legacy comment endpoints+/// while the final import into the shared message store runs.+pub const DISABLED_MESSAGE: &str = "Document comments are read-only while they move to the new message store. Refresh the app in a few minutes and try again.";++/// Answers 503 for legacy comment writes once the operator has frozen them.+pub async fn handler(+ State(switch): State<LegacyCommentWrites>,+ request: Request,+ next: Next,+) -> Response {+ if switch.enabled {+ return next.run(request).await;+ }+ (+ StatusCode::SERVICE_UNAVAILABLE,+ Json(ErrorResponse {+ message: DISABLED_MESSAGE.into(),+ }),+ )+ .into_response()+}++#[cfg(test)]+mod test;diff --git a/services/document_storage_service/src/api/annotations/write_switch/test.rs b/services/document_storage_service/src/api/annotations/write_switch/test.rsnew file mode 100644index 0000000000..79beece009--- /dev/null+++ b/services/document_storage_service/src/api/annotations/write_switch/test.rs@@ -0,0 +1,37 @@+use super::*;+use axum::{Router, body::Body, http::Request, routing::post};+use tower::ServiceExt as _;++fn router(enabled: bool) -> Router {+ Router::new()+ .route("/comments", post(|| async { StatusCode::OK }))+ .layer(axum::middleware::from_fn_with_state(+ LegacyCommentWrites { enabled },+ handler,+ ))+}++fn write_request() -> Request<Body> {+ Request::builder()+ .method("POST")+ .uri("/comments")+ .body(Body::empty())+ .unwrap()+}++#[tokio::test]+async fn enabled_switch_passes_writes_through() {+ let response = router(true).oneshot(write_request()).await.unwrap();+ assert_eq!(response.status(), StatusCode::OK);+}++#[tokio::test]+async fn disabled_switch_answers_service_unavailable_with_the_reason() {+ let response = router(false).oneshot(write_request()).await.unwrap();+ assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);+ let body = axum::body::to_bytes(response.into_body(), usize::MAX)+ .await+ .unwrap();+ let error: ErrorResponse = serde_json::from_slice(&body).unwrap();+ assert_eq!(error.message, DISABLED_MESSAGE);+}diff --git a/services/document_storage_service/src/config.rs b/services/document_storage_service/src/config.rsindex 5125e64102..b4e4768745 100644--- a/services/document_storage_service/src/config.rs+++ b/services/document_storage_service/src/config.rs@@ -157,6 +157,14 @@ pub struct Config {#[macro_config_default(false)]pub calendar_reminder_dispatch_enabled: bool,+ /// Master switch for the legacy document comment writers: comment create,+ /// edit and delete, and anchor delete, which also deletes the thread.+ /// Set to `false` for the final pass of the legacy comment importer, before+ /// the new document discussion UI is enabled; those handlers then answer+ /// 503 and the importer works from a frozen source.+ #[macro_config_default(true)]+ pub legacy_comment_writes_enabled: bool,+/// The number of seconds a signed document or call recording URL is valid for.#[macro_config_default(DEFAULT_PRESIGNED_URL_EXPIRY_SECONDS)]pub document_storage_service_presigned_url_expiry_seconds: u64,