Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
perf(crm): read one page per companies-list call instead of the whole team (#7570)
teo2f0dabe950dc
diff --git a/.sqlx/query-1050af5f7932e65db18f613cd12e2dce03b99ccd25a98625d98979d2182831ba.json b/.sqlx/query-1050af5f7932e65db18f613cd12e2dce03b99ccd25a98625d98979d2182831ba.jsonnew file mode 100644index 0000000000..6776688dd2--- /dev/null+++ b/.sqlx/query-1050af5f7932e65db18f613cd12e2dce03b99ccd25a98625d98979d2182831ba.json@@ -0,0 +1,95 @@+{+ "db_name": "PostgreSQL",+ "query": "-- Each sort reads the companies this user has viewed from their history, and\n-- the rest of the team from an index in the sort's order, stopping after $3\n-- rows either way. Branch conditions on parameters alone are checked once per\n-- execution, so only the requested sort's branches run, also under a cached\n-- generic plan. A single ORDER BY CASE $4 reads and sorts the whole team.\n--\n-- Viewed companies get their own branch because a generic plan can look up a\n-- company's view time by scanning the user's whole history for each company.\n-- Only the explicit-ids branch, which reads few companies, looks views up per\n-- company.\nWITH limited_companies AS (\n SELECT *\n FROM (\n -- Explicit ids are few, so this branch has no index order or limit.\n (\n SELECT\n c.id, c.team_id, c.custom_name, c.email_sync, c.hidden,\n c.first_interaction, c.last_interaction,\n CASE $4\n WHEN 'created_at' THEN c.first_interaction\n WHEN 'viewed_at' THEN uh.\"updatedAt\"\n WHEN 'viewed_updated'\n THEN COALESCE(uh.\"updatedAt\", c.last_interaction)\n ELSE c.last_interaction\n END AS sort_ts,\n uh.\"updatedAt\"::timestamptz AS viewed_at\n FROM crm_companies c\n LEFT JOIN \"UserHistory\" uh\n ON uh.\"itemId\" = c.id::text\n AND uh.\"itemType\" = 'crm_company'\n AND uh.\"userId\" = $8\n WHERE cardinality($2::uuid[]) > 0\n AND c.id = ANY($2::uuid[])\n AND c.team_id = $1\n AND c.hidden = COALESCE($5::bool, FALSE)\n )\n UNION ALL\n -- Companies the user has viewed, under every sort. Bounded by the size\n -- of the user's company history rather than the team: each history\n -- row looks up its company. As a plain join, a generic plan that\n -- expects a small team and a long history reads and sorts the whole\n -- team for a merge join instead. The LIMIT keeps the lookup per row;\n -- ids are unique, so it drops nothing.\n (\n SELECT *\n FROM (\n SELECT\n c.id, c.team_id, c.custom_name, c.email_sync, c.hidden,\n c.first_interaction, c.last_interaction,\n CASE $4\n WHEN 'created_at' THEN c.first_interaction\n WHEN 'updated_at' THEN c.last_interaction\n ELSE uh.\"updatedAt\"::timestamptz\n END AS sort_ts,\n uh.\"updatedAt\"::timestamptz AS viewed_at\n FROM \"UserHistory\" uh\n CROSS JOIN LATERAL (\n SELECT\n c.id, c.team_id, c.custom_name, c.email_sync, c.hidden,\n c.first_interaction, c.last_interaction\n FROM crm_companies c\n WHERE c.id::text = uh.\"itemId\"\n AND c.team_id = $1\n AND c.hidden = COALESCE($5::bool, FALSE)\n LIMIT 1\n ) c\n WHERE cardinality($2::uuid[]) = 0\n AND uh.\"userId\" = $8\n AND uh.\"itemType\" = 'crm_company'\n ) viewed\n WHERE $6::timestamptz IS NULL OR (sort_ts, id::text) < ($6, $7)\n ORDER BY sort_ts DESC NULLS LAST, id DESC\n LIMIT $3\n )\n UNION ALL\n -- The remaining branches read unviewed companies. NOT IN rather than\n -- NOT EXISTS: \"itemId\" is NOT NULL, so they are equivalent, and NOT IN\n -- is planned as one hashed lookup of the user's history instead of a\n -- scan of it per company.\n --\n -- The seek bound repeats the keyset's leading column so the index\n -- scan starts at the cursor. Both interaction columns are NOT NULL,\n -- so it drops no rows; the row comparison stays exact.\n --\n -- Unviewed companies sort by last_interaction under viewed_updated too.\n (\n SELECT\n c.id, c.team_id, c.custom_name, c.email_sync, c.hidden,\n c.first_interaction, c.last_interaction,\n c.last_interaction,\n NULL::timestamptz\n FROM crm_companies c\n WHERE cardinality($2::uuid[]) = 0\n AND $4 IN ('updated_at', 'viewed_updated')\n AND c.team_id = $1\n AND c.hidden = COALESCE($5::bool, FALSE)\n AND c.last_interaction <= COALESCE($6::timestamptz, 'infinity')\n AND ($6::timestamptz IS NULL OR (c.last_interaction, c.id::text) < ($6, $7))\n AND c.id::text NOT IN (\n SELECT uh.\"itemId\" FROM \"UserHistory\" uh\n WHERE uh.\"userId\" = $8\n AND uh.\"itemType\" = 'crm_company'\n )\n ORDER BY c.last_interaction DESC NULLS LAST, c.id DESC\n LIMIT $3\n )\n UNION ALL\n (\n SELECT\n c.id, c.team_id, c.custom_name, c.email_sync, c.hidden,\n c.first_interaction, c.last_interaction,\n c.first_interaction,\n NULL::timestamptz\n FROM crm_companies c\n WHERE cardinality($2::uuid[]) = 0\n AND $4 = 'created_at'\n AND c.team_id = $1\n AND c.hidden = COALESCE($5::bool, FALSE)\n AND c.first_interaction <= COALESCE($6::timestamptz, 'infinity')\n AND ($6::timestamptz IS NULL OR (c.first_interaction, c.id::text) < ($6, $7))\n AND c.id::text NOT IN (\n SELECT uh.\"itemId\" FROM \"UserHistory\" uh\n WHERE uh.\"userId\" = $8\n AND uh.\"itemType\" = 'crm_company'\n )\n ORDER BY c.first_interaction DESC NULLS LAST, c.id DESC\n LIMIT $3\n )\n UNION ALL\n -- Unviewed companies sort as NULL under viewed_at. NULL never passes a\n -- cursor, so they only appear on a first page.\n (\n SELECT\n c.id, c.team_id, c.custom_name, c.email_sync, c.hidden,\n c.first_interaction, c.last_interaction,\n NULL::timestamptz,\n NULL::timestamptz\n FROM crm_companies c\n WHERE cardinality($2::uuid[]) = 0\n AND $4 = 'viewed_at'\n AND $6::timestamptz IS NULL\n AND c.team_id = $1\n AND c.hidden = COALESCE($5::bool, FALSE)\n AND c.id::text NOT IN (\n SELECT uh.\"itemId\" FROM \"UserHistory\" uh\n WHERE uh.\"userId\" = $8\n AND uh.\"itemType\" = 'crm_company'\n )\n ORDER BY c.id DESC\n LIMIT $3\n )\n ) candidates\n WHERE EXISTS (\n SELECT 1 FROM team_crm_settings tcs\n WHERE tcs.team_id = $1 AND tcs.crm_enabled\n )\n -- Keyset seek (NULL = first page): keep only rows that sort strictly\n -- after the cursor.\n AND ($6::timestamptz IS NULL OR (sort_ts, id::text) < ($6, $7))\n ORDER BY sort_ts DESC NULLS LAST, id DESC\n LIMIT $3\n)\nSELECT\n lc.id AS \"company_id!\",\n lc.team_id AS \"company_team_id!\",\n lc.email_sync AS \"company_email_sync!\",\n lc.hidden AS \"company_hidden!\",\n lc.first_interaction AS \"company_created_at!\",\n lc.last_interaction AS \"company_updated_at!\",\n d.id AS \"domain_id?\",\n d.domain AS \"domain?\",\n d.created_at AS \"domain_created_at?\",\n COALESCE(lc.custom_name, dd.name) AS \"display_name?\",\n dd.description AS \"dir_description?\",\n lc.viewed_at AS \"viewed_at?\"\nFROM limited_companies lc\nLEFT JOIN crm_domains d ON d.company_id = lc.id\nLEFT JOIN crm_domain_directory dd\n ON LOWER(dd.domain) = LOWER(d.domain)\nORDER BY\n lc.sort_ts DESC NULLS LAST,\n lc.id DESC,\n d.created_at ASC NULLS LAST\n",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "company_id!",+ "type_info": "Uuid"+ },+ {+ "ordinal": 1,+ "name": "company_team_id!",+ "type_info": "Uuid"+ },+ {+ "ordinal": 2,+ "name": "company_email_sync!",+ "type_info": "Bool"+ },+ {+ "ordinal": 3,+ "name": "company_hidden!",+ "type_info": "Bool"+ },+ {+ "ordinal": 4,+ "name": "company_created_at!",+ "type_info": "Timestamptz"+ },+ {+ "ordinal": 5,+ "name": "company_updated_at!",+ "type_info": "Timestamptz"+ },+ {+ "ordinal": 6,+ "name": "domain_id?",+ "type_info": "Uuid"+ },+ {+ "ordinal": 7,+ "name": "domain?",+ "type_info": "Text"+ },+ {+ "ordinal": 8,+ "name": "domain_created_at?",+ "type_info": "Timestamptz"+ },+ {+ "ordinal": 9,+ "name": "display_name?",+ "type_info": "Text"+ },+ {+ "ordinal": 10,+ "name": "dir_description?",+ "type_info": "Text"+ },+ {+ "ordinal": 11,+ "name": "viewed_at?",+ "type_info": "Timestamptz"+ }+ ],+ "parameters": {+ "Left": [+ "Uuid",+ "UuidArray",+ "Int8",+ "Text",+ "Bool",+ "Timestamptz",+ "Text",+ "Text"+ ]+ },+ "nullable": [+ null,+ null,+ null,+ null,+ null,+ null,+ false,+ false,+ false,+ null,+ true,+ null+ ]+ },+ "hash": "1050af5f7932e65db18f613cd12e2dce03b99ccd25a98625d98979d2182831ba"+}diff --git a/.sqlx/query-9d46745c3514bf518340dd31f909163c257890c5008bfbdb5db4d2b7aa01ac35.json b/.sqlx/query-9d46745c3514bf518340dd31f909163c257890c5008bfbdb5db4d2b7aa01ac35.jsondeleted file mode 100644index 8fd31b5c8a..0000000000--- a/.sqlx/query-9d46745c3514bf518340dd31f909163c257890c5008bfbdb5db4d2b7aa01ac35.json+++ /dev/null@@ -1,95 +0,0 @@-{- "db_name": "PostgreSQL",- "query": "\n WITH limited_companies AS (\n SELECT\n c.id,\n c.team_id,\n c.custom_name,\n c.email_sync,\n c.hidden,\n c.first_interaction,\n c.last_interaction\n FROM crm_companies c\n LEFT JOIN \"UserHistory\" uh\n ON uh.\"itemId\" = c.id::text\n AND uh.\"itemType\" = 'crm_company'\n AND uh.\"userId\" = $8\n WHERE c.team_id = $1\n AND c.hidden = COALESCE($5::bool, FALSE)\n AND EXISTS (\n SELECT 1 FROM team_crm_settings tcs\n WHERE tcs.team_id = $1 AND tcs.crm_enabled\n )\n AND (cardinality($2::uuid[]) = 0 OR c.id = ANY($2::uuid[]))\n -- Keyset seek (NULL = first page): keep only rows that\n -- sort strictly after the cursor.\n AND (\n $6::timestamptz IS NULL\n OR (\n CASE $4\n WHEN 'created_at' THEN c.first_interaction\n WHEN 'viewed_at' THEN uh.\"updatedAt\"\n WHEN 'viewed_updated'\n THEN COALESCE(uh.\"updatedAt\", c.last_interaction)\n ELSE c.last_interaction\n END,\n c.id::text\n ) < ($6, $7)\n )\n ORDER BY\n CASE $4\n WHEN 'created_at' THEN c.first_interaction\n WHEN 'viewed_at' THEN uh.\"updatedAt\"\n WHEN 'viewed_updated'\n THEN COALESCE(uh.\"updatedAt\", c.last_interaction)\n ELSE c.last_interaction\n END DESC NULLS LAST,\n c.id DESC\n LIMIT $3\n )\n SELECT\n lc.id AS \"company_id!\",\n lc.team_id AS \"company_team_id!\",\n lc.email_sync AS \"company_email_sync!\",\n lc.hidden AS \"company_hidden!\",\n lc.first_interaction AS \"company_created_at!\",\n lc.last_interaction AS \"company_updated_at!\",\n d.id AS \"domain_id?\",\n d.domain AS \"domain?\",\n d.created_at AS \"domain_created_at?\",\n COALESCE(lc.custom_name, dd.name) AS \"display_name?\",\n dd.description AS \"dir_description?\",\n uh.\"updatedAt\"::timestamptz AS \"viewed_at?\"\n FROM limited_companies lc\n LEFT JOIN \"UserHistory\" uh\n ON uh.\"itemId\" = lc.id::text\n AND uh.\"itemType\" = 'crm_company'\n AND uh.\"userId\" = $8\n LEFT JOIN crm_domains d ON d.company_id = lc.id\n LEFT JOIN crm_domain_directory dd\n ON LOWER(dd.domain) = LOWER(d.domain)\n ORDER BY\n CASE $4\n WHEN 'created_at' THEN lc.first_interaction\n WHEN 'viewed_at' THEN uh.\"updatedAt\"\n WHEN 'viewed_updated'\n THEN COALESCE(uh.\"updatedAt\", lc.last_interaction)\n ELSE lc.last_interaction\n END DESC NULLS LAST,\n lc.id DESC,\n d.created_at ASC NULLS LAST\n ",- "describe": {- "columns": [- {- "ordinal": 0,- "name": "company_id!",- "type_info": "Uuid"- },- {- "ordinal": 1,- "name": "company_team_id!",- "type_info": "Uuid"- },- {- "ordinal": 2,- "name": "company_email_sync!",- "type_info": "Bool"- },- {- "ordinal": 3,- "name": "company_hidden!",- "type_info": "Bool"- },- {- "ordinal": 4,- "name": "company_created_at!",- "type_info": "Timestamptz"- },- {- "ordinal": 5,- "name": "company_updated_at!",- "type_info": "Timestamptz"- },- {- "ordinal": 6,- "name": "domain_id?",- "type_info": "Uuid"- },- {- "ordinal": 7,- "name": "domain?",- "type_info": "Text"- },- {- "ordinal": 8,- "name": "domain_created_at?",- "type_info": "Timestamptz"- },- {- "ordinal": 9,- "name": "display_name?",- "type_info": "Text"- },- {- "ordinal": 10,- "name": "dir_description?",- "type_info": "Text"- },- {- "ordinal": 11,- "name": "viewed_at?",- "type_info": "Timestamptz"- }- ],- "parameters": {- "Left": [- "Uuid",- "UuidArray",- "Int8",- "Text",- "Bool",- "Timestamptz",- "Text",- "Text"- ]- },- "nullable": [- false,- false,- false,- false,- false,- false,- false,- false,- false,- null,- true,- null- ]- },- "hash": "9d46745c3514bf518340dd31f909163c257890c5008bfbdb5db4d2b7aa01ac35"-}diff --git a/crates/crm/src/outbound/companies_repo.rs b/crates/crm/src/outbound/companies_repo.rsindex 6062852256..79b1006c17 100644--- a/crates/crm/src/outbound/companies_repo.rs+++ b/crates/crm/src/outbound/companies_repo.rs@@ -1660,100 +1660,12 @@ impl CompaniesRepository for CompaniesRepositoryImpl {let cursor_ts = cursor.map(|c| c.last_sort_ts);let cursor_id = cursor.map(|c| c.last_id.to_string());- // CTE limits companies before the domain/directory joins; the- // outer ORDER BY repeats the CTE's sort + `d.created_at ASC`- // so rows arrive contiguous per company with the primary- // domain first. Sort columns are `first_interaction` /- // `last_interaction` from populate_contact (both NOT NULL —- // see the `crm_interaction_timestamps` migration). `Viewed*`- // variants join `UserHistory` per-user — same shape as the- // soup repo's `viewed_at` / `viewed_updated` sorts for- // docs/chats/projects (see pg_soup_repo/expanded/by_cursor.rs).- //+ // Rows arrive contiguous per company with the primary domain first.// `$5` (`hidden`) defaults to visible-only when `NULL`; the// admin/owner role check for `Some(true)` is enforced upstream// in soup's axum router.- let rows = sqlx::query!(- r#"- WITH limited_companies AS (- SELECT- c.id,- c.team_id,- c.custom_name,- c.email_sync,- c.hidden,- c.first_interaction,- c.last_interaction- FROM crm_companies c- LEFT JOIN "UserHistory" uh- ON uh."itemId" = c.id::text- AND uh."itemType" = 'crm_company'- AND uh."userId" = $8- WHERE c.team_id = $1- AND c.hidden = COALESCE($5::bool, FALSE)- AND EXISTS (- SELECT 1 FROM team_crm_settings tcs- WHERE tcs.team_id = $1 AND tcs.crm_enabled- )- AND (cardinality($2::uuid[]) = 0 OR c.id = ANY($2::uuid[]))- -- Keyset seek (NULL = first page): keep only rows that- -- sort strictly after the cursor.- AND (- $6::timestamptz IS NULL- OR (- CASE $4- WHEN 'created_at' THEN c.first_interaction- WHEN 'viewed_at' THEN uh."updatedAt"- WHEN 'viewed_updated'- THEN COALESCE(uh."updatedAt", c.last_interaction)- ELSE c.last_interaction- END,- c.id::text- ) < ($6, $7)- )- ORDER BY- CASE $4- WHEN 'created_at' THEN c.first_interaction- WHEN 'viewed_at' THEN uh."updatedAt"- WHEN 'viewed_updated'- THEN COALESCE(uh."updatedAt", c.last_interaction)- ELSE c.last_interaction- END DESC NULLS LAST,- c.id DESC- LIMIT $3- )- SELECT- lc.id AS "company_id!",- lc.team_id AS "company_team_id!",- lc.email_sync AS "company_email_sync!",- lc.hidden AS "company_hidden!",- lc.first_interaction AS "company_created_at!",- lc.last_interaction AS "company_updated_at!",- d.id AS "domain_id?",- d.domain AS "domain?",- d.created_at AS "domain_created_at?",- COALESCE(lc.custom_name, dd.name) AS "display_name?",- dd.description AS "dir_description?",- uh."updatedAt"::timestamptz AS "viewed_at?"- FROM limited_companies lc- LEFT JOIN "UserHistory" uh- ON uh."itemId" = lc.id::text- AND uh."itemType" = 'crm_company'- AND uh."userId" = $8- LEFT JOIN crm_domains d ON d.company_id = lc.id- LEFT JOIN crm_domain_directory dd- ON LOWER(dd.domain) = LOWER(d.domain)- ORDER BY- CASE $4- WHEN 'created_at' THEN lc.first_interaction- WHEN 'viewed_at' THEN uh."updatedAt"- WHEN 'viewed_updated'- THEN COALESCE(uh."updatedAt", lc.last_interaction)- ELSE lc.last_interaction- END DESC NULLS LAST,- lc.id DESC,- d.created_at ASC NULLS LAST- "#,+ let rows = sqlx::query_file!(+ "src/outbound/companies_repo/list_companies_for_soup.sql",team_id,company_ids,limit,diff --git a/crates/crm/src/outbound/companies_repo/list_companies_for_soup.sql b/crates/crm/src/outbound/companies_repo/list_companies_for_soup.sqlnew file mode 100644index 0000000000..3dc3b69a74--- /dev/null+++ b/crates/crm/src/outbound/companies_repo/list_companies_for_soup.sql@@ -0,0 +1,183 @@+-- Each sort reads the companies this user has viewed from their history, and+-- the rest of the team from an index in the sort's order, stopping after $3+-- rows either way. Branch conditions on parameters alone are checked once per+-- execution, so only the requested sort's branches run, also under a cached+-- generic plan. A single ORDER BY CASE $4 reads and sorts the whole team.+--+-- Viewed companies get their own branch because a generic plan can look up a+-- company's view time by scanning the user's whole history for each company.+-- Only the explicit-ids branch, which reads few companies, looks views up per+-- company.+WITH limited_companies AS (+ SELECT *+ FROM (+ -- Explicit ids are few, so this branch has no index order or limit.+ (+ SELECT+ c.id, c.team_id, c.custom_name, c.email_sync, c.hidden,+ c.first_interaction, c.last_interaction,+ CASE $4+ WHEN 'created_at' THEN c.first_interaction+ WHEN 'viewed_at' THEN uh."updatedAt"+ WHEN 'viewed_updated'+ THEN COALESCE(uh."updatedAt", c.last_interaction)+ ELSE c.last_interaction+ END AS sort_ts,+ uh."updatedAt"::timestamptz AS viewed_at+ FROM crm_companies c+ LEFT JOIN "UserHistory" uh+ ON uh."itemId" = c.id::text+ AND uh."itemType" = 'crm_company'+ AND uh."userId" = $8+ WHERE cardinality($2::uuid[]) > 0+ AND c.id = ANY($2::uuid[])+ AND c.team_id = $1+ AND c.hidden = COALESCE($5::bool, FALSE)+ )+ UNION ALL+ -- Companies the user has viewed, under every sort. Bounded by the size+ -- of the user's company history rather than the team: each history+ -- row looks up its company. As a plain join, a generic plan that+ -- expects a small team and a long history reads and sorts the whole+ -- team for a merge join instead. The LIMIT keeps the lookup per row;+ -- ids are unique, so it drops nothing.+ (+ SELECT *+ FROM (+ SELECT+ c.id, c.team_id, c.custom_name, c.email_sync, c.hidden,+ c.first_interaction, c.last_interaction,+ CASE $4+ WHEN 'created_at' THEN c.first_interaction+ WHEN 'updated_at' THEN c.last_interaction+ ELSE uh."updatedAt"::timestamptz+ END AS sort_ts,+ uh."updatedAt"::timestamptz AS viewed_at+ FROM "UserHistory" uh+ CROSS JOIN LATERAL (+ SELECT+ c.id, c.team_id, c.custom_name, c.email_sync, c.hidden,+ c.first_interaction, c.last_interaction+ FROM crm_companies c+ WHERE c.id::text = uh."itemId"+ AND c.team_id = $1+ AND c.hidden = COALESCE($5::bool, FALSE)+ LIMIT 1+ ) c+ WHERE cardinality($2::uuid[]) = 0+ AND uh."userId" = $8+ AND uh."itemType" = 'crm_company'+ ) viewed+ WHERE $6::timestamptz IS NULL OR (sort_ts, id::text) < ($6, $7)+ ORDER BY sort_ts DESC NULLS LAST, id DESC+ LIMIT $3+ )+ UNION ALL+ -- The remaining branches read unviewed companies. NOT IN rather than+ -- NOT EXISTS: "itemId" is NOT NULL, so they are equivalent, and NOT IN+ -- is planned as one hashed lookup of the user's history instead of a+ -- scan of it per company.+ --+ -- The seek bound repeats the keyset's leading column so the index+ -- scan starts at the cursor. Both interaction columns are NOT NULL,+ -- so it drops no rows; the row comparison stays exact.+ --+ -- Unviewed companies sort by last_interaction under viewed_updated too.+ (+ SELECT+ c.id, c.team_id, c.custom_name, c.email_sync, c.hidden,+ c.first_interaction, c.last_interaction,+ c.last_interaction,+ NULL::timestamptz+ FROM crm_companies c+ WHERE cardinality($2::uuid[]) = 0+ AND $4 IN ('updated_at', 'viewed_updated')+ AND c.team_id = $1+ AND c.hidden = COALESCE($5::bool, FALSE)+ AND c.last_interaction <= COALESCE($6::timestamptz, 'infinity')+ AND ($6::timestamptz IS NULL OR (c.last_interaction, c.id::text) < ($6, $7))+ AND c.id::text NOT IN (+ SELECT uh."itemId" FROM "UserHistory" uh+ WHERE uh."userId" = $8+ AND uh."itemType" = 'crm_company'+ )+ ORDER BY c.last_interaction DESC NULLS LAST, c.id DESC+ LIMIT $3+ )+ UNION ALL+ (+ SELECT+ c.id, c.team_id, c.custom_name, c.email_sync, c.hidden,+ c.first_interaction, c.last_interaction,+ c.first_interaction,+ NULL::timestamptz+ FROM crm_companies c+ WHERE cardinality($2::uuid[]) = 0+ AND $4 = 'created_at'+ AND c.team_id = $1+ AND c.hidden = COALESCE($5::bool, FALSE)+ AND c.first_interaction <= COALESCE($6::timestamptz, 'infinity')+ AND ($6::timestamptz IS NULL OR (c.first_interaction, c.id::text) < ($6, $7))+ AND c.id::text NOT IN (+ SELECT uh."itemId" FROM "UserHistory" uh+ WHERE uh."userId" = $8+ AND uh."itemType" = 'crm_company'+ )+ ORDER BY c.first_interaction DESC NULLS LAST, c.id DESC+ LIMIT $3+ )+ UNION ALL+ -- Unviewed companies sort as NULL under viewed_at. NULL never passes a+ -- cursor, so they only appear on a first page.+ (+ SELECT+ c.id, c.team_id, c.custom_name, c.email_sync, c.hidden,+ c.first_interaction, c.last_interaction,+ NULL::timestamptz,+ NULL::timestamptz+ FROM crm_companies c+ WHERE cardinality($2::uuid[]) = 0+ AND $4 = 'viewed_at'+ AND $6::timestamptz IS NULL+ AND c.team_id = $1+ AND c.hidden = COALESCE($5::bool, FALSE)+ AND c.id::text NOT IN (+ SELECT uh."itemId" FROM "UserHistory" uh+ WHERE uh."userId" = $8+ AND uh."itemType" = 'crm_company'+ )+ ORDER BY c.id DESC+ LIMIT $3+ )+ ) candidates+ WHERE EXISTS (+ SELECT 1 FROM team_crm_settings tcs+ WHERE tcs.team_id = $1 AND tcs.crm_enabled+ )+ -- Keyset seek (NULL = first page): keep only rows that sort strictly+ -- after the cursor.+ AND ($6::timestamptz IS NULL OR (sort_ts, id::text) < ($6, $7))+ ORDER BY sort_ts DESC NULLS LAST, id DESC+ LIMIT $3+)+SELECT+ lc.id AS "company_id!",+ lc.team_id AS "company_team_id!",+ lc.email_sync AS "company_email_sync!",+ lc.hidden AS "company_hidden!",+ lc.first_interaction AS "company_created_at!",+ lc.last_interaction AS "company_updated_at!",+ d.id AS "domain_id?",+ d.domain AS "domain?",+ d.created_at AS "domain_created_at?",+ COALESCE(lc.custom_name, dd.name) AS "display_name?",+ dd.description AS "dir_description?",+ lc.viewed_at AS "viewed_at?"+FROM limited_companies lc+LEFT JOIN crm_domains d ON d.company_id = lc.id+LEFT JOIN crm_domain_directory dd+ ON LOWER(dd.domain) = LOWER(d.domain)+ORDER BY+ lc.sort_ts DESC NULLS LAST,+ lc.id DESC,+ d.created_at ASC NULLS LASTdiff --git a/crates/crm/src/outbound/companies_repo/test/list_companies_for_soup.rs b/crates/crm/src/outbound/companies_repo/test/list_companies_for_soup.rsindex b85b12fc5c..c63c6de9df 100644--- a/crates/crm/src/outbound/companies_repo/test/list_companies_for_soup.rs+++ b/crates/crm/src/outbound/companies_repo/test/list_companies_for_soup.rs@@ -1,10 +1,111 @@use super::helpers::*;use crate::domain::companies_repo::*;+use crate::domain::model::CrmCompanyForSoup;use crate::outbound::companies_repo::*;+use chrono::{DateTime, Utc};use macro_db_migrator::MACRO_DB_MIGRATIONS;use sqlx::PgPool;+use std::collections::HashMap;use uuid::Uuid;+const VIEWER: &str = "macro|[email protected]";+const CRM_COMPANY: &str = "crm_company";+const ALL_SORTS: [CrmCompanyListSort; 4] = [+ CrmCompanyListSort::UpdatedAt,+ CrmCompanyListSort::CreatedAt,+ CrmCompanyListSort::ViewedAt,+ CrmCompanyListSort::ViewedUpdated,+];++/// Whole hours, so the values survive `UserHistory`'s millisecond+/// timestamps and the cursor compare unchanged.+fn ts(day: u32, hour: u32) -> DateTime<Utc> {+ format!("2024-01-{day:02}T{hour:02}:00:00Z")+ .parse()+ .expect("valid RFC 3339 timestamp")+}++fn ids(page: &[CrmCompanyForSoup]) -> Vec<Uuid> {+ page.iter().map(|c| c.company.id).collect()+}++async fn insert_company_at(+ pool: &PgPool,+ team_id: Uuid,+ domain: &str,+ first_interaction: DateTime<Utc>,+ last_interaction: DateTime<Utc>,+) -> sqlx::Result<Uuid> {+ let id = insert_company(pool, team_id, true, &[domain]).await?;+ sqlx::query(+ r#"UPDATE crm_companies+ SET first_interaction = $2, last_interaction = $3+ WHERE id = $1"#,+ )+ .bind(id)+ .bind(first_interaction)+ .bind(last_interaction)+ .execute(pool)+ .await?;+ Ok(id)+}++async fn record_history(+ pool: &PgPool,+ user_id: &str,+ item_type: &str,+ item_id: Uuid,+ at: DateTime<Utc>,+) -> sqlx::Result<()> {+ sqlx::query(+ r#"INSERT INTO "UserHistory" ("userId", "itemId", "itemType", "createdAt", "updatedAt")+ VALUES ($1, $2, $3, $4, $4)"#,+ )+ .bind(user_id)+ .bind(item_id.to_string())+ .bind(item_type)+ .bind(at)+ .execute(pool)+ .await?;+ Ok(())+}++/// Pages through `VIEWER`'s list the way the soup paginator does: each+/// cursor is the last row's sort timestamp (`SoupItem::cursor_timestamp`)+/// and id.+async fn walk(+ repo: &CompaniesRepositoryImpl,+ team: &Uuid,+ sort: CrmCompanyListSort,+ limit: i64,+) -> anyhow::Result<Vec<Uuid>> {+ let mut seen = Vec::new();+ let mut cursor = None;+ for _ in 0..20 {+ let page = repo+ .list_companies_for_soup(team, VIEWER, &[], None, sort, cursor, limit)+ .await?;+ seen.extend(ids(&page));+ let Some(last) = page.last() else {+ break;+ };+ if i64::try_from(page.len())? < limit {+ break;+ }+ let last_sort_ts = match sort {+ CrmCompanyListSort::UpdatedAt => last.company.updated_at,+ CrmCompanyListSort::CreatedAt => last.company.created_at,+ CrmCompanyListSort::ViewedAt => last.viewed_at.unwrap_or_default(),+ CrmCompanyListSort::ViewedUpdated => last.viewed_at.unwrap_or(last.company.updated_at),+ };+ cursor = Some(CrmCompanySoupCursor {+ last_sort_ts,+ last_id: last.company.id,+ });+ }+ Ok(seen)+}+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]async fn list_for_soup_returns_empty_when_killswitch_missing(pool: PgPool) -> anyhow::Result<()> {let team = Uuid::now_v7();@@ -41,21 +142,17 @@ async fn list_for_soup_returns_empty_when_killswitch_off(pool: PgPool) -> anyhow.bind(team).execute(&pool).await?;- insert_company(&pool, team, true, &["acme.com"]).await?;+ let viewed = insert_company(&pool, team, true, &["acme.com"]).await?;+ insert_company(&pool, team, true, &["zeta.com"]).await?;+ record_history(&pool, owner, CRM_COMPANY, viewed, ts(1, 0)).await?;let repo = CompaniesRepositoryImpl::new(pool);- let result = repo- .list_companies_for_soup(- &team,- "macro|[email protected]",- &[],- None,- CrmCompanyListSort::UpdatedAt,- None,- 100,- )- .await?;- assert!(result.is_empty());+ for sort in ALL_SORTS {+ let result = repo+ .list_companies_for_soup(&team, owner, &[], None, sort, None, 100)+ .await?;+ assert!(result.is_empty(), "{sort:?}");+ }Ok(())}@@ -435,3 +532,277 @@ async fn list_for_soup_pagination_breaks_ties_on_id(pool: PgPool) -> anyhow::Res);Ok(())}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn list_for_soup_created_at_orders_by_first_interaction(pool: PgPool) -> anyhow::Result<()> {+ let team = Uuid::now_v7();+ seed_team(&pool, team, VIEWER).await?;+ enable_crm_for_team(&pool, team).await?;++ // First interactions run opposite to last interactions, so a+ // last_interaction order would come out reversed. Two companies tie on+ // first_interaction to exercise the id tiebreak.+ let mut want = Vec::new();+ for (idx, first_day) in [4, 3, 3, 1].into_iter().enumerate() {+ let last_day = 10 + u32::try_from(idx)?;+ let domain = format!("created{idx}.com");+ let id = insert_company_at(&pool, team, &domain, ts(first_day, 0), ts(last_day, 0)).await?;+ want.push((ts(first_day, 0), id));+ }+ want.sort_by(|a, b| b.0.cmp(&a.0).then(b.1.cmp(&a.1)));+ let expected: Vec<Uuid> = want.iter().map(|(_, id)| *id).collect();++ let repo = CompaniesRepositoryImpl::new(pool);+ for limit in [1, 2, 100] {+ let seen = walk(&repo, &team, CrmCompanyListSort::CreatedAt, limit).await?;+ assert_eq!(seen, expected, "limit {limit}");+ }+ Ok(())+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn list_for_soup_interaction_sorts_ignore_views(pool: PgPool) -> anyhow::Result<()> {+ let team = Uuid::now_v7();+ seed_team(&pool, team, VIEWER).await?;+ enable_crm_for_team(&pool, team).await?;++ // Viewed and never-viewed companies alternate under both interaction+ // sorts, one of each tie on both keys, and every view is newer than every+ // interaction, so sorting a viewed company by its view time misplaces it.+ let mut by_first = Vec::new();+ let mut by_last = Vec::new();+ for (idx, (first_day, last_day, viewed)) in [+ (1, 20, true),+ (2, 19, false),+ (3, 18, true),+ (3, 18, false),+ (5, 16, true),+ (6, 15, false),+ ]+ .into_iter()+ .enumerate()+ {+ let domain = format!("interaction{idx}.com");+ let id = insert_company_at(&pool, team, &domain, ts(first_day, 0), ts(last_day, 0)).await?;+ if viewed {+ let view_day = 25 + u32::try_from(idx)?;+ record_history(&pool, VIEWER, CRM_COMPANY, id, ts(view_day, 0)).await?;+ }+ by_first.push((ts(first_day, 0), id));+ by_last.push((ts(last_day, 0), id));+ }++ let repo = CompaniesRepositoryImpl::new(pool);+ for (sort, mut want) in [+ (CrmCompanyListSort::UpdatedAt, by_last),+ (CrmCompanyListSort::CreatedAt, by_first),+ ] {+ want.sort_by(|a, b| b.cmp(a));+ let expected: Vec<Uuid> = want.into_iter().map(|(_, id)| id).collect();+ for limit in [1, 2, 100] {+ let seen = walk(&repo, &team, sort, limit).await?;+ assert_eq!(seen, expected, "{sort:?} limit {limit}");+ }+ }+ Ok(())+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn list_for_soup_viewed_sorts_use_only_this_users_company_views(+ pool: PgPool,+) -> anyhow::Result<()> {+ let team = Uuid::now_v7();+ let other_team = Uuid::now_v7();+ let other_user = "macro|[email protected]";+ seed_team(&pool, team, VIEWER).await?;+ seed_team(&pool, other_team, other_user).await?;+ enable_crm_for_team(&pool, team).await?;+ enable_crm_for_team(&pool, other_team).await?;++ let viewed_old = insert_company_at(&pool, team, "viewed-old.com", ts(1, 0), ts(1, 0)).await?;+ let viewed_new = insert_company_at(&pool, team, "viewed-new.com", ts(1, 0), ts(1, 0)).await?;+ let unviewed_a = insert_company_at(&pool, team, "unviewed-a.com", ts(6, 0), ts(6, 0)).await?;+ let unviewed_b = insert_company_at(&pool, team, "unviewed-b.com", ts(7, 0), ts(7, 0)).await?;+ let elsewhere =+ insert_company_at(&pool, other_team, "elsewhere.com", ts(9, 0), ts(9, 0)).await?;+ record_history(&pool, VIEWER, CRM_COMPANY, viewed_old, ts(2, 0)).await?;+ record_history(&pool, VIEWER, CRM_COMPANY, viewed_new, ts(3, 0)).await?;+ // None of these is this user's view of one of this team's companies.+ record_history(&pool, other_user, CRM_COMPANY, unviewed_a, ts(8, 0)).await?;+ record_history(&pool, VIEWER, "document", unviewed_b, ts(8, 0)).await?;+ record_history(&pool, VIEWER, CRM_COMPANY, elsewhere, ts(8, 0)).await?;++ let repo = CompaniesRepositoryImpl::new(pool);++ // Views newest first, then never-viewed companies by id.+ let viewed_at = repo+ .list_companies_for_soup(+ &team,+ VIEWER,+ &[],+ None,+ CrmCompanyListSort::ViewedAt,+ None,+ 100,+ )+ .await?;+ assert_eq!(+ ids(&viewed_at),+ [+ viewed_new,+ viewed_old,+ unviewed_a.max(unviewed_b),+ unviewed_a.min(unviewed_b)+ ]+ );+ let view_times: Vec<Option<DateTime<Utc>>> = viewed_at.iter().map(|c| c.viewed_at).collect();+ assert_eq!(view_times, [Some(ts(3, 0)), Some(ts(2, 0)), None, None]);++ let after_newest_view = repo+ .list_companies_for_soup(+ &team,+ VIEWER,+ &[],+ None,+ CrmCompanyListSort::ViewedAt,+ Some(CrmCompanySoupCursor {+ last_sort_ts: ts(3, 0),+ last_id: viewed_new,+ }),+ 1,+ )+ .await?;+ assert_eq!(ids(&after_newest_view), [viewed_old]);++ // Views stand in for last_interaction.+ let viewed_updated = walk(&repo, &team, CrmCompanyListSort::ViewedUpdated, 100).await?;+ assert_eq!(+ viewed_updated,+ [unviewed_b, unviewed_a, viewed_new, viewed_old]+ );++ // Every sort reports this user's view time, with or without an id list.+ let want_views = HashMap::from([+ (viewed_old, Some(ts(2, 0))),+ (viewed_new, Some(ts(3, 0))),+ (unviewed_a, None),+ (unviewed_b, None),+ ]);+ let all = [viewed_old, viewed_new, unviewed_a, unviewed_b];+ for sort in ALL_SORTS {+ for company_ids in [&[][..], &all[..]] {+ let page = repo+ .list_companies_for_soup(&team, VIEWER, company_ids, None, sort, None, 100)+ .await?;+ let got: HashMap<Uuid, Option<DateTime<Utc>>> =+ page.iter().map(|c| (c.company.id, c.viewed_at)).collect();+ assert_eq!(got, want_views, "{sort:?} ids={}", company_ids.len());+ }+ }+ Ok(())+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn list_for_soup_viewed_updated_interleaves_views_and_interactions(+ pool: PgPool,+) -> anyhow::Result<()> {+ let team = Uuid::now_v7();+ seed_team(&pool, team, VIEWER).await?;+ enable_crm_for_team(&pool, team).await?;++ let c1 = insert_company_at(&pool, team, "vu1.com", ts(1, 0), ts(1, 0)).await?;+ let c2 = insert_company_at(&pool, team, "vu2.com", ts(2, 0), ts(2, 0)).await?;+ let c3 = insert_company_at(&pool, team, "vu3.com", ts(3, 0), ts(3, 0)).await?;+ let c4 = insert_company_at(&pool, team, "vu4.com", ts(4, 0), ts(4, 0)).await?;+ // Sort keys: c1 = day 5 (view), c4 = day 4, c2 = day 2, c3 = day 1 noon+ // (a view older than its last interaction still replaces it).+ record_history(&pool, VIEWER, CRM_COMPANY, c1, ts(5, 0)).await?;+ record_history(&pool, VIEWER, CRM_COMPANY, c3, ts(1, 12)).await?;++ let repo = CompaniesRepositoryImpl::new(pool);+ for limit in [1, 2, 100] {+ let seen = walk(&repo, &team, CrmCompanyListSort::ViewedUpdated, limit).await?;+ assert_eq!(seen, [c1, c4, c2, c3], "limit {limit}");+ }++ // An explicit id list keeps each sort's order, the limit, and the cursor.+ let subset = [c3, c4, c1];+ let after_c1 = CrmCompanySoupCursor {+ last_sort_ts: ts(5, 0),+ last_id: c1,+ };+ for (sort, cursor, limit, want) in [+ (+ CrmCompanyListSort::ViewedUpdated,+ None,+ 100,+ vec![c1, c4, c3],+ ),+ (CrmCompanyListSort::ViewedAt, None, 100, vec![c1, c3, c4]),+ (CrmCompanyListSort::UpdatedAt, None, 100, vec![c4, c3, c1]),+ (CrmCompanyListSort::ViewedUpdated, None, 2, vec![c1, c4]),+ (+ CrmCompanyListSort::ViewedUpdated,+ Some(after_c1),+ 100,+ vec![c4, c3],+ ),+ ] {+ let page = repo+ .list_companies_for_soup(&team, VIEWER, &subset, None, sort, cursor, limit)+ .await?;+ assert_eq!(ids(&page), want, "{sort:?} cursor={cursor:?} limit={limit}");+ }+ Ok(())+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn list_for_soup_hidden_filter_applies_to_every_sort(pool: PgPool) -> anyhow::Result<()> {+ let team = Uuid::now_v7();+ seed_team(&pool, team, VIEWER).await?;+ enable_crm_for_team(&pool, team).await?;++ // A viewed and a never-viewed company on each side of the filter.+ let mut visible = Vec::new();+ let mut hidden = Vec::new();+ for (day, hide, viewed) in [+ (1, false, true),+ (2, false, false),+ (3, true, true),+ (4, true, false),+ ] {+ let domain = format!("hidden{day}.com");+ let id = insert_company_at(&pool, team, &domain, ts(day, 0), ts(day, 0)).await?;+ if viewed {+ record_history(&pool, VIEWER, CRM_COMPANY, id, ts(day, 6)).await?;+ }+ if hide {+ sqlx::query("UPDATE crm_companies SET hidden = TRUE WHERE id = $1")+ .bind(id)+ .execute(&pool)+ .await?;+ hidden.push(id);+ } else {+ visible.push(id);+ }+ }+ visible.sort();+ hidden.sort();++ let repo = CompaniesRepositoryImpl::new(pool);+ for sort in ALL_SORTS {+ for (filter, want) in [+ (None, &visible),+ (Some(false), &visible),+ (Some(true), &hidden),+ ] {+ let page = repo+ .list_companies_for_soup(&team, VIEWER, &[], filter, sort, None, 100)+ .await?;+ let mut got = ids(&page);+ got.sort();+ assert_eq!(&got, want, "{sort:?} hidden={filter:?}");+ }+ }+ Ok(())+}diff --git a/crates/macro_db_client/migrations/20261101054827_crm_companies_team_hidden_last_interaction_index.sql b/crates/macro_db_client/migrations/20261101054827_crm_companies_team_hidden_last_interaction_index.sqlnew file mode 100644index 0000000000..d3301ac0fc--- /dev/null+++ b/crates/macro_db_client/migrations/20261101054827_crm_companies_team_hidden_last_interaction_index.sql@@ -0,0 +1,10 @@+-- no-transaction+-- The CRM companies list under the updated_at and viewed_updated sorts reads a+-- team's companies in this order and stops at the page size, instead of+-- reading and sorting every company in the team. Must stay a single+-- statement: sqlx sends no-transaction migrations as one batch, and+-- CONCURRENTLY cannot run inside a transaction. No IF NOT EXISTS: if an+-- interrupted build leaves an INVALID index, the next run fails on it instead+-- of recording success; drop it with DROP INDEX CONCURRENTLY and re-run.+CREATE INDEX CONCURRENTLY crm_companies_team_hidden_last_interaction_idx+ ON crm_companies (team_id, hidden, last_interaction DESC NULLS LAST, id DESC);diff --git a/crates/macro_db_client/migrations/20261101054828_crm_companies_team_hidden_first_interaction_index.sql b/crates/macro_db_client/migrations/20261101054828_crm_companies_team_hidden_first_interaction_index.sqlnew file mode 100644index 0000000000..06c40e15ed--- /dev/null+++ b/crates/macro_db_client/migrations/20261101054828_crm_companies_team_hidden_first_interaction_index.sql@@ -0,0 +1,5 @@+-- no-transaction+-- The same ordered read for the CRM companies list's created_at sort. Must stay+-- a single statement, without IF NOT EXISTS (see the last_interaction index).+CREATE INDEX CONCURRENTLY crm_companies_team_hidden_first_interaction_idx+ ON crm_companies (team_id, hidden, first_interaction DESC NULLS LAST, id DESC);diff --git a/crates/macro_db_client/migrations/20261101054829_crm_companies_team_hidden_id_index.sql b/crates/macro_db_client/migrations/20261101054829_crm_companies_team_hidden_id_index.sqlnew file mode 100644index 0000000000..aa58c7611d--- /dev/null+++ b/crates/macro_db_client/migrations/20261101054829_crm_companies_team_hidden_id_index.sql@@ -0,0 +1,8 @@+-- no-transaction+-- The CRM companies list's viewed_at sort lists never-viewed companies by id+-- descending. Its (team_id, hidden = false) prefix also serves the lookups of+-- crm_companies_visible_team_id_idx, dropped after it. Must stay a single+-- statement, without IF NOT EXISTS (see the last_interaction index), so the+-- drop never runs while this index is INVALID.+CREATE INDEX CONCURRENTLY crm_companies_team_hidden_id_idx+ ON crm_companies (team_id, hidden, id DESC);diff --git a/crates/macro_db_client/migrations/20261101054830_crm_companies_id_text_index.sql b/crates/macro_db_client/migrations/20261101054830_crm_companies_id_text_index.sqlnew file mode 100644index 0000000000..71589e7709--- /dev/null+++ b/crates/macro_db_client/migrations/20261101054830_crm_companies_id_text_index.sql@@ -0,0 +1,7 @@+-- no-transaction+-- "UserHistory"."itemId" is text, so the CRM companies list joins a user's+-- viewed companies on id::text. Without this index that join reads the team.+-- Must stay a single statement, without IF NOT EXISTS (see the last_interaction+-- index).+CREATE INDEX CONCURRENTLY crm_companies_id_text_idx+ ON crm_companies ((id::text));diff --git a/crates/macro_db_client/migrations/20261101054831_drop_crm_companies_visible_team_id_index.sql b/crates/macro_db_client/migrations/20261101054831_drop_crm_companies_visible_team_id_index.sqlnew file mode 100644index 0000000000..0684bf7c91--- /dev/null+++ b/crates/macro_db_client/migrations/20261101054831_drop_crm_companies_visible_team_id_index.sql@@ -0,0 +1,6 @@+-- no-transaction+-- Superseded by crm_companies_team_hidden_id_idx: visible companies are its+-- (team_id, hidden = false) prefix. crm_companies_team_id_idx stays: queries+-- that read every company in a team, such as the thread-access CRM check, read+-- fewer pages through it than through the wider composite index.+DROP INDEX CONCURRENTLY IF EXISTS crm_companies_visible_team_id_idx;