Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
fix(database-sql): respect pagination and DISTINCT in grouped reads (#7576)
Wolf Mermelstein7c0bfaa89fdf
diff --git a/crates/database_sql/src/fold.rs b/crates/database_sql/src/fold.rsindex 8d72bd307c..90be56e554 100644--- a/crates/database_sql/src/fold.rs+++ b/crates/database_sql/src/fold.rs@@ -155,7 +155,7 @@ fn fold_joined(catalog: &Catalog, plan: &Plan, rows: Vec<Row>) -> (Table, Vec<Ro(row.id, cells)});let (ids, table): (Vec<RowId>, Table) = if plan.distinct {- window(plan, distinct(projected)).unzip()+ window(plan, distinct(projected, |(_, cells)| cells)).unzip()} else {window(plan, projected).unzip()};@@ -164,6 +164,11 @@ fn fold_joined(catalog: &Catalog, plan: &Plan, rows: Vec<Row>) -> (Table, Vec<RoShape::Aggregate { group_by, items } => {let mut groups = aggregate::groups(rows, *group_by, items);sort::groups(catalog, plan, &mut groups, *group_by, items);+ let groups = if plan.distinct {+ distinct(groups.into_iter(), |group| &group.cells).collect()+ } else {+ groups+ };(window(plan, groups.into_iter().map(|group| group.cells)).collect(),Vec::new(),@@ -179,13 +184,14 @@ fn window<Item>(plan: &Plan, rows: impl Iterator<Item = Item>) -> impl Iterator<}/// Keep the first of every set of equal result rows, in order.-fn distinct(- rows: impl Iterator<Item = (RowId, Vec<Option<Cell>>)>,-) -> impl Iterator<Item = (RowId, Vec<Option<Cell>>)> {+fn distinct<Item>(+ rows: impl Iterator<Item = Item>,+ cells: impl Fn(&Item) -> &[Option<Cell>],+) -> impl Iterator<Item = Item> {let mut seen: HashSet<Vec<Option<CellKey>>> = HashSet::new();- rows.filter(move |(_, cells)| {+ rows.filter(move |row| {seen.insert(- cells+ cells(row).iter().map(|cell| cell.as_ref().map(CellKey::from)).collect(),@@ -220,7 +226,7 @@ pub fn fold_bins(catalog: &Catalog, plan: &Plan, bins: Vec<Bin>) -> Result<Table}).collect::<Result<_, RunError>>()?;sort::groups(catalog, plan, &mut groups, *group_by, items);- Ok(groups.into_iter().map(|group| group.cells).collect())+ Ok(window(plan, groups.into_iter().map(|group| group.cells)).collect())}/// Where an `ORDER BY` key lives in a group's output.diff --git a/crates/database_sql/src/fold/test.rs b/crates/database_sql/src/fold/test.rsindex 8f2a55609f..404ba6038b 100644--- a/crates/database_sql/src/fold/test.rs+++ b/crates/database_sql/src/fold/test.rs@@ -211,6 +211,58 @@ fn bins_answer_a_count_only_group() {);}+#[test]+fn grouped_bins_apply_limit_and_offset_after_sorting() {+ use crate::engine::{Engine, Step};++ for (window, expected) in [+ (+ "LIMIT 1",+ vec![vec![+ Some(Cell::Options(vec![WON])),+ Some(Cell::Number(9.0)),+ ]],+ ),+ (+ "LIMIT 1 OFFSET 1",+ vec![vec![+ Some(Cell::Options(vec![LEAD])),+ Some(Cell::Number(4.0)),+ ]],+ ),+ ("LIMIT 0", vec![]),+ ("LIMIT 1 OFFSET 3", vec![]),+ ] {+ let sql = format!(+ "SELECT stage, COUNT(*) FROM crm.deals \+ GROUP BY stage ORDER BY 2 DESC, stage {window}"+ );+ let (mut engine, step) = Engine::start(&catalog(), &sql).unwrap();+ let Step::Bins(request) = step else {+ panic!("expected grouped bins for {sql}");+ };+ let bins = vec![+ Bin {+ key: Some(Cell::Options(vec![LEAD])),+ count: 4,+ },+ Bin {+ key: None,+ count: 4,+ },+ Bin {+ key: Some(Cell::Options(vec![WON])),+ count: 9,+ },+ ];+ let Step::Done(outcome) = engine.feed_bins(request.id, bins).unwrap() else {+ panic!("expected a completed read for {sql}");+ };+ assert_eq!(outcome.rows, expected, "{sql}");+ assert!(outcome.row_ids.is_empty());+ }+}+#[test]fn limit_and_offset_apply_after_ordering() {let windowed = plan("SELECT name FROM crm.deals ORDER BY name LIMIT 2 OFFSET 1");@@ -321,7 +373,7 @@ fn distinct_keeps_the_first_of_equal_rows_after_sorting() {}#[test]-fn distinct_over_aggregates_changes_nothing() {+fn distinct_keeps_different_group_keys() {let plan =plan("SELECT DISTINCT stage, COUNT(*) FROM crm.deals GROUP BY stage ORDER BY 2 DESC");let (rows, ids) = fold_relations(&catalog(), &plan, vec![deals()]);@@ -329,6 +381,44 @@ fn distinct_over_aggregates_changes_nothing() {assert_eq!(ids, Vec::<RowId>::new());}+#[test]+fn distinct_aggregate_results_are_deduplicated_before_limit_and_offset() {+ use crate::engine::{Engine, Step};+ use crate::run::Page;++ for (window, expected) in [+ (+ "",+ vec![vec![Some(Cell::Number(1.0))], vec![Some(Cell::Number(2.0))]],+ ),+ ("LIMIT 1 OFFSET 1", vec![vec![Some(Cell::Number(2.0))]]),+ ("LIMIT 1 OFFSET 2", vec![]),+ ] {+ let sql = format!(+ "SELECT DISTINCT COUNT(*) AS total FROM crm.deals \+ GROUP BY stage ORDER BY total {window}"+ );+ let (mut engine, step) = Engine::start(&catalog(), &sql).unwrap();+ let Step::Fetch(request) = step else {+ panic!("expected rows for distinct groups");+ };+ let Step::Done(outcome) = engine+ .feed_page(+ request.id,+ Page {+ rows: deals(),+ next: None,+ },+ )+ .unwrap()+ else {+ panic!("expected a completed read");+ };+ assert_eq!(outcome.rows, expected, "{sql}");+ assert!(outcome.row_ids.is_empty());+ }+}+#[test]fn join_matches_by_membership_and_leaves_empty_cells_unmatched() {use crate::catalog::PEOPLE_ID;