Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(database-sql): respect pagination and DISTINCT in grouped reads (#7576)

Wolf Mermelstein7c0bfaa89fdf
diff --git a/crates/database_sql/src/fold.rs b/crates/database_sql/src/fold.rs
index 8d72bd307c..90be56e554 100644
--- a/crates/database_sql/src/fold.rs
+++ b/crates/database_sql/src/fold.rs
@@ -155,7 +155,7 @@ fn fold_joined(catalog: &Catalog, plan: &Plan, rows: Vec<Row>) -> (Table, Vec<Ro
(row.id, cells)
});
let (ids, table): (Vec<RowId>, Table) = if plan.distinct {
- window(plan, distinct(projected)).unzip()
+ window(plan, distinct(projected, |(_, cells)| cells)).unzip()
} else {
window(plan, projected).unzip()
};
@@ -164,6 +164,11 @@ fn fold_joined(catalog: &Catalog, plan: &Plan, rows: Vec<Row>) -> (Table, Vec<Ro
Shape::Aggregate { group_by, items } => {
let mut groups = aggregate::groups(rows, *group_by, items);
sort::groups(catalog, plan, &mut groups, *group_by, items);
+ let groups = if plan.distinct {
+ distinct(groups.into_iter(), |group| &group.cells).collect()
+ } else {
+ groups
+ };
(
window(plan, groups.into_iter().map(|group| group.cells)).collect(),
Vec::new(),
@@ -179,13 +184,14 @@ fn window<Item>(plan: &Plan, rows: impl Iterator<Item = Item>) -> impl Iterator<
}
/// Keep the first of every set of equal result rows, in order.
-fn distinct(
- rows: impl Iterator<Item = (RowId, Vec<Option<Cell>>)>,
-) -> impl Iterator<Item = (RowId, Vec<Option<Cell>>)> {
+fn distinct<Item>(
+ rows: impl Iterator<Item = Item>,
+ cells: impl Fn(&Item) -> &[Option<Cell>],
+) -> impl Iterator<Item = Item> {
let mut seen: HashSet<Vec<Option<CellKey>>> = HashSet::new();
- rows.filter(move |(_, cells)| {
+ rows.filter(move |row| {
seen.insert(
- cells
+ cells(row)
.iter()
.map(|cell| cell.as_ref().map(CellKey::from))
.collect(),
@@ -220,7 +226,7 @@ pub fn fold_bins(catalog: &Catalog, plan: &Plan, bins: Vec<Bin>) -> Result<Table
})
.collect::<Result<_, RunError>>()?;
sort::groups(catalog, plan, &mut groups, *group_by, items);
- Ok(groups.into_iter().map(|group| group.cells).collect())
+ Ok(window(plan, groups.into_iter().map(|group| group.cells)).collect())
}
/// Where an `ORDER BY` key lives in a group's output.
diff --git a/crates/database_sql/src/fold/test.rs b/crates/database_sql/src/fold/test.rs
index 8f2a55609f..404ba6038b 100644
--- a/crates/database_sql/src/fold/test.rs
+++ b/crates/database_sql/src/fold/test.rs
@@ -211,6 +211,58 @@ fn bins_answer_a_count_only_group() {
);
}
+#[test]
+fn grouped_bins_apply_limit_and_offset_after_sorting() {
+ use crate::engine::{Engine, Step};
+
+ for (window, expected) in [
+ (
+ "LIMIT 1",
+ vec![vec![
+ Some(Cell::Options(vec![WON])),
+ Some(Cell::Number(9.0)),
+ ]],
+ ),
+ (
+ "LIMIT 1 OFFSET 1",
+ vec![vec![
+ Some(Cell::Options(vec![LEAD])),
+ Some(Cell::Number(4.0)),
+ ]],
+ ),
+ ("LIMIT 0", vec![]),
+ ("LIMIT 1 OFFSET 3", vec![]),
+ ] {
+ let sql = format!(
+ "SELECT stage, COUNT(*) FROM crm.deals \
+ GROUP BY stage ORDER BY 2 DESC, stage {window}"
+ );
+ let (mut engine, step) = Engine::start(&catalog(), &sql).unwrap();
+ let Step::Bins(request) = step else {
+ panic!("expected grouped bins for {sql}");
+ };
+ let bins = vec![
+ Bin {
+ key: Some(Cell::Options(vec![LEAD])),
+ count: 4,
+ },
+ Bin {
+ key: None,
+ count: 4,
+ },
+ Bin {
+ key: Some(Cell::Options(vec![WON])),
+ count: 9,
+ },
+ ];
+ let Step::Done(outcome) = engine.feed_bins(request.id, bins).unwrap() else {
+ panic!("expected a completed read for {sql}");
+ };
+ assert_eq!(outcome.rows, expected, "{sql}");
+ assert!(outcome.row_ids.is_empty());
+ }
+}
+
#[test]
fn limit_and_offset_apply_after_ordering() {
let windowed = plan("SELECT name FROM crm.deals ORDER BY name LIMIT 2 OFFSET 1");
@@ -321,7 +373,7 @@ fn distinct_keeps_the_first_of_equal_rows_after_sorting() {
}
#[test]
-fn distinct_over_aggregates_changes_nothing() {
+fn distinct_keeps_different_group_keys() {
let plan =
plan("SELECT DISTINCT stage, COUNT(*) FROM crm.deals GROUP BY stage ORDER BY 2 DESC");
let (rows, ids) = fold_relations(&catalog(), &plan, vec![deals()]);
@@ -329,6 +381,44 @@ fn distinct_over_aggregates_changes_nothing() {
assert_eq!(ids, Vec::<RowId>::new());
}
+#[test]
+fn distinct_aggregate_results_are_deduplicated_before_limit_and_offset() {
+ use crate::engine::{Engine, Step};
+ use crate::run::Page;
+
+ for (window, expected) in [
+ (
+ "",
+ vec![vec![Some(Cell::Number(1.0))], vec![Some(Cell::Number(2.0))]],
+ ),
+ ("LIMIT 1 OFFSET 1", vec![vec![Some(Cell::Number(2.0))]]),
+ ("LIMIT 1 OFFSET 2", vec![]),
+ ] {
+ let sql = format!(
+ "SELECT DISTINCT COUNT(*) AS total FROM crm.deals \
+ GROUP BY stage ORDER BY total {window}"
+ );
+ let (mut engine, step) = Engine::start(&catalog(), &sql).unwrap();
+ let Step::Fetch(request) = step else {
+ panic!("expected rows for distinct groups");
+ };
+ let Step::Done(outcome) = engine
+ .feed_page(
+ request.id,
+ Page {
+ rows: deals(),
+ next: None,
+ },
+ )
+ .unwrap()
+ else {
+ panic!("expected a completed read");
+ };
+ assert_eq!(outcome.rows, expected, "{sql}");
+ assert!(outcome.row_ids.is_empty());
+ }
+}
+
#[test]
fn join_matches_by_membership_and_leaves_empty_cells_unmatched() {
use crate::catalog::PEOPLE_ID;