Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

Seanaye/fix/invalid projection ovrflow (#6548)

Sean Ayeb69a06064aa9
diff --git a/Cargo.lock b/Cargo.lock
index 2f5c39d3d0..d75cb78f33 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3058,7 +3058,7 @@ dependencies = [
[[package]]
name = "cache-wasm"
-version = "0.6.14"
+version = "0.6.16"
dependencies = [
"async-lock",
"cache-core",
@@ -15810,7 +15810,6 @@ dependencies = [
"document_sub_type",
"item-filter-index",
"macro_user_id",
- "model_file_type",
"models_soup",
"postcard",
"predicate-index",
diff --git a/apps/web/src/lib/queries/soup/backfill.test.tsx b/apps/web/src/lib/queries/soup/backfill.test.tsx
index c1583b4d78..5e1cdf010c 100644
--- a/apps/web/src/lib/queries/soup/backfill.test.tsx
+++ b/apps/web/src/lib/queries/soup/backfill.test.tsx
@@ -407,9 +407,9 @@ describe('runSoupBackfills', () => {
rendered.unmount();
});
- it('does not reuse pre-projection native backfill checkpoints', () => {
+ it.each([13, 14])('does not reuse v%i backfill checkpoints', (version) => {
localStorage.setItem(
- 'graphql-soup-backfill:v13:user-1:email-filter-metadata',
+ `graphql-soup-backfill:v${version}:user-1:email-filter-metadata`,
JSON.stringify({
userId: 'user-1',
nextCursor: 'old-page',
@@ -431,7 +431,7 @@ describe('runSoupBackfills', () => {
it('restarts from the beginning when the cache generation is replaced', async () => {
localStorage.setItem(
- 'graphql-soup-backfill:v14:user-1:core-entities',
+ 'graphql-soup-backfill:v15:user-1:core-entities',
JSON.stringify({
userId: 'user-1',
nextCursor: 'stale-cursor',
diff --git a/apps/web/src/lib/queries/soup/backfill.ts b/apps/web/src/lib/queries/soup/backfill.ts
index 537badeb5c..56013b0969 100644
--- a/apps/web/src/lib/queries/soup/backfill.ts
+++ b/apps/web/src/lib/queries/soup/backfill.ts
@@ -26,9 +26,9 @@ import { createEffect, createSignal, onCleanup } from 'solid-js';
// Bump when a default backfill input or completion guarantee changes so
// persisted cursors cannot retain an older hydration contract.
-// Rehydrate complete property snapshots for the property-aware Soup/Mail
-// profiles. Preserve queued work; old or partial scans cannot supply new proof.
-const BACKFILL_VERSION = 14;
+// Rehydrate raw file-type projections after retiring enum-normalized facts.
+// Old cursors must not skip records when the cache compatibility epoch changes.
+const BACKFILL_VERSION = 15;
const PAGE_LIMIT = 100;
// Five threads × twenty messages reaches the backend's 100-message cap.
const EMAIL_CONTENT_PAGE_LIMIT = 5;
diff --git a/crates/client/cache-core/src/codec.rs b/crates/client/cache-core/src/codec.rs
index fb06dca717..b1eeb58079 100644
--- a/crates/client/cache-core/src/codec.rs
+++ b/crates/client/cache-core/src/codec.rs
@@ -22,7 +22,8 @@ pub const CACHE_FORMAT_VERSION: u32 = 3;
/// fields, so older records remain usable until a newly selected field is
/// fetched. Bump this epoch for incompatible changes to normalized identity,
/// field storage shape, or other schema-derived cache semantics.
-pub const CACHE_SCHEMA_COMPATIBILITY_EPOCH: u32 = 2;
+// File-type facts now preserve raw server text instead of enum-normalizing it.
+pub const CACHE_SCHEMA_COMPATIBILITY_EPOCH: u32 = 3;
#[derive(Debug, Error)]
pub enum CodecError {
@@ -141,7 +142,7 @@ mod tests {
fn namespace_uses_schema_compatibility_epoch_not_schema_hash() {
assert_eq!(
cache_namespace("client-token-1"),
- "graphql-cache:client-token-1:s2:v3"
+ "graphql-cache:client-token-1:s3:v3"
);
}
}
diff --git a/crates/client/cache-core/src/predicate.rs b/crates/client/cache-core/src/predicate.rs
index d13d368368..ea44bd13cb 100644
--- a/crates/client/cache-core/src/predicate.rs
+++ b/crates/client/cache-core/src/predicate.rs
@@ -19,7 +19,7 @@ use predicate_index::{
ValidatedIndexQuery, ValidationError,
};
use serde::{Deserialize, Serialize};
-use std::collections::{BTreeSet, HashMap};
+use std::collections::{BTreeSet, HashMap, HashSet};
use thiserror::Error;
/// Persisted projection state for one supported normalized record.
@@ -570,6 +570,7 @@ fn patch_exact_members(
remove: &[ExactFact],
insert: &[ExactFact],
) -> Result<(), ValidationError> {
+ let remove = remove.iter().collect::<HashSet<_>>();
document.exact_facts.retain(|fact| !remove.contains(fact));
document.exact_facts.extend_from_slice(insert);
document.canonicalize();
diff --git a/crates/client/cache-core/src/predicate/test.rs b/crates/client/cache-core/src/predicate/test.rs
index d757a879af..d445c2f744 100644
--- a/crates/client/cache-core/src/predicate/test.rs
+++ b/crates/client/cache-core/src/predicate/test.rs
@@ -71,10 +71,53 @@ fn member_edits_are_idempotent_and_preserve_unknown_coverage() {
}
#[test]
-fn member_growth_beyond_budget_is_incomplete_not_an_enqueue_failure() {
+fn large_member_edits_preserve_sets_and_are_idempotent() {
let mut doc = document();
- doc.exact_facts = (0..predicate_index::MAX_FACTS_PER_DOCUMENT)
- .map(|i| fact(&i.to_string()))
+ doc.exact_facts
+ .extend((0..10_000).map(|i| fact(&i.to_string())));
+ let removed = doc.exact_facts.clone();
+ let mutation = OptimisticProjectionMutation::PatchExact {
+ record_key: doc.record_key.clone(),
+ profile: doc.profile.clone(),
+ partition: doc.partition.clone(),
+ remove: removed.clone(),
+ // Overlapping removal/insertion keeps the inserted member.
+ insert: vec![fact("a"), fact("new"), fact("new")],
+ };
+ let projection = compose_effective_optimistic_projection(
+ &doc.record_key,
+ Some(&ProjectionState::Complete(doc.clone())),
+ &[ProjectionMutationLayer {
+ owner: 1,
+ mutations: &[mutation.clone(), mutation],
+ }],
+ )
+ .unwrap()
+ .unwrap();
+ let OptimisticProjectionState::Complete(effective) = projection.state else {
+ panic!("large member sets remain complete");
+ };
+ assert_eq!(effective.exact_facts, vec![fact("a"), fact("new")]);
+
+ let state = apply_authoritative_exact_members(
+ Some(&ProjectionState::Complete(doc.clone())),
+ &doc.record_key,
+ &doc.profile,
+ &doc.partition,
+ &removed,
+ &[fact("a"), fact("new")],
+ );
+ assert_eq!(state, ProjectionState::Complete(effective));
+}
+
+#[test]
+fn attribute_growth_beyond_budget_is_incomplete_not_an_enqueue_failure() {
+ let mut doc = document();
+ doc.exact_facts = (0..predicate_index::MAX_ATTRIBUTES_PER_DOCUMENT)
+ .map(|i| ExactFact {
+ attribute: token(&format!("attribute-{i}")),
+ value: ExactValue::utf8("value").unwrap(),
+ })
.collect();
let mutation = OptimisticProjectionMutation::PatchExact {
record_key: doc.record_key.clone(),
diff --git a/crates/client/cache-wasm/Cargo.toml b/crates/client/cache-wasm/Cargo.toml
index 9343cd74fc..29bcba1214 100644
--- a/crates/client/cache-wasm/Cargo.toml
+++ b/crates/client/cache-wasm/Cargo.toml
@@ -2,8 +2,8 @@
edition = "2024"
name = "cache-wasm"
publish = false
-# Rebuild for canonical Mail tab previews and viewer-scoped shared eligibility.
-version = "0.6.14"
+# Rebuild for raw file-type projections and SQL-compatible nullable filters.
+version = "0.6.16"
[features]
default = []
diff --git a/crates/item_filter_index/src/document.rs b/crates/item_filter_index/src/document.rs
new file mode 100644
index 0000000000..a626a711c1
--- /dev/null
+++ b/crates/item_filter_index/src/document.rs
@@ -0,0 +1,53 @@
+//! Document predicates preserve SQL NULL behavior for the raw file-type column.
+
+use super::{CompileError, DocumentLiteral, Expr, PredicateExpr, vocabulary};
+
+pub(super) fn compile(
+ expr: Option<&Expr<DocumentLiteral>>,
+ literal: impl Fn(&DocumentLiteral) -> Result<PredicateExpr, CompileError> + Copy,
+) -> Result<PredicateExpr, CompileError> {
+ match expr {
+ Some(expr) => compile_inner(expr, false, literal),
+ None => Ok(PredicateExpr::All),
+ }
+}
+
+fn compile_inner(
+ expr: &Expr<DocumentLiteral>,
+ negated: bool,
+ literal: impl Fn(&DocumentLiteral) -> Result<PredicateExpr, CompileError> + Copy,
+) -> Result<PredicateExpr, CompileError> {
+ Ok(match expr {
+ Expr::Literal(value) => {
+ let predicate = literal(value)?;
+ if !negated {
+ predicate
+ } else if matches!(value, DocumentLiteral::FileType(_)) {
+ // SQL NULL makes both fileType = x and NOT(fileType = x)
+ // UNKNOWN, so neither matches a WHERE clause. An absent fact
+ // already fails equality; negated equality also needs presence.
+ PredicateExpr::And(
+ Box::new(PredicateExpr::ExactExists {
+ attribute: vocabulary::file_type(),
+ }),
+ Box::new(PredicateExpr::Not(Box::new(predicate))),
+ )
+ } else {
+ PredicateExpr::Not(Box::new(predicate))
+ }
+ }
+ Expr::Not(inner) => compile_inner(inner, !negated, literal)?,
+ Expr::And(left, right) | Expr::Or(left, right) => {
+ let left = Box::new(compile_inner(left, negated, literal)?);
+ let right = Box::new(compile_inner(right, negated, literal)?);
+ // Push NOT to the leaves using De Morgan's laws. Guarding the whole
+ // tree would incorrectly reject NULL rows matched by another OR arm;
+ // complementing a compiled subtree would incorrectly include NULLs.
+ if matches!(expr, Expr::And(..)) != negated {
+ PredicateExpr::And(left, right)
+ } else {
+ PredicateExpr::Or(left, right)
+ }
+ }
+ })
+}
diff --git a/crates/item_filter_index/src/lib.rs b/crates/item_filter_index/src/lib.rs
index 8137da5f9b..657d346d77 100644
--- a/crates/item_filter_index/src/lib.rs
+++ b/crates/item_filter_index/src/lib.rs
@@ -29,6 +29,7 @@ use uuid::Uuid;
mod test;
mod channels;
+mod document;
pub mod mail;
pub mod properties;
@@ -148,7 +149,7 @@ pub mod vocabulary {
token("owner")
}
- /// Document file-type attribute.
+ /// Raw stored document file-type text; absent for SQL NULL.
pub fn file_type() -> Token {
token("file-type")
}
@@ -529,7 +530,7 @@ fn compile_soup_flat(
SoupIndexSort::Unsupported => unreachable!("eligibility checked sort"),
};
let mut document_predicate =
- compile_expr(ast.document_filter.as_deref(), compile_document_literal)?;
+ document::compile(ast.document_filter.as_deref(), compile_document_literal)?;
if let Some(properties_filter) = ast.properties_filter.as_deref() {
let compile_properties_literal =
compile_properties_literal.expect("eligibility checked property support");
diff --git a/crates/predicate_index/src/lib.rs b/crates/predicate_index/src/lib.rs
index b6e37d3767..8687a37a36 100644
--- a/crates/predicate_index/src/lib.rs
+++ b/crates/predicate_index/src/lib.rs
@@ -23,8 +23,9 @@ pub const MAX_QUERY_LIMIT: u16 = 500;
pub const MAX_TOKEN_BYTES: usize = 128;
/// Maximum bytes in an exact value.
pub const MAX_EXACT_VALUE_BYTES: usize = 16 * 1_024;
-/// Maximum facts in one index document.
-pub const MAX_FACTS_PER_DOCUMENT: usize = 256;
+/// Maximum distinct attribute names in one index document or patch.
+/// Set-valued attributes may contain any number of individually bounded values.
+pub const MAX_ATTRIBUTES_PER_DOCUMENT: usize = 256;
/// Maximum distinct records whose optimistic projections may be merged into one query.
pub const MAX_OPTIMISTIC_RECORDS_PER_QUERY: usize = 128;
@@ -227,7 +228,7 @@ impl PredicateExpr {
}
/// One exact fact in an index document.
-#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub struct ExactFact {
/// Fact vocabulary token.
pub attribute: Token,
@@ -261,6 +262,19 @@ pub struct IndexDocument {
pub sort_facts: Vec<IntegerFact>,
}
+fn validate_attribute_count<'a>(
+ attributes: impl IntoIterator<Item = &'a Token>,
+) -> Result<(), ValidationError> {
+ let mut distinct = HashSet::new();
+ for attribute in attributes {
+ distinct.insert(attribute);
+ if distinct.len() > MAX_ATTRIBUTES_PER_DOCUMENT {
+ return Err(ValidationError::DocumentAttributes);
+ }
+ }
+ Ok(())
+}
+
impl IndexDocument {
/// Put facts in deterministic order and remove duplicate membership facts.
pub fn canonicalize(&mut self) {
@@ -271,12 +285,15 @@ impl IndexDocument {
self.sort_facts.sort();
}
- /// Validate bounded fact counts and unique sort attributes.
+ /// Validate bounded attribute counts and unique sort attributes.
pub fn validate(&self) -> Result<(), ValidationError> {
- let facts = self.exact_facts.len() + self.integer_facts.len() + self.sort_facts.len();
- if facts > MAX_FACTS_PER_DOCUMENT {
- return Err(ValidationError::DocumentFacts);
- }
+ validate_attribute_count(
+ self.exact_facts
+ .iter()
+ .map(|fact| &fact.attribute)
+ .chain(self.integer_facts.iter().map(|fact| &fact.attribute))
+ .chain(self.sort_facts.iter().map(|fact| &fact.attribute)),
+ )?;
let mut sort_attributes = HashSet::new();
if self
.sort_facts
@@ -730,17 +747,20 @@ impl OptimisticProjectionMutation {
return match self {
Self::Replace(document) => document.validate(),
Self::Delete { .. } | Self::Unknown { .. } => Ok(()),
- Self::PatchExact { remove, insert, .. } => {
- if remove.len() + insert.len() > MAX_FACTS_PER_DOCUMENT {
- Err(ValidationError::DocumentFacts)
- } else {
- Ok(())
- }
- }
+ Self::PatchExact { remove, insert, .. } => validate_attribute_count(
+ remove.iter().chain(insert).map(|fact| &fact.attribute),
+ ),
Self::Patch { .. } => unreachable!(),
};
};
+ validate_attribute_count(
+ exact
+ .iter()
+ .map(|patch| &patch.attribute)
+ .chain(integers.iter().map(|patch| &patch.attribute))
+ .chain(sorts.iter().map(|fact| &fact.attribute)),
+ )?;
let mut attributes = HashSet::new();
if exact
.iter()
@@ -835,9 +855,9 @@ pub enum ValidationError {
/// An integer range is inverted or empty.
#[error("invalid integer range")]
InvalidRange,
- /// A document contains too many facts.
- #[error("index document contains too many facts")]
- DocumentFacts,
+ /// A document or patch contains too many distinct attribute names.
+ #[error("index document contains too many distinct attributes")]
+ DocumentAttributes,
/// A document has more than one sort value for an attribute.
#[error("duplicate sort fact attribute")]
DuplicateSortFact,
diff --git a/crates/predicate_index/src/test.rs b/crates/predicate_index/src/test.rs
index a8f4b14ab9..d091f0bc27 100644
--- a/crates/predicate_index/src/test.rs
+++ b/crates/predicate_index/src/test.rs
@@ -238,6 +238,94 @@ fn rejects_empty_ranges_duplicate_partitions_and_duplicate_sort_facts() {
assert_eq!(document.validate(), Err(ValidationError::DuplicateSortFact));
}
+#[test]
+fn set_cardinality_does_not_consume_the_attribute_budget() {
+ let mut doc = document("Document:1", "user-1", 10);
+ doc.exact_facts.extend((0..10_000).map(|i| ExactFact {
+ attribute: token("members"),
+ value: exact(&i.to_string()),
+ }));
+ doc.integer_facts
+ .extend((0..10_000).map(|value| IntegerFact {
+ attribute: token("numbers"),
+ value,
+ }));
+ doc.validate().unwrap();
+
+ let patch = OptimisticProjectionMutation::PatchExact {
+ record_key: doc.record_key.clone(),
+ profile: doc.profile.clone(),
+ partition: doc.partition.clone(),
+ remove: doc.exact_facts.clone(),
+ insert: doc.exact_facts.clone(),
+ };
+ patch.validate().unwrap();
+ assert!(matches!(
+ ExactValue::new(vec![0; MAX_EXACT_VALUE_BYTES + 1]),
+ Err(ValidationError::ExactValueTooLarge(_))
+ ));
+}
+
+#[test]
+fn document_and_patch_attribute_budgets_include_every_fact_family() {
+ let mut doc = document("Document:1", "user-1", 10);
+ doc.exact_facts = (0..MAX_ATTRIBUTES_PER_DOCUMENT - 1)
+ .map(|i| ExactFact {
+ attribute: token(&format!("attribute-{i}")),
+ value: exact("value"),
+ })
+ .collect();
+ // updated-at is shared by integer and sort facts: one attribute name.
+ doc.validate().unwrap();
+ doc.integer_facts.push(IntegerFact {
+ attribute: token("extra"),
+ value: 1,
+ });
+ assert_eq!(doc.validate(), Err(ValidationError::DocumentAttributes));
+
+ let patch = OptimisticProjectionMutation::Patch {
+ record_key: doc.record_key.clone(),
+ profile: doc.profile.clone(),
+ partition: doc.partition.clone(),
+ exact: doc
+ .exact_facts
+ .iter()
+ .map(|fact| ExactAttributePatch {
+ attribute: fact.attribute.clone(),
+ values: vec![fact.value.clone()],
+ })
+ .collect(),
+ integers: doc
+ .integer_facts
+ .iter()
+ .map(|fact| IntegerAttributePatch {
+ attribute: fact.attribute.clone(),
+ values: vec![fact.value],
+ })
+ .collect(),
+ sorts: doc.sort_facts.clone(),
+ };
+ assert_eq!(patch.validate(), Err(ValidationError::DocumentAttributes));
+
+ let patch = OptimisticProjectionMutation::PatchExact {
+ record_key: doc.record_key,
+ profile: doc.profile,
+ partition: doc.partition,
+ remove: doc.exact_facts,
+ insert: vec![
+ ExactFact {
+ attribute: token("updated-at"),
+ value: exact("one"),
+ },
+ ExactFact {
+ attribute: token("extra"),
+ value: exact("two"),
+ },
+ ],
+ };
+ assert_eq!(patch.validate(), Err(ValidationError::DocumentAttributes));
+}
+
#[test]
fn uncertainty_can_clear_exact_fields_after_a_wildcard() {
let owner = token("owner");
diff --git a/crates/soup_filter_cache_adapter/src/notifications.rs b/crates/soup_filter_cache_adapter/src/notifications.rs
index a70a7a1f91..7916c17931 100644
--- a/crates/soup_filter_cache_adapter/src/notifications.rs
+++ b/crates/soup_filter_cache_adapter/src/notifications.rs
@@ -9,6 +9,8 @@ use cache_core::{
};
use predicate_index::ExactFact;
+mod batch;
+
#[cfg(all(test, not(target_arch = "wasm32")))]
mod test;
@@ -237,6 +239,7 @@ async fn retain_complete_parents<S: Storage>(
storage: &S,
mutations: Vec<ProjectionMutation>,
) -> Result<Vec<ProjectionMutation>, SoupFilterCacheAdapterError> {
+ let mutations = batch::batch_member_changes(mutations);
if mutations.is_empty() {
return Ok(mutations);
}
diff --git a/crates/soup_filter_cache_adapter/src/notifications/batch.rs b/crates/soup_filter_cache_adapter/src/notifications/batch.rs
new file mode 100644
index 0000000000..41304f848a
--- /dev/null
+++ b/crates/soup_filter_cache_adapter/src/notifications/batch.rs
@@ -0,0 +1,97 @@
+//! Coalesce notification member edits before composing a parent's projection.
+
+use cache_core::predicate::{ProjectionIncompleteKind, ProjectionMutation};
+use indexmap::IndexMap;
+use predicate_index::{ExactFact, Profile, RecordKey, Token};
+use std::collections::BTreeSet;
+
+struct Batch {
+ profile: Profile,
+ partition: Token,
+ state: BatchState,
+}
+
+enum BatchState {
+ Members {
+ remove: BTreeSet<ExactFact>,
+ insert: BTreeSet<ExactFact>,
+ },
+ Incomplete(ProjectionIncompleteKind),
+}
+
+/// Notification updates contain only member edits or invalidation. Combining
+/// them per parent avoids copying and sorting a large projection for every child.
+/// This does not combine separate optimistic queue layers or reorder snapshots.
+pub(super) fn batch_member_changes(mutations: Vec<ProjectionMutation>) -> Vec<ProjectionMutation> {
+ let mut batches = IndexMap::<RecordKey, Batch>::new();
+ for mutation in mutations {
+ match mutation {
+ ProjectionMutation::PatchExact {
+ record_key,
+ profile,
+ partition,
+ remove,
+ insert,
+ } => {
+ let batch = batches.entry(record_key).or_insert_with(|| Batch {
+ profile,
+ partition,
+ state: BatchState::Members {
+ remove: BTreeSet::new(),
+ insert: BTreeSet::new(),
+ },
+ });
+ if let BatchState::Members {
+ remove: removed,
+ insert: inserted,
+ } = &mut batch.state
+ {
+ // Later removals override earlier inserts. Later inserts win
+ // over removals, including a move between unseen and seen.
+ for fact in remove {
+ inserted.remove(&fact);
+ removed.insert(fact);
+ }
+ inserted.extend(insert);
+ }
+ }
+ ProjectionMutation::MarkIncomplete {
+ record_key,
+ profile,
+ partition,
+ kind,
+ } => {
+ batches.insert(
+ record_key,
+ Batch {
+ profile,
+ partition,
+ state: BatchState::Incomplete(kind),
+ },
+ );
+ }
+ _ => unreachable!("notification updates are member edits or invalidation"),
+ }
+ }
+ batches
+ .into_iter()
+ .map(|(record_key, batch)| match batch.state {
+ BatchState::Members { remove, insert } => ProjectionMutation::PatchExact {
+ record_key,
+ profile: batch.profile,
+ partition: batch.partition,
+ remove: remove.into_iter().collect(),
+ insert: insert.into_iter().collect(),
+ },
+ BatchState::Incomplete(kind) => ProjectionMutation::MarkIncomplete {
+ record_key,
+ profile: batch.profile,
+ partition: batch.partition,
+ kind,
+ },
+ })
+ .collect()
+}
+
+#[cfg(test)]
+mod test;
diff --git a/crates/soup_filter_cache_adapter/src/notifications/batch/test.rs b/crates/soup_filter_cache_adapter/src/notifications/batch/test.rs
new file mode 100644
index 0000000000..7dd135fbb4
--- /dev/null
+++ b/crates/soup_filter_cache_adapter/src/notifications/batch/test.rs
@@ -0,0 +1,85 @@
+use super::*;
+use crate::notifications::change;
+use cache_core::predicate::{ProjectionState, apply_authoritative_projection_mutations};
+use item_filter_index::vocabulary;
+use predicate_index::IndexDocument;
+use std::collections::HashMap;
+
+fn key(id: u128) -> RecordKey {
+ RecordKey::new(format!("GraphqlSoupDocument:{}", uuid::Uuid::from_u128(id))).unwrap()
+}
+
+fn update(parent: u128, id: u128, state: &str) -> ProjectionMutation {
+ change(
+ key(parent),
+ vocabulary::document_partition(),
+ uuid::Uuid::from_u128(id),
+ Some(state),
+ )
+}
+
+#[test]
+fn batching_preserves_last_member_state_and_independent_parents() {
+ let mutations = vec![
+ update(1, 10, "UNSEEN"),
+ update(2, 10, "SEEN"),
+ update(1, 11, "SEEN"),
+ update(1, 10, "SEEN"),
+ update(1, 11, "DONE"),
+ update(1, 10, "SEEN"),
+ update(1, 11, "UNSEEN"),
+ ];
+ let bases = [1, 2]
+ .into_iter()
+ .map(|id| {
+ let key = key(id);
+ (
+ key.clone(),
+ ProjectionState::Complete(IndexDocument {
+ record_key: key,
+ profile: vocabulary::profile_v4(),
+ partition: vocabulary::document_partition(),
+ exact_facts: vec![],
+ integer_facts: vec![],
+ sort_facts: vec![],
+ }),
+ )
+ })
+ .collect::<HashMap<_, _>>();
+ let mut expected = bases.clone();
+ apply_authoritative_projection_mutations(&mut expected, &mutations);
+ let batched = batch_member_changes(mutations);
+ assert_eq!(batched.len(), 2);
+ let mut actual = bases;
+ apply_authoritative_projection_mutations(&mut actual, &batched);
+ assert_eq!(actual, expected);
+ apply_authoritative_projection_mutations(&mut actual, &batched);
+ assert_eq!(actual, expected, "replaying a batch must be idempotent");
+}
+
+#[test]
+fn member_edits_cannot_clear_invalidation_in_the_same_batch() {
+ for mutations in [
+ vec![update(1, 10, "INVALID"), update(1, 11, "UNSEEN")],
+ vec![update(1, 10, "UNSEEN"), update(1, 11, "INVALID")],
+ ] {
+ let batched = batch_member_changes(mutations);
+ assert!(matches!(
+ batched.as_slice(),
+ [ProjectionMutation::MarkIncomplete {
+ kind: ProjectionIncompleteKind::Dirty,
+ ..
+ }]
+ ));
+ }
+}
+
+#[test]
+fn many_notifications_produce_one_member_edit_per_parent() {
+ let batched = batch_member_changes((1..=10_000).map(|id| update(1, id, "SEEN")).collect());
+ let [ProjectionMutation::PatchExact { remove, insert, .. }] = batched.as_slice() else {
+ panic!("one parent should be composed only once");
+ };
+ assert_eq!(remove.len(), 20_000);
+ assert_eq!(insert.len(), 10_000);
+}
diff --git a/crates/soup_filter_cache_adapter/src/notifications/test.rs b/crates/soup_filter_cache_adapter/src/notifications/test.rs
index 091f962252..2552479921 100644
--- a/crates/soup_filter_cache_adapter/src/notifications/test.rs
+++ b/crates/soup_filter_cache_adapter/src/notifications/test.rs
@@ -11,6 +11,8 @@ use predicate_index::{
use serde_json::{Value, json};
use soup_filter_projection::{SoupCacheProjectionSupplement, encode_cache_projection_supplement};
+mod large;
+
const D: &str = "00000000-0000-0000-0000-000000000001";
const A: &str = "00000000-0000-0000-0000-000000000011";
const B: &str = "00000000-0000-0000-0000-000000000012";
@@ -564,7 +566,7 @@ fn optimistic_notification_conversion_rejects_unsupported_mutations() {
}
#[test]
-fn snapshot_completeness_aliases_primary_scope_and_bounds() {
+fn snapshot_completeness_aliases_and_primary_scope() {
let data = snapshot(vec![notification(A, "UNSEEN"), notification(B, "SEEN")]);
let mutations = authoritative_projection_mutations(SNAPSHOT, None, &data).unwrap();
let [ProjectionMutation::Replace(document)] = mutations.as_slice() else {
@@ -622,15 +624,33 @@ fn snapshot_completeness_aliases_primary_scope_and_bounds() {
authoritative_projection_mutations(alias_query.as_str(), None, &alias_data).unwrap(),
authoritative_projection_mutations(SNAPSHOT, None, &data).unwrap()
);
- let too_many = (1..=256)
+}
+
+#[test]
+fn backfill_accepts_large_notification_sets_without_losing_members() {
+ let notifications = (1..=10_000)
.map(|n| notification(&uuid::Uuid::from_u128(n + 1000).to_string(), "UNSEEN"))
.collect();
- assert!(matches!(
- authoritative_projection_mutations(SNAPSHOT, None, &snapshot(too_many))
- .unwrap()
- .as_slice(),
- [ProjectionMutation::MarkIncomplete { .. }]
- ));
+ let backfill = SNAPSHOT.replace("query Snapshot", "query SoupBackfill");
+ let mutations = authoritative_projection_mutations(
+ &backfill,
+ Some("SoupBackfill"),
+ &snapshot(notifications),
+ )
+ .unwrap();
+ let [ProjectionMutation::Replace(document)] = mutations.as_slice() else {
+ panic!("large notification snapshots must remain complete");
+ };
+ assert_eq!(
+ document
+ .exact_facts
+ .iter()
+ .filter(|fact| { fact.attribute == vocabulary::notification_unseen() })
+ .count(),
+ 10_000
+ );
+ assert!(document.matches(&unseen()));
+ assert!(!document.matches(&seen()));
}
#[test]
diff --git a/crates/soup_filter_cache_adapter/src/notifications/test/large.rs b/crates/soup_filter_cache_adapter/src/notifications/test/large.rs
new file mode 100644
index 0000000000..3916056461
--- /dev/null
+++ b/crates/soup_filter_cache_adapter/src/notifications/test/large.rs
@@ -0,0 +1,148 @@
+use super::*;
+
+const MEMBER_COUNT: u128 = 1_024;
+
+fn member_id(index: u128) -> String {
+ uuid::Uuid::from_u128(index + 1000).to_string()
+}
+
+fn large_snapshot() -> Value {
+ let mut notifications = (1..=MEMBER_COUNT)
+ .map(|id| notification(&member_id(id), "SEEN"))
+ .collect::<Vec<_>>();
+ notifications.extend([notification(A, "UNSEEN"), notification(B, "UNSEEN")]);
+ snapshot(notifications)
+}
+
+async fn large_set_lifecycle<S: PredicateIndexStorage>(storage: S) {
+ let mut engine = Engine::new(storage);
+ write(&mut engine, SNAPSHOT, &large_snapshot()).await;
+ assert!(matches(&mut engine, unseen()).await);
+ assert!(matches(&mut engine, seen()).await);
+
+ let data = json!({"updateNotifications": (1..=MEMBER_COUNT).map(|id| json!({
+ "__typename": "GraphqlNotification", "id": member_id(id), "state": "DONE"
+ })).collect::<Vec<_>>()});
+ let vars = variables();
+ let edits = notification_projection_updates(engine.storage(), UPDATE, None, &vars, &data)
+ .await
+ .unwrap();
+ let [ProjectionMutation::PatchExact { remove, insert, .. }] = edits.as_slice() else {
+ panic!("bulk updates compose the parent once");
+ };
+ assert_eq!(remove.len(), MEMBER_COUNT as usize * 2);
+ assert!(insert.is_empty());
+ let (bulk, _) = engine
+ .begin_optimistic_write_with_projections(
+ None,
+ BeginOptimisticWrite {
+ uuid: "00000000-0000-0000-0000-000000000101",
+ query: UPDATE,
+ operation_name: None,
+ variables: &vars,
+ data: &data,
+ link_patches: &[],
+ revalidations: &[],
+ created_at_ms: 1,
+ },
+ optimistic_notification_updates(edits).unwrap(),
+ )
+ .await
+ .unwrap();
+ let single = mark_done(&mut engine, A, "00000000-0000-0000-0000-000000000102").await;
+ assert!(!matches(&mut engine, seen()).await);
+ assert!(matches(&mut engine, unseen()).await, "B is still unseen");
+
+ let mut engine = Engine::new(engine.into_storage());
+ assert!(
+ !matches(&mut engine, seen()).await,
+ "pending bulk edit survives reopen"
+ );
+ let token = claim(&mut engine, bulk).await;
+ engine.rollback_optimistic_write(bulk, token).await.unwrap();
+ assert!(
+ matches(&mut engine, seen()).await,
+ "rollback restores all seen members"
+ );
+
+ // A remains optimistically done while B changes through realtime.
+ write(
+ &mut engine,
+ UPDATE,
+ &json!({"updateNotifications":[notification(B, "SEEN")]}),
+ )
+ .await;
+ assert!(!matches(&mut engine, unseen()).await);
+ // A full refresh rebases, rather than replacing, the pending removal of A.
+ write(&mut engine, SNAPSHOT, &large_snapshot()).await;
+ assert!(matches(&mut engine, unseen()).await);
+ write(
+ &mut engine,
+ UPDATE,
+ &json!({"updateNotifications":[notification(B, "SEEN")]}),
+ )
+ .await;
+ assert!(
+ !matches(&mut engine, unseen()).await,
+ "refresh must not resurrect A"
+ );
+
+ let token = claim(&mut engine, single).await;
+ let settled = json!({"updateNotifications":[notification(A, "DONE")]});
+ let edits = notification_projection_updates(engine.storage(), UPDATE, None, &vars, &settled)
+ .await
+ .unwrap();
+ engine
+ .commit_optimistic_write_with_projections(
+ single, token, UPDATE, None, &vars, &settled, edits,
+ )
+ .await
+ .unwrap();
+
+ let keys = (1..=MEMBER_COUNT)
+ .map(|id| format!("GraphqlNotification:{}", member_id(id)))
+ .collect::<Vec<_>>();
+ let edits = notification_deletion_updates(engine.storage(), &keys, false)
+ .await
+ .unwrap();
+ assert_eq!(
+ edits.len(),
+ 1,
+ "bulk deletions also compose the parent once"
+ );
+ let keys = keys
+ .into_iter()
+ .map(|key| EntityKey(key.into()))
+ .collect::<Vec<_>>();
+ engine
+ .delete_keys_with_projection_changes(&keys, edits)
+ .await
+ .unwrap();
+ assert!(
+ matches(&mut engine, seen()).await,
+ "B remains after unrelated members are deleted"
+ );
+ write(
+ &mut engine,
+ UPDATE,
+ &json!({"updateNotifications":[notification(B, "DONE")]}),
+ )
+ .await;
+ assert!(!matches(&mut engine, seen()).await);
+ assert!(!matches(&mut engine, unseen()).await);
+}
+
+#[test]
+fn large_notification_lifecycle_in_memory() {
+ pollster::block_on(large_set_lifecycle(InMemoryStorage::new()));
+}
+
+#[test]
+fn large_notification_lifecycle_in_turso() {
+ pollster::block_on(async {
+ large_set_lifecycle(
+ cache_turso::TursoStorage::open_in_memory("large-notifications").unwrap(),
+ )
+ .await
+ });
+}
diff --git a/crates/soup_filter_cache_adapter/src/test.rs b/crates/soup_filter_cache_adapter/src/test.rs
index 9d6c6bce23..9bc823889e 100644
--- a/crates/soup_filter_cache_adapter/src/test.rs
+++ b/crates/soup_filter_cache_adapter/src/test.rs
@@ -7,6 +7,8 @@ use std::collections::BTreeMap;
use super::*;
+mod file_type;
+
#[test]
fn optimistic_soup_payloads_compile_to_durable_projection_layers() {
let mutations = optimistic_projection_mutations(
diff --git a/crates/soup_filter_cache_adapter/src/test/file_type.rs b/crates/soup_filter_cache_adapter/src/test/file_type.rs
new file mode 100644
index 0000000000..91f0a29e36
--- /dev/null
+++ b/crates/soup_filter_cache_adapter/src/test/file_type.rs
@@ -0,0 +1,420 @@
+use super::*;
+use cache_core::{
+ engine::{BeginOptimisticWrite, Engine, NetworkWrite},
+ predicate::{PredicateIndexStorage, PredicateQueryResult},
+ queue::{MutationClaimRequest, MutationClaimToken},
+ store::InMemoryStorage,
+};
+use serde_json::{Value, json};
+
+const FILE_TYPES: [Option<&str>; 8] = [
+ Some("pdf"),
+ Some("doc"),
+ Some("custom.ext"),
+ Some("PDF"),
+ Some(".pdf"),
+ Some(""),
+ None,
+ Some("md"),
+];
+const RENAME: &str = r#"mutation Rename($inputs: [RenameEntityInput!]!) {
+ renameEntities(inputs: $inputs) { results { __typename
+ ... on GraphqlMutationSuccess { effects { __typename
+ ... on SoupUpdated { item { __typename id
+ ... on GraphqlSoupDocument { fileType updatedAt }
+ } }
+ } }
+ } }
+}"#;
+
+fn id(index: usize) -> String {
+ uuid::Uuid::from_u128(index as u128 + 1).to_string()
+}
+fn key(index: usize) -> RecordKey {
+ RecordKey::new(format!("GraphqlSoupDocument:{}", id(index))).unwrap()
+}
+fn item(index: usize) -> Value {
+ let mut item = selected_document(
+ &id(index),
+ json!(document_supplement(&id(index), false)),
+ Value::Null,
+ );
+ item["fileType"] = json!(FILE_TYPES[index]);
+ item["properties"] = json!([]);
+ item
+}
+fn filters(document: Value) -> Value {
+ let nil = uuid::Uuid::nil().to_string();
+ json!({
+ "documentFilter": document,
+ "calendarEventFilter": {"literal":{"id":nil}},
+ "projectFilter": {"literal":{"projectIdSelf":nil}},
+ "chatFilter": {"literal":{"chatId":nil}},
+ "emailFilter": {"tree":{"literal":{"threadId":nil}}},
+ "channelFilter": {"literal":{"channelId":nil}},
+ "channelThreadFilter": {"literal":{"threadId":nil}},
+ "callFilter": {"literal":{"callId":nil}},
+ "crmCompanyFilter": {"literal":{"id":nil}},
+ "foreignEntityFilter": {"literal":{"id":nil}}
+ })
+}
+fn query(document: Value) -> ValidatedIndexQuery {
+ let SoupFilterCompileOutcome::Supported(query) =
+ compile_current_filter_request(filters(document), "UPDATED_AT", "DESC", 20).unwrap()
+ else {
+ panic!("file-type filters must be locally supported");
+ };
+ query
+}
+fn not(expr: Value) -> Value {
+ json!({"not":expr})
+}
+
+// Independent SQL three-valued reference: WHERE selects only True, not Unknown.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+enum SqlTruth {
+ True,
+ False,
+ Unknown,
+}
+impl SqlTruth {
+ fn from_bool(value: bool) -> Self {
+ if value { Self::True } else { Self::False }
+ }
+ fn negate(self) -> Self {
+ match self {
+ Self::True => Self::False,
+ Self::False => Self::True,
+ Self::Unknown => Self::Unknown,
+ }
+ }
+ fn and(self, other: Self) -> Self {
+ match (self, other) {
+ (Self::False, _) | (_, Self::False) => Self::False,
+ (Self::True, Self::True) => Self::True,
+ _ => Self::Unknown,
+ }
+ }
+ fn or(self, other: Self) -> Self {
+ match (self, other) {
+ (Self::True, _) | (_, Self::True) => Self::True,
+ (Self::False, Self::False) => Self::False,
+ _ => Self::Unknown,
+ }
+ }
+}
+fn sql_reference(expr: &Value, index: usize) -> SqlTruth {
+ if let Some(literal) = expr.get("literal") {
+ if let Some(file_type) = literal.get("fileType") {
+ return FILE_TYPES[index].map_or(SqlTruth::Unknown, |stored| {
+ SqlTruth::from_bool(stored == file_type.as_str().unwrap())
+ });
+ }
+ return SqlTruth::from_bool(literal["id"].as_str().unwrap() == id(index));
+ }
+ if let Some(expr) = expr.get("not") {
+ return sql_reference(expr, index).negate();
+ }
+ if let Some(and) = expr.get("and") {
+ return sql_reference(&and["left"], index).and(sql_reference(&and["right"], index));
+ }
+ let or = &expr["or"];
+ sql_reference(&or["left"], index).or(sql_reference(&or["right"], index))
+}
+fn expressions() -> Vec<Value> {
+ let atoms = [
+ json!({"literal":{"fileType":"pdf"}}),
+ json!({"literal":{"fileType":"md"}}),
+ json!({"literal":{"id":id(6)}}),
+ ];
+ let mut expressions = Vec::new();
+ for atom in &atoms {
+ expressions.extend([atom.clone(), not(atom.clone()), not(not(atom.clone()))]);
+ }
+ for left in &atoms {
+ for right in &atoms {
+ for op in ["and", "or"] {
+ let expr = json!({op:{"left":left,"right":right}});
+ expressions.extend([expr.clone(), not(expr.clone()), not(not(expr))]);
+ }
+ }
+ }
+ expressions.push(not(json!({"or": {
+ "left": not(atoms[0].clone()),
+ "right": {"and":{"left":atoms[1],"right":not(atoms[2].clone())}}
+ }})));
+ expressions
+}
+
+async fn seed<S: PredicateIndexStorage>(engine: &mut Engine<S>) {
+ let query = SUPPLEMENT_BACKFILL
+ .replace("user {", "user { id")
+ .replace("fileType", "fileType properties { id }");
+ let data = json!({"user":{"id":"viewer","soup":{"items":(0..FILE_TYPES.len()).map(item).collect::<Vec<_>>()}}});
+ let vars = serde_json::Map::new();
+ let projections =
+ authoritative_projection_mutations(&query, Some("SoupBackfill"), &data).unwrap();
+ let projections = crate::properties::augment_authoritative(
+ engine.storage(),
+ &query,
+ Some("SoupBackfill"),
+ &vars,
+ &data,
+ true,
+ projections,
+ )
+ .await
+ .unwrap();
+ engine
+ .hydrate_query_with_projections(
+ &query,
+ Some("SoupBackfill"),
+ &vars,
+ &data,
+ Some("viewer"),
+ projections,
+ )
+ .await
+ .unwrap();
+}
+async fn assert_sql_parity<S: PredicateIndexStorage>(engine: &mut Engine<S>) {
+ for expression in expressions() {
+ let expected = (0..FILE_TYPES.len())
+ .rev()
+ .filter(|&i| sql_reference(&expression, i) == SqlTruth::True)
+ .map(key)
+ .collect();
+ let actual = engine
+ .query_predicate_index(&query(expression.clone()))
+ .await
+ .unwrap()
+ .value;
+ assert_eq!(
+ actual,
+ PredicateQueryResult::Complete(expected),
+ "{expression}"
+ );
+ }
+}
+async fn assert_updated_type<S: PredicateIndexStorage>(engine: &mut Engine<S>, value: &Value) {
+ let file_type = json!({"literal":{"fileType":"pdf"}});
+ for (filter, matches) in [
+ (file_type.clone(), value == "pdf"),
+ (not(file_type), !value.is_null() && value != "pdf"),
+ ] {
+ let query = query(json!({"and":{"left":{"literal":{"id":id(0)}},"right":filter}}));
+ let result = engine.query_predicate_index(&query).await.unwrap().value;
+ let keys = match result {
+ PredicateQueryResult::Complete(keys) | PredicateQueryResult::Optimistic(keys) => keys,
+ PredicateQueryResult::Incomplete => {
+ panic!("file-type changes must preserve projection completeness")
+ }
+ };
+ assert_eq!(keys, if matches { vec![key(0)] } else { vec![] });
+ }
+}
+
+async fn lifecycle<S: PredicateIndexStorage>(storage: S) {
+ let mut engine = Engine::new(storage);
+ seed(&mut engine).await;
+ assert_sql_parity(&mut engine).await;
+ let mut engine = Engine::new(engine.into_storage());
+ assert_sql_parity(&mut engine).await;
+
+ // A partial authoritative update can replace a known type with an unknown
+ // string or NULL without losing the complete supplement/property proof.
+ let vars =
+ json!({"inputs":[{"entity":{"type":"DOCUMENT","id":id(0)},"displayName":"renamed"}]})
+ .as_object()
+ .unwrap()
+ .clone();
+ for value in [json!("legacy-format"), Value::Null, json!("pdf")] {
+ seed(&mut engine).await;
+ let data = json!({"renameEntities":{"results":[{"__typename":"GraphqlMutationSuccess","effects":[{"__typename":"SoupUpdated","item":{
+ "__typename":"GraphqlSoupDocument","id":id(0),"fileType":value,"updatedAt":"2025-01-02T00:00:00.000001Z"
+ }}]}]}});
+ // Start from pdf, then change its membership only in the pending layer.
+ let projections = crate::properties::augment_optimistic(
+ engine.storage(),
+ RENAME,
+ Some("Rename"),
+ &vars,
+ &data,
+ optimistic_projection_mutations(&data, 1),
+ )
+ .await
+ .unwrap();
+ let (transaction, _) = engine
+ .begin_optimistic_write_with_projections(
+ None,
+ BeginOptimisticWrite {
+ uuid: "00000000-0000-0000-0000-000000000100",
+ query: RENAME,
+ operation_name: Some("Rename"),
+ variables: &vars,
+ data: &data,
+ link_patches: &[],
+ revalidations: &[],
+ created_at_ms: 1,
+ },
+ projections,
+ )
+ .await
+ .unwrap();
+ assert_updated_type(&mut engine, &value).await;
+ let mut reopened = Engine::new(engine.into_storage());
+ assert_updated_type(&mut reopened, &value).await;
+ let claimed = reopened
+ .claim_next_mutation(MutationClaimRequest {
+ owner: "test".into(),
+ now_ms: 1,
+ lease_expires_at_ms: 100,
+ })
+ .await
+ .unwrap()
+ .unwrap();
+ assert_eq!(claimed.queued.id, transaction);
+ reopened
+ .rollback_optimistic_write(
+ transaction,
+ MutationClaimToken {
+ owner: "test".into(),
+ generation: claimed.lease_generation,
+ },
+ )
+ .await
+ .unwrap();
+ engine = reopened;
+ assert_updated_type(&mut engine, &json!("pdf")).await;
+
+ let projections =
+ authoritative_projection_mutations(RENAME, Some("Rename"), &data).unwrap();
+ let projections = crate::properties::augment_authoritative(
+ engine.storage(),
+ RENAME,
+ Some("Rename"),
+ &vars,
+ &data,
+ true,
+ projections,
+ )
+ .await
+ .unwrap();
+ engine
+ .write_query_with_registration_and_projections(
+ None,
+ None,
+ NetworkWrite {
+ query: RENAME,
+ operation_name: Some("Rename"),
+ variables: &vars,
+ data: &data,
+ identity: Some("viewer"),
+ },
+ projections,
+ )
+ .await
+ .unwrap();
+ assert_updated_type(&mut engine, &value).await;
+ }
+ assert_sql_parity(&mut engine).await;
+}
+
+#[test]
+fn every_soup_profile_compiles_file_type_negation_with_sql_null_semantics() {
+ let data =
+ json!({"user":{"soup":{"items":(0..FILE_TYPES.len()).map(item).collect::<Vec<_>>()}}});
+ let documents =
+ authoritative_projection_mutations(SUPPLEMENT_BACKFILL, Some("SoupBackfill"), &data)
+ .unwrap();
+ for compile in [
+ item_filter_index::compile_soup_flat_v1,
+ item_filter_index::compile_soup_flat_v2,
+ item_filter_index::compile_soup_flat_v3,
+ item_filter_index::compile_soup_flat_v4,
+ ] {
+ for expression in expressions() {
+ let ast = materialize_graphql_filter(filters(expression.clone())).unwrap();
+ let item_filter_index::LocalCompileOutcome::Supported(query) = compile(
+ &ast,
+ SoupFlatRequest {
+ sort: SoupIndexSort::UpdatedAt,
+ direction: SortDirection::Desc,
+ limit: 20,
+ has_cursor: false,
+ },
+ )
+ .unwrap() else {
+ panic!("file-type filters must be supported");
+ };
+ let predicate = &query.as_query().partitions[0].predicate;
+ for (index, mutation) in documents.iter().enumerate() {
+ let ProjectionMutation::Replace(document) = mutation else {
+ panic!("complete document");
+ };
+ assert_eq!(
+ document.matches(predicate),
+ sql_reference(&expression, index) == SqlTruth::True,
+ "{expression}: {:?}",
+ FILE_TYPES[index]
+ );
+ }
+ }
+ }
+}
+
+#[test]
+fn raw_file_type_and_sql_null_filtering_in_memory() {
+ pollster::block_on(lifecycle(InMemoryStorage::new()));
+}
+#[test]
+fn raw_file_type_and_sql_null_filtering_in_turso() {
+ pollster::block_on(async {
+ lifecycle(cache_turso::TursoStorage::open_in_memory("raw-file-types").unwrap()).await
+ });
+}
+#[test]
+fn malformed_file_type_data_is_still_rejected_but_unknown_filter_inputs_are_not_added() {
+ for file_type in [
+ json!(false),
+ json!(12),
+ json!([]),
+ json!({}),
+ json!("x".repeat(predicate_index::MAX_EXACT_VALUE_BYTES + 1)),
+ ] {
+ let mut item = item(0);
+ item["fileType"] = file_type;
+ assert!(
+ authoritative_projection_mutations(
+ SUPPLEMENT_BACKFILL,
+ Some("SoupBackfill"),
+ &json!({"user":{"soup":{"items":[item]}}})
+ )
+ .is_err()
+ );
+ }
+ let mut missing = item(0);
+ missing.as_object_mut().unwrap().remove("fileType");
+ assert!(
+ authoritative_projection_mutations(
+ SUPPLEMENT_BACKFILL,
+ Some("SoupBackfill"),
+ &json!({"user":{"soup":{"items":[missing]}}})
+ )
+ .is_err()
+ );
+ assert!(
+ compile_current_filter_request(
+ filters(json!({"literal":{"fileType":"doc"}})),
+ "UPDATED_AT",
+ "DESC",
+ 20
+ )
+ .is_err()
+ );
+ // Query inputs still normalize known spellings; stored values do not.
+ assert_eq!(
+ query(json!({"literal":{"fileType":".PDF"}})),
+ query(json!({"literal":{"fileType":"pdf"}}))
+ );
+}
diff --git a/crates/soup_filter_projection/Cargo.toml b/crates/soup_filter_projection/Cargo.toml
index 68cadebb77..35fad43629 100644
--- a/crates/soup_filter_projection/Cargo.toml
+++ b/crates/soup_filter_projection/Cargo.toml
@@ -14,7 +14,6 @@ base64 = { workspace = true }
chrono = { workspace = true }
document_sub_type = { path = "../document_sub_type", default-features = false }
item-filter-index = { path = "../item_filter_index" }
-model_file_type = { path = "../model_file_type" }
models_soup = { path = "../models_soup", optional = true }
postcard = { workspace = true }
predicate-index = { path = "../predicate_index" }
diff --git a/crates/soup_filter_projection/src/lib.rs b/crates/soup_filter_projection/src/lib.rs
index 89aea56b99..c128daa688 100644
--- a/crates/soup_filter_projection/src/lib.rs
+++ b/crates/soup_filter_projection/src/lib.rs
@@ -1,11 +1,8 @@
//! Direct-field Soup projection helpers and typed server-fact supplements.
#![deny(missing_docs)]
-use std::str::FromStr;
-
pub use document_sub_type::DocumentSubType;
use item_filter_index::vocabulary;
-use model_file_type::FileType;
#[cfg(feature = "models")]
use models_soup::{chat::SoupChat, document::SoupDocument, item::SoupItem, project::SoupProject};
use predicate_index::{
@@ -36,15 +33,12 @@ pub use wire::{
/// Maximum authoritative task Status options accepted in one complete projection.
pub const MAX_TASK_STATUS_OPTION_IDS: usize = 64;
-#[cfg(test)]
+#[cfg(all(test, feature = "models"))]
mod test;
/// Failure to project an authoritative Soup item.
#[derive(Debug, Error, PartialEq, Eq)]
pub enum ProjectionError {
- /// The authoritative document contained an unknown file-type value.
- #[error("invalid authoritative Soup document file type `{0}`")]
- InvalidFileType(String),
/// Document server facts do not match the accompanying item variant.
#[error("document server facts do not match Soup item variant")]
SourceMismatch,
@@ -154,7 +148,8 @@ pub struct DirectProjectionInput {
pub owner: String,
/// Project or parent UUID, when present.
pub project_id: Option<uuid::Uuid>,
- /// Document file type, when present. Ignored for other kinds.
+ /// Raw document file type, when present. Ignored for other kinds.
+ /// Preserve the stored spelling: Soup's SQL compares this text literally.
pub file_type: Option<String>,
/// Creation timestamp.
pub created_at: chrono::DateTime<chrono::Utc>,
@@ -173,7 +168,8 @@ pub struct DirectProjectionPatchInput {
pub owner: Option<String>,
/// Replacement project/parent value. Outer `None` means unchanged.
pub project_id: Option<Option<uuid::Uuid>>,
- /// Replacement document file type. Outer `None` means unchanged.
+ /// Replacement raw document file type. Outer `None` means unchanged;
+ /// `Some(None)` removes the fact for a SQL NULL.
pub file_type: Option<Option<String>>,
/// Replacement creation timestamp when supplied.
pub created_at: Option<chrono::DateTime<chrono::Utc>>,
@@ -192,10 +188,7 @@ pub fn project_direct_fields(
if input.kind == SoupFlatEntityKind::Document
&& let Some(file_type) = input.file_type
{
- let canonical = FileType::from_str(&file_type)
- .map_err(|_| ProjectionError::InvalidFileType(file_type))?
- .to_string();
- exact_facts.push(utf8_fact(vocabulary::file_type(), canonical)?);
+ exact_facts.push(utf8_fact(vocabulary::file_type(), file_type)?);
}
projection(
input.record_key,
@@ -308,12 +301,7 @@ pub fn patch_direct_fields(
&& let Some(file_type) = input.file_type
{
let values = file_type
- .map(|file_type| -> Result<ExactValue, ProjectionError> {
- let canonical = FileType::from_str(&file_type)
- .map_err(|_| ProjectionError::InvalidFileType(file_type))?
- .to_string();
- Ok(ExactValue::utf8(canonical)?)
- })
+ .map(ExactValue::utf8)
.transpose()?
.into_iter()
.collect();
diff --git a/crates/soup_filter_projection/src/profile.rs b/crates/soup_filter_projection/src/profile.rs
index f45d7166f3..1416e5f63b 100644
--- a/crates/soup_filter_projection/src/profile.rs
+++ b/crates/soup_filter_projection/src/profile.rs
@@ -1,10 +1,12 @@
-use std::str::FromStr;
+use std::collections::HashMap;
use item_filter_index::vocabulary;
-use model_file_type::FileType;
use predicate_index::{ExactFact, IndexDocument, IntegerFact, Token, ValidationError};
use thiserror::Error;
+#[cfg(test)]
+mod test;
+
/// Semantic validation failure for one complete versioned Soup document.
#[derive(Debug, Error, PartialEq, Eq)]
pub enum ProfileValidationError {
@@ -125,6 +127,7 @@ fn validate_exact_facts(
let mut channel_participant = 0;
let mut channel_team = 0;
let mut channel_organization = 0;
+ let mut notification_states = HashMap::new();
for fact in facts {
let attribute = &fact.attribute;
@@ -146,11 +149,9 @@ fn validate_exact_facts(
}
} else if attribute == &vocabulary::file_type() && kind == PartitionKind::Document {
file_type += 1;
- let value = std::str::from_utf8(value)
+ // The GraphQL field is nullable text, not the supported-format enum.
+ std::str::from_utf8(value)
.map_err(|_| ProfileValidationError::InvalidValue("file-type"))?;
- if FileType::from_str(value).is_err() {
- return Err(ProfileValidationError::InvalidValue("file-type"));
- }
} else if attribute == &vocabulary::document_sub_type() && kind == PartitionKind::Document {
sub_type += 1;
if !matches!(value, b"task" | b"snippet" | b"skill") {
@@ -206,12 +207,10 @@ fn validate_exact_facts(
if value.len() != 16 {
return Err(ProfileValidationError::InvalidValue("notification-id"));
}
- if facts.iter().any(|other| {
- other.value == fact.value
- && other.attribute != *attribute
- && (other.attribute == vocabulary::notification_seen()
- || other.attribute == vocabulary::notification_unseen())
- }) {
+ if notification_states
+ .insert(&fact.value, attribute)
+ .is_some_and(|previous| previous != attribute)
+ {
return Err(ProfileValidationError::InvalidValue("notification-state"));
}
} else {
diff --git a/crates/soup_filter_projection/src/profile/test.rs b/crates/soup_filter_projection/src/profile/test.rs
new file mode 100644
index 0000000000..082c2153f1
--- /dev/null
+++ b/crates/soup_filter_projection/src/profile/test.rs
@@ -0,0 +1,163 @@
+use super::*;
+use crate::{
+ DirectProjectionInput, DirectProjectionPatchInput, SoupCacheProjectionSupplement,
+ SoupFlatEntityKind, compose_soup_flat_v3, patch_direct_fields, project_direct_fields,
+};
+use predicate_index::{ExactValue, RecordKey};
+
+fn notification(attribute: Token, id: u128) -> ExactFact {
+ ExactFact {
+ attribute,
+ value: ExactValue::new(uuid::Uuid::from_u128(id).as_bytes()).unwrap(),
+ }
+}
+
+fn project() -> IndexDocument {
+ let timestamp = chrono::DateTime::from_timestamp(0, 0).unwrap();
+ let mut document = compose_soup_flat_v3(
+ DirectProjectionInput {
+ record_key: RecordKey::new("GraphqlSoupProject:00000000-0000-0000-0000-000000000001")
+ .unwrap(),
+ kind: SoupFlatEntityKind::Project,
+ id: uuid::Uuid::from_u128(1),
+ owner: "macro|[email protected]".into(),
+ project_id: None,
+ file_type: None,
+ created_at: timestamp,
+ updated_at: timestamp,
+ },
+ None,
+ None,
+ )
+ .unwrap();
+ document.profile = vocabulary::profile_v4();
+ document
+}
+
+#[test]
+fn large_notification_sets_validate_but_conflicting_states_do_not() {
+ let mut document = project();
+ document
+ .exact_facts
+ .extend((1..=10_000).map(|id| notification(vocabulary::notification_unseen(), id)));
+ document
+ .exact_facts
+ .push(notification(vocabulary::notification_seen(), 10_001));
+ // Duplicate same-state membership is safe and canonicalization removes it.
+ document
+ .exact_facts
+ .push(notification(vocabulary::notification_unseen(), 1));
+ validate_soup_flat_v4(&document).unwrap();
+ document
+ .exact_facts
+ .push(notification(vocabulary::notification_seen(), 1));
+ assert_eq!(
+ validate_soup_flat_v4(&document),
+ Err(ProfileValidationError::InvalidValue("notification-state")),
+ );
+ // Conflict detection must not depend on which state is visited first.
+ document.exact_facts.reverse();
+ assert_eq!(
+ validate_soup_flat_v4(&document),
+ Err(ProfileValidationError::InvalidValue("notification-state")),
+ );
+}
+
+fn document_input(file_type: Option<&str>) -> DirectProjectionInput {
+ DirectProjectionInput {
+ kind: SoupFlatEntityKind::Document,
+ record_key: RecordKey::new("GraphqlSoupDocument:00000000-0000-0000-0000-000000000001")
+ .unwrap(),
+ id: uuid::Uuid::from_u128(1),
+ owner: "macro|[email protected]".into(),
+ project_id: None,
+ file_type: file_type.map(str::to_owned),
+ created_at: chrono::DateTime::from_timestamp(0, 0).unwrap(),
+ updated_at: chrono::DateTime::from_timestamp(0, 0).unwrap(),
+ }
+}
+
+#[test]
+fn raw_file_types_survive_snapshots_and_patches() {
+ for file_type in [
+ None,
+ Some(""),
+ Some("doc"),
+ Some("custom.ext"),
+ Some("pdf"),
+ Some("PDF"),
+ Some(".pdf"),
+ Some("日本語"),
+ ] {
+ let input = document_input(file_type);
+ let supplement =
+ SoupCacheProjectionSupplement::document(input.record_key.clone(), false, false, vec![]);
+ let mut document = compose_soup_flat_v3(input, None, Some(&supplement)).unwrap();
+ let values = document
+ .exact_facts
+ .iter()
+ .filter(|fact| fact.attribute == vocabulary::file_type())
+ .map(|fact| fact.value.clone())
+ .collect::<Vec<_>>();
+ let expected = file_type
+ .map(|value| ExactValue::utf8(value).unwrap())
+ .into_iter()
+ .collect::<Vec<_>>();
+ assert_eq!(values, expected);
+ document.profile = vocabulary::profile_v4();
+ validate_soup_flat_v4(&document).unwrap();
+ let patch = patch_direct_fields(DirectProjectionPatchInput {
+ record_key: document.record_key,
+ kind: SoupFlatEntityKind::Document,
+ owner: None,
+ project_id: None,
+ file_type: Some(file_type.map(str::to_owned)),
+ created_at: None,
+ updated_at: None,
+ })
+ .unwrap();
+ let predicate_index::OptimisticProjectionMutation::Patch { exact, .. } = patch else {
+ panic!("direct fields produce a patch");
+ };
+ assert_eq!(exact.len(), 1);
+ assert_eq!(exact[0].attribute, vocabulary::file_type());
+ assert_eq!(exact[0].values, expected);
+ }
+}
+
+#[test]
+fn file_type_size_utf8_and_cardinality_are_still_validated() {
+ let oversized = "a".repeat(predicate_index::MAX_EXACT_VALUE_BYTES + 1);
+ assert!(project_direct_fields(document_input(Some(&oversized))).is_err());
+ let input = document_input(Some("doc"));
+ let supplement =
+ SoupCacheProjectionSupplement::document(input.record_key.clone(), false, false, vec![]);
+ let mut document = compose_soup_flat_v3(input, None, Some(&supplement)).unwrap();
+ document.profile = vocabulary::profile_v4();
+ document.exact_facts.push(ExactFact {
+ attribute: vocabulary::file_type(),
+ value: ExactValue::new([0xff]).unwrap(),
+ });
+ assert_eq!(
+ validate_soup_flat_v4(&document),
+ Err(ProfileValidationError::InvalidValue("file-type"))
+ );
+ document.exact_facts.last_mut().unwrap().value = ExactValue::utf8("other").unwrap();
+ assert_eq!(
+ validate_soup_flat_v4(&document),
+ Err(ProfileValidationError::Duplicate("file-type"))
+ );
+}
+
+#[test]
+fn notification_value_shape_is_still_validated() {
+ let mut document = project();
+ document.exact_facts.push(ExactFact {
+ attribute: vocabulary::notification_seen(),
+ value: ExactValue::utf8("not-a-uuid").unwrap(),
+ });
+ assert_eq!(
+ validate_soup_flat_v4(&document),
+ Err(ProfileValidationError::InvalidValue("notification-id")),
+ );
+}
diff --git a/crates/soup_filter_projection/src/test.rs b/crates/soup_filter_projection/src/test.rs
index ca02f87e6b..efe52f70ea 100644
--- a/crates/soup_filter_projection/src/test.rs
+++ b/crates/soup_filter_projection/src/test.rs
@@ -106,7 +106,7 @@ fn optimistic_direct_projection_and_patch_share_authoritative_vocabulary() {
})
.unwrap();
assert!(projection.exact_facts.iter().any(|fact| {
- fact.attribute == vocabulary::file_type() && fact.value == ExactValue::utf8("md").unwrap()
+ fact.attribute == vocabulary::file_type() && fact.value == ExactValue::utf8(".MD").unwrap()
}));
let patch = patch_direct_fields(DirectProjectionPatchInput {