Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
Seanaye/fix/invalid projection ovrflow (#6548)
Sean Ayeb69a06064aa9
diff --git a/Cargo.lock b/Cargo.lockindex 2f5c39d3d0..d75cb78f33 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -3058,7 +3058,7 @@ dependencies = [[[package]]name = "cache-wasm"-version = "0.6.14"+version = "0.6.16"dependencies = ["async-lock","cache-core",@@ -15810,7 +15810,6 @@ dependencies = ["document_sub_type","item-filter-index","macro_user_id",- "model_file_type","models_soup","postcard","predicate-index",diff --git a/apps/web/src/lib/queries/soup/backfill.test.tsx b/apps/web/src/lib/queries/soup/backfill.test.tsxindex c1583b4d78..5e1cdf010c 100644--- a/apps/web/src/lib/queries/soup/backfill.test.tsx+++ b/apps/web/src/lib/queries/soup/backfill.test.tsx@@ -407,9 +407,9 @@ describe('runSoupBackfills', () => {rendered.unmount();});- it('does not reuse pre-projection native backfill checkpoints', () => {+ it.each([13, 14])('does not reuse v%i backfill checkpoints', (version) => {localStorage.setItem(- 'graphql-soup-backfill:v13:user-1:email-filter-metadata',+ `graphql-soup-backfill:v${version}:user-1:email-filter-metadata`,JSON.stringify({userId: 'user-1',nextCursor: 'old-page',@@ -431,7 +431,7 @@ describe('runSoupBackfills', () => {it('restarts from the beginning when the cache generation is replaced', async () => {localStorage.setItem(- 'graphql-soup-backfill:v14:user-1:core-entities',+ 'graphql-soup-backfill:v15:user-1:core-entities',JSON.stringify({userId: 'user-1',nextCursor: 'stale-cursor',diff --git a/apps/web/src/lib/queries/soup/backfill.ts b/apps/web/src/lib/queries/soup/backfill.tsindex 537badeb5c..56013b0969 100644--- a/apps/web/src/lib/queries/soup/backfill.ts+++ b/apps/web/src/lib/queries/soup/backfill.ts@@ -26,9 +26,9 @@ import { createEffect, createSignal, onCleanup } from 'solid-js';// Bump when a default backfill input or completion guarantee changes so// persisted cursors cannot retain an older hydration contract.-// Rehydrate complete property snapshots for the property-aware Soup/Mail-// profiles. Preserve queued work; old or partial scans cannot supply new proof.-const BACKFILL_VERSION = 14;+// Rehydrate raw file-type projections after retiring enum-normalized facts.+// Old cursors must not skip records when the cache compatibility epoch changes.+const BACKFILL_VERSION = 15;const PAGE_LIMIT = 100;// Five threads × twenty messages reaches the backend's 100-message cap.const EMAIL_CONTENT_PAGE_LIMIT = 5;diff --git a/crates/client/cache-core/src/codec.rs b/crates/client/cache-core/src/codec.rsindex fb06dca717..b1eeb58079 100644--- a/crates/client/cache-core/src/codec.rs+++ b/crates/client/cache-core/src/codec.rs@@ -22,7 +22,8 @@ pub const CACHE_FORMAT_VERSION: u32 = 3;/// fields, so older records remain usable until a newly selected field is/// fetched. Bump this epoch for incompatible changes to normalized identity,/// field storage shape, or other schema-derived cache semantics.-pub const CACHE_SCHEMA_COMPATIBILITY_EPOCH: u32 = 2;+// File-type facts now preserve raw server text instead of enum-normalizing it.+pub const CACHE_SCHEMA_COMPATIBILITY_EPOCH: u32 = 3;#[derive(Debug, Error)]pub enum CodecError {@@ -141,7 +142,7 @@ mod tests {fn namespace_uses_schema_compatibility_epoch_not_schema_hash() {assert_eq!(cache_namespace("client-token-1"),- "graphql-cache:client-token-1:s2:v3"+ "graphql-cache:client-token-1:s3:v3");}}diff --git a/crates/client/cache-core/src/predicate.rs b/crates/client/cache-core/src/predicate.rsindex d13d368368..ea44bd13cb 100644--- a/crates/client/cache-core/src/predicate.rs+++ b/crates/client/cache-core/src/predicate.rs@@ -19,7 +19,7 @@ use predicate_index::{ValidatedIndexQuery, ValidationError,};use serde::{Deserialize, Serialize};-use std::collections::{BTreeSet, HashMap};+use std::collections::{BTreeSet, HashMap, HashSet};use thiserror::Error;/// Persisted projection state for one supported normalized record.@@ -570,6 +570,7 @@ fn patch_exact_members(remove: &[ExactFact],insert: &[ExactFact],) -> Result<(), ValidationError> {+ let remove = remove.iter().collect::<HashSet<_>>();document.exact_facts.retain(|fact| !remove.contains(fact));document.exact_facts.extend_from_slice(insert);document.canonicalize();diff --git a/crates/client/cache-core/src/predicate/test.rs b/crates/client/cache-core/src/predicate/test.rsindex d757a879af..d445c2f744 100644--- a/crates/client/cache-core/src/predicate/test.rs+++ b/crates/client/cache-core/src/predicate/test.rs@@ -71,10 +71,53 @@ fn member_edits_are_idempotent_and_preserve_unknown_coverage() {}#[test]-fn member_growth_beyond_budget_is_incomplete_not_an_enqueue_failure() {+fn large_member_edits_preserve_sets_and_are_idempotent() {let mut doc = document();- doc.exact_facts = (0..predicate_index::MAX_FACTS_PER_DOCUMENT)- .map(|i| fact(&i.to_string()))+ doc.exact_facts+ .extend((0..10_000).map(|i| fact(&i.to_string())));+ let removed = doc.exact_facts.clone();+ let mutation = OptimisticProjectionMutation::PatchExact {+ record_key: doc.record_key.clone(),+ profile: doc.profile.clone(),+ partition: doc.partition.clone(),+ remove: removed.clone(),+ // Overlapping removal/insertion keeps the inserted member.+ insert: vec![fact("a"), fact("new"), fact("new")],+ };+ let projection = compose_effective_optimistic_projection(+ &doc.record_key,+ Some(&ProjectionState::Complete(doc.clone())),+ &[ProjectionMutationLayer {+ owner: 1,+ mutations: &[mutation.clone(), mutation],+ }],+ )+ .unwrap()+ .unwrap();+ let OptimisticProjectionState::Complete(effective) = projection.state else {+ panic!("large member sets remain complete");+ };+ assert_eq!(effective.exact_facts, vec![fact("a"), fact("new")]);++ let state = apply_authoritative_exact_members(+ Some(&ProjectionState::Complete(doc.clone())),+ &doc.record_key,+ &doc.profile,+ &doc.partition,+ &removed,+ &[fact("a"), fact("new")],+ );+ assert_eq!(state, ProjectionState::Complete(effective));+}++#[test]+fn attribute_growth_beyond_budget_is_incomplete_not_an_enqueue_failure() {+ let mut doc = document();+ doc.exact_facts = (0..predicate_index::MAX_ATTRIBUTES_PER_DOCUMENT)+ .map(|i| ExactFact {+ attribute: token(&format!("attribute-{i}")),+ value: ExactValue::utf8("value").unwrap(),+ }).collect();let mutation = OptimisticProjectionMutation::PatchExact {record_key: doc.record_key.clone(),diff --git a/crates/client/cache-wasm/Cargo.toml b/crates/client/cache-wasm/Cargo.tomlindex 9343cd74fc..29bcba1214 100644--- a/crates/client/cache-wasm/Cargo.toml+++ b/crates/client/cache-wasm/Cargo.toml@@ -2,8 +2,8 @@edition = "2024"name = "cache-wasm"publish = false-# Rebuild for canonical Mail tab previews and viewer-scoped shared eligibility.-version = "0.6.14"+# Rebuild for raw file-type projections and SQL-compatible nullable filters.+version = "0.6.16"[features]default = []diff --git a/crates/item_filter_index/src/document.rs b/crates/item_filter_index/src/document.rsnew file mode 100644index 0000000000..a626a711c1--- /dev/null+++ b/crates/item_filter_index/src/document.rs@@ -0,0 +1,53 @@+//! Document predicates preserve SQL NULL behavior for the raw file-type column.++use super::{CompileError, DocumentLiteral, Expr, PredicateExpr, vocabulary};++pub(super) fn compile(+ expr: Option<&Expr<DocumentLiteral>>,+ literal: impl Fn(&DocumentLiteral) -> Result<PredicateExpr, CompileError> + Copy,+) -> Result<PredicateExpr, CompileError> {+ match expr {+ Some(expr) => compile_inner(expr, false, literal),+ None => Ok(PredicateExpr::All),+ }+}++fn compile_inner(+ expr: &Expr<DocumentLiteral>,+ negated: bool,+ literal: impl Fn(&DocumentLiteral) -> Result<PredicateExpr, CompileError> + Copy,+) -> Result<PredicateExpr, CompileError> {+ Ok(match expr {+ Expr::Literal(value) => {+ let predicate = literal(value)?;+ if !negated {+ predicate+ } else if matches!(value, DocumentLiteral::FileType(_)) {+ // SQL NULL makes both fileType = x and NOT(fileType = x)+ // UNKNOWN, so neither matches a WHERE clause. An absent fact+ // already fails equality; negated equality also needs presence.+ PredicateExpr::And(+ Box::new(PredicateExpr::ExactExists {+ attribute: vocabulary::file_type(),+ }),+ Box::new(PredicateExpr::Not(Box::new(predicate))),+ )+ } else {+ PredicateExpr::Not(Box::new(predicate))+ }+ }+ Expr::Not(inner) => compile_inner(inner, !negated, literal)?,+ Expr::And(left, right) | Expr::Or(left, right) => {+ let left = Box::new(compile_inner(left, negated, literal)?);+ let right = Box::new(compile_inner(right, negated, literal)?);+ // Push NOT to the leaves using De Morgan's laws. Guarding the whole+ // tree would incorrectly reject NULL rows matched by another OR arm;+ // complementing a compiled subtree would incorrectly include NULLs.+ if matches!(expr, Expr::And(..)) != negated {+ PredicateExpr::And(left, right)+ } else {+ PredicateExpr::Or(left, right)+ }+ }+ })+}diff --git a/crates/item_filter_index/src/lib.rs b/crates/item_filter_index/src/lib.rsindex 8137da5f9b..657d346d77 100644--- a/crates/item_filter_index/src/lib.rs+++ b/crates/item_filter_index/src/lib.rs@@ -29,6 +29,7 @@ use uuid::Uuid;mod test;mod channels;+mod document;pub mod mail;pub mod properties;@@ -148,7 +149,7 @@ pub mod vocabulary {token("owner")}- /// Document file-type attribute.+ /// Raw stored document file-type text; absent for SQL NULL.pub fn file_type() -> Token {token("file-type")}@@ -529,7 +530,7 @@ fn compile_soup_flat(SoupIndexSort::Unsupported => unreachable!("eligibility checked sort"),};let mut document_predicate =- compile_expr(ast.document_filter.as_deref(), compile_document_literal)?;+ document::compile(ast.document_filter.as_deref(), compile_document_literal)?;if let Some(properties_filter) = ast.properties_filter.as_deref() {let compile_properties_literal =compile_properties_literal.expect("eligibility checked property support");diff --git a/crates/predicate_index/src/lib.rs b/crates/predicate_index/src/lib.rsindex b6e37d3767..8687a37a36 100644--- a/crates/predicate_index/src/lib.rs+++ b/crates/predicate_index/src/lib.rs@@ -23,8 +23,9 @@ pub const MAX_QUERY_LIMIT: u16 = 500;pub const MAX_TOKEN_BYTES: usize = 128;/// Maximum bytes in an exact value.pub const MAX_EXACT_VALUE_BYTES: usize = 16 * 1_024;-/// Maximum facts in one index document.-pub const MAX_FACTS_PER_DOCUMENT: usize = 256;+/// Maximum distinct attribute names in one index document or patch.+/// Set-valued attributes may contain any number of individually bounded values.+pub const MAX_ATTRIBUTES_PER_DOCUMENT: usize = 256;/// Maximum distinct records whose optimistic projections may be merged into one query.pub const MAX_OPTIMISTIC_RECORDS_PER_QUERY: usize = 128;@@ -227,7 +228,7 @@ impl PredicateExpr {}/// One exact fact in an index document.-#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]pub struct ExactFact {/// Fact vocabulary token.pub attribute: Token,@@ -261,6 +262,19 @@ pub struct IndexDocument {pub sort_facts: Vec<IntegerFact>,}+fn validate_attribute_count<'a>(+ attributes: impl IntoIterator<Item = &'a Token>,+) -> Result<(), ValidationError> {+ let mut distinct = HashSet::new();+ for attribute in attributes {+ distinct.insert(attribute);+ if distinct.len() > MAX_ATTRIBUTES_PER_DOCUMENT {+ return Err(ValidationError::DocumentAttributes);+ }+ }+ Ok(())+}+impl IndexDocument {/// Put facts in deterministic order and remove duplicate membership facts.pub fn canonicalize(&mut self) {@@ -271,12 +285,15 @@ impl IndexDocument {self.sort_facts.sort();}- /// Validate bounded fact counts and unique sort attributes.+ /// Validate bounded attribute counts and unique sort attributes.pub fn validate(&self) -> Result<(), ValidationError> {- let facts = self.exact_facts.len() + self.integer_facts.len() + self.sort_facts.len();- if facts > MAX_FACTS_PER_DOCUMENT {- return Err(ValidationError::DocumentFacts);- }+ validate_attribute_count(+ self.exact_facts+ .iter()+ .map(|fact| &fact.attribute)+ .chain(self.integer_facts.iter().map(|fact| &fact.attribute))+ .chain(self.sort_facts.iter().map(|fact| &fact.attribute)),+ )?;let mut sort_attributes = HashSet::new();if self.sort_facts@@ -730,17 +747,20 @@ impl OptimisticProjectionMutation {return match self {Self::Replace(document) => document.validate(),Self::Delete { .. } | Self::Unknown { .. } => Ok(()),- Self::PatchExact { remove, insert, .. } => {- if remove.len() + insert.len() > MAX_FACTS_PER_DOCUMENT {- Err(ValidationError::DocumentFacts)- } else {- Ok(())- }- }+ Self::PatchExact { remove, insert, .. } => validate_attribute_count(+ remove.iter().chain(insert).map(|fact| &fact.attribute),+ ),Self::Patch { .. } => unreachable!(),};};+ validate_attribute_count(+ exact+ .iter()+ .map(|patch| &patch.attribute)+ .chain(integers.iter().map(|patch| &patch.attribute))+ .chain(sorts.iter().map(|fact| &fact.attribute)),+ )?;let mut attributes = HashSet::new();if exact.iter()@@ -835,9 +855,9 @@ pub enum ValidationError {/// An integer range is inverted or empty.#[error("invalid integer range")]InvalidRange,- /// A document contains too many facts.- #[error("index document contains too many facts")]- DocumentFacts,+ /// A document or patch contains too many distinct attribute names.+ #[error("index document contains too many distinct attributes")]+ DocumentAttributes,/// A document has more than one sort value for an attribute.#[error("duplicate sort fact attribute")]DuplicateSortFact,diff --git a/crates/predicate_index/src/test.rs b/crates/predicate_index/src/test.rsindex a8f4b14ab9..d091f0bc27 100644--- a/crates/predicate_index/src/test.rs+++ b/crates/predicate_index/src/test.rs@@ -238,6 +238,94 @@ fn rejects_empty_ranges_duplicate_partitions_and_duplicate_sort_facts() {assert_eq!(document.validate(), Err(ValidationError::DuplicateSortFact));}+#[test]+fn set_cardinality_does_not_consume_the_attribute_budget() {+ let mut doc = document("Document:1", "user-1", 10);+ doc.exact_facts.extend((0..10_000).map(|i| ExactFact {+ attribute: token("members"),+ value: exact(&i.to_string()),+ }));+ doc.integer_facts+ .extend((0..10_000).map(|value| IntegerFact {+ attribute: token("numbers"),+ value,+ }));+ doc.validate().unwrap();++ let patch = OptimisticProjectionMutation::PatchExact {+ record_key: doc.record_key.clone(),+ profile: doc.profile.clone(),+ partition: doc.partition.clone(),+ remove: doc.exact_facts.clone(),+ insert: doc.exact_facts.clone(),+ };+ patch.validate().unwrap();+ assert!(matches!(+ ExactValue::new(vec![0; MAX_EXACT_VALUE_BYTES + 1]),+ Err(ValidationError::ExactValueTooLarge(_))+ ));+}++#[test]+fn document_and_patch_attribute_budgets_include_every_fact_family() {+ let mut doc = document("Document:1", "user-1", 10);+ doc.exact_facts = (0..MAX_ATTRIBUTES_PER_DOCUMENT - 1)+ .map(|i| ExactFact {+ attribute: token(&format!("attribute-{i}")),+ value: exact("value"),+ })+ .collect();+ // updated-at is shared by integer and sort facts: one attribute name.+ doc.validate().unwrap();+ doc.integer_facts.push(IntegerFact {+ attribute: token("extra"),+ value: 1,+ });+ assert_eq!(doc.validate(), Err(ValidationError::DocumentAttributes));++ let patch = OptimisticProjectionMutation::Patch {+ record_key: doc.record_key.clone(),+ profile: doc.profile.clone(),+ partition: doc.partition.clone(),+ exact: doc+ .exact_facts+ .iter()+ .map(|fact| ExactAttributePatch {+ attribute: fact.attribute.clone(),+ values: vec![fact.value.clone()],+ })+ .collect(),+ integers: doc+ .integer_facts+ .iter()+ .map(|fact| IntegerAttributePatch {+ attribute: fact.attribute.clone(),+ values: vec![fact.value],+ })+ .collect(),+ sorts: doc.sort_facts.clone(),+ };+ assert_eq!(patch.validate(), Err(ValidationError::DocumentAttributes));++ let patch = OptimisticProjectionMutation::PatchExact {+ record_key: doc.record_key,+ profile: doc.profile,+ partition: doc.partition,+ remove: doc.exact_facts,+ insert: vec![+ ExactFact {+ attribute: token("updated-at"),+ value: exact("one"),+ },+ ExactFact {+ attribute: token("extra"),+ value: exact("two"),+ },+ ],+ };+ assert_eq!(patch.validate(), Err(ValidationError::DocumentAttributes));+}+#[test]fn uncertainty_can_clear_exact_fields_after_a_wildcard() {let owner = token("owner");diff --git a/crates/soup_filter_cache_adapter/src/notifications.rs b/crates/soup_filter_cache_adapter/src/notifications.rsindex a70a7a1f91..7916c17931 100644--- a/crates/soup_filter_cache_adapter/src/notifications.rs+++ b/crates/soup_filter_cache_adapter/src/notifications.rs@@ -9,6 +9,8 @@ use cache_core::{};use predicate_index::ExactFact;+mod batch;+#[cfg(all(test, not(target_arch = "wasm32")))]mod test;@@ -237,6 +239,7 @@ async fn retain_complete_parents<S: Storage>(storage: &S,mutations: Vec<ProjectionMutation>,) -> Result<Vec<ProjectionMutation>, SoupFilterCacheAdapterError> {+ let mutations = batch::batch_member_changes(mutations);if mutations.is_empty() {return Ok(mutations);}diff --git a/crates/soup_filter_cache_adapter/src/notifications/batch.rs b/crates/soup_filter_cache_adapter/src/notifications/batch.rsnew file mode 100644index 0000000000..41304f848a--- /dev/null+++ b/crates/soup_filter_cache_adapter/src/notifications/batch.rs@@ -0,0 +1,97 @@+//! Coalesce notification member edits before composing a parent's projection.++use cache_core::predicate::{ProjectionIncompleteKind, ProjectionMutation};+use indexmap::IndexMap;+use predicate_index::{ExactFact, Profile, RecordKey, Token};+use std::collections::BTreeSet;++struct Batch {+ profile: Profile,+ partition: Token,+ state: BatchState,+}++enum BatchState {+ Members {+ remove: BTreeSet<ExactFact>,+ insert: BTreeSet<ExactFact>,+ },+ Incomplete(ProjectionIncompleteKind),+}++/// Notification updates contain only member edits or invalidation. Combining+/// them per parent avoids copying and sorting a large projection for every child.+/// This does not combine separate optimistic queue layers or reorder snapshots.+pub(super) fn batch_member_changes(mutations: Vec<ProjectionMutation>) -> Vec<ProjectionMutation> {+ let mut batches = IndexMap::<RecordKey, Batch>::new();+ for mutation in mutations {+ match mutation {+ ProjectionMutation::PatchExact {+ record_key,+ profile,+ partition,+ remove,+ insert,+ } => {+ let batch = batches.entry(record_key).or_insert_with(|| Batch {+ profile,+ partition,+ state: BatchState::Members {+ remove: BTreeSet::new(),+ insert: BTreeSet::new(),+ },+ });+ if let BatchState::Members {+ remove: removed,+ insert: inserted,+ } = &mut batch.state+ {+ // Later removals override earlier inserts. Later inserts win+ // over removals, including a move between unseen and seen.+ for fact in remove {+ inserted.remove(&fact);+ removed.insert(fact);+ }+ inserted.extend(insert);+ }+ }+ ProjectionMutation::MarkIncomplete {+ record_key,+ profile,+ partition,+ kind,+ } => {+ batches.insert(+ record_key,+ Batch {+ profile,+ partition,+ state: BatchState::Incomplete(kind),+ },+ );+ }+ _ => unreachable!("notification updates are member edits or invalidation"),+ }+ }+ batches+ .into_iter()+ .map(|(record_key, batch)| match batch.state {+ BatchState::Members { remove, insert } => ProjectionMutation::PatchExact {+ record_key,+ profile: batch.profile,+ partition: batch.partition,+ remove: remove.into_iter().collect(),+ insert: insert.into_iter().collect(),+ },+ BatchState::Incomplete(kind) => ProjectionMutation::MarkIncomplete {+ record_key,+ profile: batch.profile,+ partition: batch.partition,+ kind,+ },+ })+ .collect()+}++#[cfg(test)]+mod test;diff --git a/crates/soup_filter_cache_adapter/src/notifications/batch/test.rs b/crates/soup_filter_cache_adapter/src/notifications/batch/test.rsnew file mode 100644index 0000000000..7dd135fbb4--- /dev/null+++ b/crates/soup_filter_cache_adapter/src/notifications/batch/test.rs@@ -0,0 +1,85 @@+use super::*;+use crate::notifications::change;+use cache_core::predicate::{ProjectionState, apply_authoritative_projection_mutations};+use item_filter_index::vocabulary;+use predicate_index::IndexDocument;+use std::collections::HashMap;++fn key(id: u128) -> RecordKey {+ RecordKey::new(format!("GraphqlSoupDocument:{}", uuid::Uuid::from_u128(id))).unwrap()+}++fn update(parent: u128, id: u128, state: &str) -> ProjectionMutation {+ change(+ key(parent),+ vocabulary::document_partition(),+ uuid::Uuid::from_u128(id),+ Some(state),+ )+}++#[test]+fn batching_preserves_last_member_state_and_independent_parents() {+ let mutations = vec![+ update(1, 10, "UNSEEN"),+ update(2, 10, "SEEN"),+ update(1, 11, "SEEN"),+ update(1, 10, "SEEN"),+ update(1, 11, "DONE"),+ update(1, 10, "SEEN"),+ update(1, 11, "UNSEEN"),+ ];+ let bases = [1, 2]+ .into_iter()+ .map(|id| {+ let key = key(id);+ (+ key.clone(),+ ProjectionState::Complete(IndexDocument {+ record_key: key,+ profile: vocabulary::profile_v4(),+ partition: vocabulary::document_partition(),+ exact_facts: vec![],+ integer_facts: vec![],+ sort_facts: vec![],+ }),+ )+ })+ .collect::<HashMap<_, _>>();+ let mut expected = bases.clone();+ apply_authoritative_projection_mutations(&mut expected, &mutations);+ let batched = batch_member_changes(mutations);+ assert_eq!(batched.len(), 2);+ let mut actual = bases;+ apply_authoritative_projection_mutations(&mut actual, &batched);+ assert_eq!(actual, expected);+ apply_authoritative_projection_mutations(&mut actual, &batched);+ assert_eq!(actual, expected, "replaying a batch must be idempotent");+}++#[test]+fn member_edits_cannot_clear_invalidation_in_the_same_batch() {+ for mutations in [+ vec![update(1, 10, "INVALID"), update(1, 11, "UNSEEN")],+ vec![update(1, 10, "UNSEEN"), update(1, 11, "INVALID")],+ ] {+ let batched = batch_member_changes(mutations);+ assert!(matches!(+ batched.as_slice(),+ [ProjectionMutation::MarkIncomplete {+ kind: ProjectionIncompleteKind::Dirty,+ ..+ }]+ ));+ }+}++#[test]+fn many_notifications_produce_one_member_edit_per_parent() {+ let batched = batch_member_changes((1..=10_000).map(|id| update(1, id, "SEEN")).collect());+ let [ProjectionMutation::PatchExact { remove, insert, .. }] = batched.as_slice() else {+ panic!("one parent should be composed only once");+ };+ assert_eq!(remove.len(), 20_000);+ assert_eq!(insert.len(), 10_000);+}diff --git a/crates/soup_filter_cache_adapter/src/notifications/test.rs b/crates/soup_filter_cache_adapter/src/notifications/test.rsindex 091f962252..2552479921 100644--- a/crates/soup_filter_cache_adapter/src/notifications/test.rs+++ b/crates/soup_filter_cache_adapter/src/notifications/test.rs@@ -11,6 +11,8 @@ use predicate_index::{use serde_json::{Value, json};use soup_filter_projection::{SoupCacheProjectionSupplement, encode_cache_projection_supplement};+mod large;+const D: &str = "00000000-0000-0000-0000-000000000001";const A: &str = "00000000-0000-0000-0000-000000000011";const B: &str = "00000000-0000-0000-0000-000000000012";@@ -564,7 +566,7 @@ fn optimistic_notification_conversion_rejects_unsupported_mutations() {}#[test]-fn snapshot_completeness_aliases_primary_scope_and_bounds() {+fn snapshot_completeness_aliases_and_primary_scope() {let data = snapshot(vec![notification(A, "UNSEEN"), notification(B, "SEEN")]);let mutations = authoritative_projection_mutations(SNAPSHOT, None, &data).unwrap();let [ProjectionMutation::Replace(document)] = mutations.as_slice() else {@@ -622,15 +624,33 @@ fn snapshot_completeness_aliases_primary_scope_and_bounds() {authoritative_projection_mutations(alias_query.as_str(), None, &alias_data).unwrap(),authoritative_projection_mutations(SNAPSHOT, None, &data).unwrap());- let too_many = (1..=256)+}++#[test]+fn backfill_accepts_large_notification_sets_without_losing_members() {+ let notifications = (1..=10_000).map(|n| notification(&uuid::Uuid::from_u128(n + 1000).to_string(), "UNSEEN")).collect();- assert!(matches!(- authoritative_projection_mutations(SNAPSHOT, None, &snapshot(too_many))- .unwrap()- .as_slice(),- [ProjectionMutation::MarkIncomplete { .. }]- ));+ let backfill = SNAPSHOT.replace("query Snapshot", "query SoupBackfill");+ let mutations = authoritative_projection_mutations(+ &backfill,+ Some("SoupBackfill"),+ &snapshot(notifications),+ )+ .unwrap();+ let [ProjectionMutation::Replace(document)] = mutations.as_slice() else {+ panic!("large notification snapshots must remain complete");+ };+ assert_eq!(+ document+ .exact_facts+ .iter()+ .filter(|fact| { fact.attribute == vocabulary::notification_unseen() })+ .count(),+ 10_000+ );+ assert!(document.matches(&unseen()));+ assert!(!document.matches(&seen()));}#[test]diff --git a/crates/soup_filter_cache_adapter/src/notifications/test/large.rs b/crates/soup_filter_cache_adapter/src/notifications/test/large.rsnew file mode 100644index 0000000000..3916056461--- /dev/null+++ b/crates/soup_filter_cache_adapter/src/notifications/test/large.rs@@ -0,0 +1,148 @@+use super::*;++const MEMBER_COUNT: u128 = 1_024;++fn member_id(index: u128) -> String {+ uuid::Uuid::from_u128(index + 1000).to_string()+}++fn large_snapshot() -> Value {+ let mut notifications = (1..=MEMBER_COUNT)+ .map(|id| notification(&member_id(id), "SEEN"))+ .collect::<Vec<_>>();+ notifications.extend([notification(A, "UNSEEN"), notification(B, "UNSEEN")]);+ snapshot(notifications)+}++async fn large_set_lifecycle<S: PredicateIndexStorage>(storage: S) {+ let mut engine = Engine::new(storage);+ write(&mut engine, SNAPSHOT, &large_snapshot()).await;+ assert!(matches(&mut engine, unseen()).await);+ assert!(matches(&mut engine, seen()).await);++ let data = json!({"updateNotifications": (1..=MEMBER_COUNT).map(|id| json!({+ "__typename": "GraphqlNotification", "id": member_id(id), "state": "DONE"+ })).collect::<Vec<_>>()});+ let vars = variables();+ let edits = notification_projection_updates(engine.storage(), UPDATE, None, &vars, &data)+ .await+ .unwrap();+ let [ProjectionMutation::PatchExact { remove, insert, .. }] = edits.as_slice() else {+ panic!("bulk updates compose the parent once");+ };+ assert_eq!(remove.len(), MEMBER_COUNT as usize * 2);+ assert!(insert.is_empty());+ let (bulk, _) = engine+ .begin_optimistic_write_with_projections(+ None,+ BeginOptimisticWrite {+ uuid: "00000000-0000-0000-0000-000000000101",+ query: UPDATE,+ operation_name: None,+ variables: &vars,+ data: &data,+ link_patches: &[],+ revalidations: &[],+ created_at_ms: 1,+ },+ optimistic_notification_updates(edits).unwrap(),+ )+ .await+ .unwrap();+ let single = mark_done(&mut engine, A, "00000000-0000-0000-0000-000000000102").await;+ assert!(!matches(&mut engine, seen()).await);+ assert!(matches(&mut engine, unseen()).await, "B is still unseen");++ let mut engine = Engine::new(engine.into_storage());+ assert!(+ !matches(&mut engine, seen()).await,+ "pending bulk edit survives reopen"+ );+ let token = claim(&mut engine, bulk).await;+ engine.rollback_optimistic_write(bulk, token).await.unwrap();+ assert!(+ matches(&mut engine, seen()).await,+ "rollback restores all seen members"+ );++ // A remains optimistically done while B changes through realtime.+ write(+ &mut engine,+ UPDATE,+ &json!({"updateNotifications":[notification(B, "SEEN")]}),+ )+ .await;+ assert!(!matches(&mut engine, unseen()).await);+ // A full refresh rebases, rather than replacing, the pending removal of A.+ write(&mut engine, SNAPSHOT, &large_snapshot()).await;+ assert!(matches(&mut engine, unseen()).await);+ write(+ &mut engine,+ UPDATE,+ &json!({"updateNotifications":[notification(B, "SEEN")]}),+ )+ .await;+ assert!(+ !matches(&mut engine, unseen()).await,+ "refresh must not resurrect A"+ );++ let token = claim(&mut engine, single).await;+ let settled = json!({"updateNotifications":[notification(A, "DONE")]});+ let edits = notification_projection_updates(engine.storage(), UPDATE, None, &vars, &settled)+ .await+ .unwrap();+ engine+ .commit_optimistic_write_with_projections(+ single, token, UPDATE, None, &vars, &settled, edits,+ )+ .await+ .unwrap();++ let keys = (1..=MEMBER_COUNT)+ .map(|id| format!("GraphqlNotification:{}", member_id(id)))+ .collect::<Vec<_>>();+ let edits = notification_deletion_updates(engine.storage(), &keys, false)+ .await+ .unwrap();+ assert_eq!(+ edits.len(),+ 1,+ "bulk deletions also compose the parent once"+ );+ let keys = keys+ .into_iter()+ .map(|key| EntityKey(key.into()))+ .collect::<Vec<_>>();+ engine+ .delete_keys_with_projection_changes(&keys, edits)+ .await+ .unwrap();+ assert!(+ matches(&mut engine, seen()).await,+ "B remains after unrelated members are deleted"+ );+ write(+ &mut engine,+ UPDATE,+ &json!({"updateNotifications":[notification(B, "DONE")]}),+ )+ .await;+ assert!(!matches(&mut engine, seen()).await);+ assert!(!matches(&mut engine, unseen()).await);+}++#[test]+fn large_notification_lifecycle_in_memory() {+ pollster::block_on(large_set_lifecycle(InMemoryStorage::new()));+}++#[test]+fn large_notification_lifecycle_in_turso() {+ pollster::block_on(async {+ large_set_lifecycle(+ cache_turso::TursoStorage::open_in_memory("large-notifications").unwrap(),+ )+ .await+ });+}diff --git a/crates/soup_filter_cache_adapter/src/test.rs b/crates/soup_filter_cache_adapter/src/test.rsindex 9d6c6bce23..9bc823889e 100644--- a/crates/soup_filter_cache_adapter/src/test.rs+++ b/crates/soup_filter_cache_adapter/src/test.rs@@ -7,6 +7,8 @@ use std::collections::BTreeMap;use super::*;+mod file_type;+#[test]fn optimistic_soup_payloads_compile_to_durable_projection_layers() {let mutations = optimistic_projection_mutations(diff --git a/crates/soup_filter_cache_adapter/src/test/file_type.rs b/crates/soup_filter_cache_adapter/src/test/file_type.rsnew file mode 100644index 0000000000..91f0a29e36--- /dev/null+++ b/crates/soup_filter_cache_adapter/src/test/file_type.rs@@ -0,0 +1,420 @@+use super::*;+use cache_core::{+ engine::{BeginOptimisticWrite, Engine, NetworkWrite},+ predicate::{PredicateIndexStorage, PredicateQueryResult},+ queue::{MutationClaimRequest, MutationClaimToken},+ store::InMemoryStorage,+};+use serde_json::{Value, json};++const FILE_TYPES: [Option<&str>; 8] = [+ Some("pdf"),+ Some("doc"),+ Some("custom.ext"),+ Some("PDF"),+ Some(".pdf"),+ Some(""),+ None,+ Some("md"),+];+const RENAME: &str = r#"mutation Rename($inputs: [RenameEntityInput!]!) {+ renameEntities(inputs: $inputs) { results { __typename+ ... on GraphqlMutationSuccess { effects { __typename+ ... on SoupUpdated { item { __typename id+ ... on GraphqlSoupDocument { fileType updatedAt }+ } }+ } }+ } }+}"#;++fn id(index: usize) -> String {+ uuid::Uuid::from_u128(index as u128 + 1).to_string()+}+fn key(index: usize) -> RecordKey {+ RecordKey::new(format!("GraphqlSoupDocument:{}", id(index))).unwrap()+}+fn item(index: usize) -> Value {+ let mut item = selected_document(+ &id(index),+ json!(document_supplement(&id(index), false)),+ Value::Null,+ );+ item["fileType"] = json!(FILE_TYPES[index]);+ item["properties"] = json!([]);+ item+}+fn filters(document: Value) -> Value {+ let nil = uuid::Uuid::nil().to_string();+ json!({+ "documentFilter": document,+ "calendarEventFilter": {"literal":{"id":nil}},+ "projectFilter": {"literal":{"projectIdSelf":nil}},+ "chatFilter": {"literal":{"chatId":nil}},+ "emailFilter": {"tree":{"literal":{"threadId":nil}}},+ "channelFilter": {"literal":{"channelId":nil}},+ "channelThreadFilter": {"literal":{"threadId":nil}},+ "callFilter": {"literal":{"callId":nil}},+ "crmCompanyFilter": {"literal":{"id":nil}},+ "foreignEntityFilter": {"literal":{"id":nil}}+ })+}+fn query(document: Value) -> ValidatedIndexQuery {+ let SoupFilterCompileOutcome::Supported(query) =+ compile_current_filter_request(filters(document), "UPDATED_AT", "DESC", 20).unwrap()+ else {+ panic!("file-type filters must be locally supported");+ };+ query+}+fn not(expr: Value) -> Value {+ json!({"not":expr})+}++// Independent SQL three-valued reference: WHERE selects only True, not Unknown.+#[derive(Clone, Copy, Debug, PartialEq, Eq)]+enum SqlTruth {+ True,+ False,+ Unknown,+}+impl SqlTruth {+ fn from_bool(value: bool) -> Self {+ if value { Self::True } else { Self::False }+ }+ fn negate(self) -> Self {+ match self {+ Self::True => Self::False,+ Self::False => Self::True,+ Self::Unknown => Self::Unknown,+ }+ }+ fn and(self, other: Self) -> Self {+ match (self, other) {+ (Self::False, _) | (_, Self::False) => Self::False,+ (Self::True, Self::True) => Self::True,+ _ => Self::Unknown,+ }+ }+ fn or(self, other: Self) -> Self {+ match (self, other) {+ (Self::True, _) | (_, Self::True) => Self::True,+ (Self::False, Self::False) => Self::False,+ _ => Self::Unknown,+ }+ }+}+fn sql_reference(expr: &Value, index: usize) -> SqlTruth {+ if let Some(literal) = expr.get("literal") {+ if let Some(file_type) = literal.get("fileType") {+ return FILE_TYPES[index].map_or(SqlTruth::Unknown, |stored| {+ SqlTruth::from_bool(stored == file_type.as_str().unwrap())+ });+ }+ return SqlTruth::from_bool(literal["id"].as_str().unwrap() == id(index));+ }+ if let Some(expr) = expr.get("not") {+ return sql_reference(expr, index).negate();+ }+ if let Some(and) = expr.get("and") {+ return sql_reference(&and["left"], index).and(sql_reference(&and["right"], index));+ }+ let or = &expr["or"];+ sql_reference(&or["left"], index).or(sql_reference(&or["right"], index))+}+fn expressions() -> Vec<Value> {+ let atoms = [+ json!({"literal":{"fileType":"pdf"}}),+ json!({"literal":{"fileType":"md"}}),+ json!({"literal":{"id":id(6)}}),+ ];+ let mut expressions = Vec::new();+ for atom in &atoms {+ expressions.extend([atom.clone(), not(atom.clone()), not(not(atom.clone()))]);+ }+ for left in &atoms {+ for right in &atoms {+ for op in ["and", "or"] {+ let expr = json!({op:{"left":left,"right":right}});+ expressions.extend([expr.clone(), not(expr.clone()), not(not(expr))]);+ }+ }+ }+ expressions.push(not(json!({"or": {+ "left": not(atoms[0].clone()),+ "right": {"and":{"left":atoms[1],"right":not(atoms[2].clone())}}+ }})));+ expressions+}++async fn seed<S: PredicateIndexStorage>(engine: &mut Engine<S>) {+ let query = SUPPLEMENT_BACKFILL+ .replace("user {", "user { id")+ .replace("fileType", "fileType properties { id }");+ let data = json!({"user":{"id":"viewer","soup":{"items":(0..FILE_TYPES.len()).map(item).collect::<Vec<_>>()}}});+ let vars = serde_json::Map::new();+ let projections =+ authoritative_projection_mutations(&query, Some("SoupBackfill"), &data).unwrap();+ let projections = crate::properties::augment_authoritative(+ engine.storage(),+ &query,+ Some("SoupBackfill"),+ &vars,+ &data,+ true,+ projections,+ )+ .await+ .unwrap();+ engine+ .hydrate_query_with_projections(+ &query,+ Some("SoupBackfill"),+ &vars,+ &data,+ Some("viewer"),+ projections,+ )+ .await+ .unwrap();+}+async fn assert_sql_parity<S: PredicateIndexStorage>(engine: &mut Engine<S>) {+ for expression in expressions() {+ let expected = (0..FILE_TYPES.len())+ .rev()+ .filter(|&i| sql_reference(&expression, i) == SqlTruth::True)+ .map(key)+ .collect();+ let actual = engine+ .query_predicate_index(&query(expression.clone()))+ .await+ .unwrap()+ .value;+ assert_eq!(+ actual,+ PredicateQueryResult::Complete(expected),+ "{expression}"+ );+ }+}+async fn assert_updated_type<S: PredicateIndexStorage>(engine: &mut Engine<S>, value: &Value) {+ let file_type = json!({"literal":{"fileType":"pdf"}});+ for (filter, matches) in [+ (file_type.clone(), value == "pdf"),+ (not(file_type), !value.is_null() && value != "pdf"),+ ] {+ let query = query(json!({"and":{"left":{"literal":{"id":id(0)}},"right":filter}}));+ let result = engine.query_predicate_index(&query).await.unwrap().value;+ let keys = match result {+ PredicateQueryResult::Complete(keys) | PredicateQueryResult::Optimistic(keys) => keys,+ PredicateQueryResult::Incomplete => {+ panic!("file-type changes must preserve projection completeness")+ }+ };+ assert_eq!(keys, if matches { vec![key(0)] } else { vec![] });+ }+}++async fn lifecycle<S: PredicateIndexStorage>(storage: S) {+ let mut engine = Engine::new(storage);+ seed(&mut engine).await;+ assert_sql_parity(&mut engine).await;+ let mut engine = Engine::new(engine.into_storage());+ assert_sql_parity(&mut engine).await;++ // A partial authoritative update can replace a known type with an unknown+ // string or NULL without losing the complete supplement/property proof.+ let vars =+ json!({"inputs":[{"entity":{"type":"DOCUMENT","id":id(0)},"displayName":"renamed"}]})+ .as_object()+ .unwrap()+ .clone();+ for value in [json!("legacy-format"), Value::Null, json!("pdf")] {+ seed(&mut engine).await;+ let data = json!({"renameEntities":{"results":[{"__typename":"GraphqlMutationSuccess","effects":[{"__typename":"SoupUpdated","item":{+ "__typename":"GraphqlSoupDocument","id":id(0),"fileType":value,"updatedAt":"2025-01-02T00:00:00.000001Z"+ }}]}]}});+ // Start from pdf, then change its membership only in the pending layer.+ let projections = crate::properties::augment_optimistic(+ engine.storage(),+ RENAME,+ Some("Rename"),+ &vars,+ &data,+ optimistic_projection_mutations(&data, 1),+ )+ .await+ .unwrap();+ let (transaction, _) = engine+ .begin_optimistic_write_with_projections(+ None,+ BeginOptimisticWrite {+ uuid: "00000000-0000-0000-0000-000000000100",+ query: RENAME,+ operation_name: Some("Rename"),+ variables: &vars,+ data: &data,+ link_patches: &[],+ revalidations: &[],+ created_at_ms: 1,+ },+ projections,+ )+ .await+ .unwrap();+ assert_updated_type(&mut engine, &value).await;+ let mut reopened = Engine::new(engine.into_storage());+ assert_updated_type(&mut reopened, &value).await;+ let claimed = reopened+ .claim_next_mutation(MutationClaimRequest {+ owner: "test".into(),+ now_ms: 1,+ lease_expires_at_ms: 100,+ })+ .await+ .unwrap()+ .unwrap();+ assert_eq!(claimed.queued.id, transaction);+ reopened+ .rollback_optimistic_write(+ transaction,+ MutationClaimToken {+ owner: "test".into(),+ generation: claimed.lease_generation,+ },+ )+ .await+ .unwrap();+ engine = reopened;+ assert_updated_type(&mut engine, &json!("pdf")).await;++ let projections =+ authoritative_projection_mutations(RENAME, Some("Rename"), &data).unwrap();+ let projections = crate::properties::augment_authoritative(+ engine.storage(),+ RENAME,+ Some("Rename"),+ &vars,+ &data,+ true,+ projections,+ )+ .await+ .unwrap();+ engine+ .write_query_with_registration_and_projections(+ None,+ None,+ NetworkWrite {+ query: RENAME,+ operation_name: Some("Rename"),+ variables: &vars,+ data: &data,+ identity: Some("viewer"),+ },+ projections,+ )+ .await+ .unwrap();+ assert_updated_type(&mut engine, &value).await;+ }+ assert_sql_parity(&mut engine).await;+}++#[test]+fn every_soup_profile_compiles_file_type_negation_with_sql_null_semantics() {+ let data =+ json!({"user":{"soup":{"items":(0..FILE_TYPES.len()).map(item).collect::<Vec<_>>()}}});+ let documents =+ authoritative_projection_mutations(SUPPLEMENT_BACKFILL, Some("SoupBackfill"), &data)+ .unwrap();+ for compile in [+ item_filter_index::compile_soup_flat_v1,+ item_filter_index::compile_soup_flat_v2,+ item_filter_index::compile_soup_flat_v3,+ item_filter_index::compile_soup_flat_v4,+ ] {+ for expression in expressions() {+ let ast = materialize_graphql_filter(filters(expression.clone())).unwrap();+ let item_filter_index::LocalCompileOutcome::Supported(query) = compile(+ &ast,+ SoupFlatRequest {+ sort: SoupIndexSort::UpdatedAt,+ direction: SortDirection::Desc,+ limit: 20,+ has_cursor: false,+ },+ )+ .unwrap() else {+ panic!("file-type filters must be supported");+ };+ let predicate = &query.as_query().partitions[0].predicate;+ for (index, mutation) in documents.iter().enumerate() {+ let ProjectionMutation::Replace(document) = mutation else {+ panic!("complete document");+ };+ assert_eq!(+ document.matches(predicate),+ sql_reference(&expression, index) == SqlTruth::True,+ "{expression}: {:?}",+ FILE_TYPES[index]+ );+ }+ }+ }+}++#[test]+fn raw_file_type_and_sql_null_filtering_in_memory() {+ pollster::block_on(lifecycle(InMemoryStorage::new()));+}+#[test]+fn raw_file_type_and_sql_null_filtering_in_turso() {+ pollster::block_on(async {+ lifecycle(cache_turso::TursoStorage::open_in_memory("raw-file-types").unwrap()).await+ });+}+#[test]+fn malformed_file_type_data_is_still_rejected_but_unknown_filter_inputs_are_not_added() {+ for file_type in [+ json!(false),+ json!(12),+ json!([]),+ json!({}),+ json!("x".repeat(predicate_index::MAX_EXACT_VALUE_BYTES + 1)),+ ] {+ let mut item = item(0);+ item["fileType"] = file_type;+ assert!(+ authoritative_projection_mutations(+ SUPPLEMENT_BACKFILL,+ Some("SoupBackfill"),+ &json!({"user":{"soup":{"items":[item]}}})+ )+ .is_err()+ );+ }+ let mut missing = item(0);+ missing.as_object_mut().unwrap().remove("fileType");+ assert!(+ authoritative_projection_mutations(+ SUPPLEMENT_BACKFILL,+ Some("SoupBackfill"),+ &json!({"user":{"soup":{"items":[missing]}}})+ )+ .is_err()+ );+ assert!(+ compile_current_filter_request(+ filters(json!({"literal":{"fileType":"doc"}})),+ "UPDATED_AT",+ "DESC",+ 20+ )+ .is_err()+ );+ // Query inputs still normalize known spellings; stored values do not.+ assert_eq!(+ query(json!({"literal":{"fileType":".PDF"}})),+ query(json!({"literal":{"fileType":"pdf"}}))+ );+}diff --git a/crates/soup_filter_projection/Cargo.toml b/crates/soup_filter_projection/Cargo.tomlindex 68cadebb77..35fad43629 100644--- a/crates/soup_filter_projection/Cargo.toml+++ b/crates/soup_filter_projection/Cargo.toml@@ -14,7 +14,6 @@ base64 = { workspace = true }chrono = { workspace = true }document_sub_type = { path = "../document_sub_type", default-features = false }item-filter-index = { path = "../item_filter_index" }-model_file_type = { path = "../model_file_type" }models_soup = { path = "../models_soup", optional = true }postcard = { workspace = true }predicate-index = { path = "../predicate_index" }diff --git a/crates/soup_filter_projection/src/lib.rs b/crates/soup_filter_projection/src/lib.rsindex 89aea56b99..c128daa688 100644--- a/crates/soup_filter_projection/src/lib.rs+++ b/crates/soup_filter_projection/src/lib.rs@@ -1,11 +1,8 @@//! Direct-field Soup projection helpers and typed server-fact supplements.#![deny(missing_docs)]-use std::str::FromStr;-pub use document_sub_type::DocumentSubType;use item_filter_index::vocabulary;-use model_file_type::FileType;#[cfg(feature = "models")]use models_soup::{chat::SoupChat, document::SoupDocument, item::SoupItem, project::SoupProject};use predicate_index::{@@ -36,15 +33,12 @@ pub use wire::{/// Maximum authoritative task Status options accepted in one complete projection.pub const MAX_TASK_STATUS_OPTION_IDS: usize = 64;-#[cfg(test)]+#[cfg(all(test, feature = "models"))]mod test;/// Failure to project an authoritative Soup item.#[derive(Debug, Error, PartialEq, Eq)]pub enum ProjectionError {- /// The authoritative document contained an unknown file-type value.- #[error("invalid authoritative Soup document file type `{0}`")]- InvalidFileType(String),/// Document server facts do not match the accompanying item variant.#[error("document server facts do not match Soup item variant")]SourceMismatch,@@ -154,7 +148,8 @@ pub struct DirectProjectionInput {pub owner: String,/// Project or parent UUID, when present.pub project_id: Option<uuid::Uuid>,- /// Document file type, when present. Ignored for other kinds.+ /// Raw document file type, when present. Ignored for other kinds.+ /// Preserve the stored spelling: Soup's SQL compares this text literally.pub file_type: Option<String>,/// Creation timestamp.pub created_at: chrono::DateTime<chrono::Utc>,@@ -173,7 +168,8 @@ pub struct DirectProjectionPatchInput {pub owner: Option<String>,/// Replacement project/parent value. Outer `None` means unchanged.pub project_id: Option<Option<uuid::Uuid>>,- /// Replacement document file type. Outer `None` means unchanged.+ /// Replacement raw document file type. Outer `None` means unchanged;+ /// `Some(None)` removes the fact for a SQL NULL.pub file_type: Option<Option<String>>,/// Replacement creation timestamp when supplied.pub created_at: Option<chrono::DateTime<chrono::Utc>>,@@ -192,10 +188,7 @@ pub fn project_direct_fields(if input.kind == SoupFlatEntityKind::Document&& let Some(file_type) = input.file_type{- let canonical = FileType::from_str(&file_type)- .map_err(|_| ProjectionError::InvalidFileType(file_type))?- .to_string();- exact_facts.push(utf8_fact(vocabulary::file_type(), canonical)?);+ exact_facts.push(utf8_fact(vocabulary::file_type(), file_type)?);}projection(input.record_key,@@ -308,12 +301,7 @@ pub fn patch_direct_fields(&& let Some(file_type) = input.file_type{let values = file_type- .map(|file_type| -> Result<ExactValue, ProjectionError> {- let canonical = FileType::from_str(&file_type)- .map_err(|_| ProjectionError::InvalidFileType(file_type))?- .to_string();- Ok(ExactValue::utf8(canonical)?)- })+ .map(ExactValue::utf8).transpose()?.into_iter().collect();diff --git a/crates/soup_filter_projection/src/profile.rs b/crates/soup_filter_projection/src/profile.rsindex f45d7166f3..1416e5f63b 100644--- a/crates/soup_filter_projection/src/profile.rs+++ b/crates/soup_filter_projection/src/profile.rs@@ -1,10 +1,12 @@-use std::str::FromStr;+use std::collections::HashMap;use item_filter_index::vocabulary;-use model_file_type::FileType;use predicate_index::{ExactFact, IndexDocument, IntegerFact, Token, ValidationError};use thiserror::Error;+#[cfg(test)]+mod test;+/// Semantic validation failure for one complete versioned Soup document.#[derive(Debug, Error, PartialEq, Eq)]pub enum ProfileValidationError {@@ -125,6 +127,7 @@ fn validate_exact_facts(let mut channel_participant = 0;let mut channel_team = 0;let mut channel_organization = 0;+ let mut notification_states = HashMap::new();for fact in facts {let attribute = &fact.attribute;@@ -146,11 +149,9 @@ fn validate_exact_facts(}} else if attribute == &vocabulary::file_type() && kind == PartitionKind::Document {file_type += 1;- let value = std::str::from_utf8(value)+ // The GraphQL field is nullable text, not the supported-format enum.+ std::str::from_utf8(value).map_err(|_| ProfileValidationError::InvalidValue("file-type"))?;- if FileType::from_str(value).is_err() {- return Err(ProfileValidationError::InvalidValue("file-type"));- }} else if attribute == &vocabulary::document_sub_type() && kind == PartitionKind::Document {sub_type += 1;if !matches!(value, b"task" | b"snippet" | b"skill") {@@ -206,12 +207,10 @@ fn validate_exact_facts(if value.len() != 16 {return Err(ProfileValidationError::InvalidValue("notification-id"));}- if facts.iter().any(|other| {- other.value == fact.value- && other.attribute != *attribute- && (other.attribute == vocabulary::notification_seen()- || other.attribute == vocabulary::notification_unseen())- }) {+ if notification_states+ .insert(&fact.value, attribute)+ .is_some_and(|previous| previous != attribute)+ {return Err(ProfileValidationError::InvalidValue("notification-state"));}} else {diff --git a/crates/soup_filter_projection/src/profile/test.rs b/crates/soup_filter_projection/src/profile/test.rsnew file mode 100644index 0000000000..082c2153f1--- /dev/null+++ b/crates/soup_filter_projection/src/profile/test.rs@@ -0,0 +1,163 @@+use super::*;+use crate::{+ DirectProjectionInput, DirectProjectionPatchInput, SoupCacheProjectionSupplement,+ SoupFlatEntityKind, compose_soup_flat_v3, patch_direct_fields, project_direct_fields,+};+use predicate_index::{ExactValue, RecordKey};++fn notification(attribute: Token, id: u128) -> ExactFact {+ ExactFact {+ attribute,+ value: ExactValue::new(uuid::Uuid::from_u128(id).as_bytes()).unwrap(),+ }+}++fn project() -> IndexDocument {+ let timestamp = chrono::DateTime::from_timestamp(0, 0).unwrap();+ let mut document = compose_soup_flat_v3(+ DirectProjectionInput {+ record_key: RecordKey::new("GraphqlSoupProject:00000000-0000-0000-0000-000000000001")+ .unwrap(),+ kind: SoupFlatEntityKind::Project,+ id: uuid::Uuid::from_u128(1),+ owner: "macro|[email protected]".into(),+ project_id: None,+ file_type: None,+ created_at: timestamp,+ updated_at: timestamp,+ },+ None,+ None,+ )+ .unwrap();+ document.profile = vocabulary::profile_v4();+ document+}++#[test]+fn large_notification_sets_validate_but_conflicting_states_do_not() {+ let mut document = project();+ document+ .exact_facts+ .extend((1..=10_000).map(|id| notification(vocabulary::notification_unseen(), id)));+ document+ .exact_facts+ .push(notification(vocabulary::notification_seen(), 10_001));+ // Duplicate same-state membership is safe and canonicalization removes it.+ document+ .exact_facts+ .push(notification(vocabulary::notification_unseen(), 1));+ validate_soup_flat_v4(&document).unwrap();+ document+ .exact_facts+ .push(notification(vocabulary::notification_seen(), 1));+ assert_eq!(+ validate_soup_flat_v4(&document),+ Err(ProfileValidationError::InvalidValue("notification-state")),+ );+ // Conflict detection must not depend on which state is visited first.+ document.exact_facts.reverse();+ assert_eq!(+ validate_soup_flat_v4(&document),+ Err(ProfileValidationError::InvalidValue("notification-state")),+ );+}++fn document_input(file_type: Option<&str>) -> DirectProjectionInput {+ DirectProjectionInput {+ kind: SoupFlatEntityKind::Document,+ record_key: RecordKey::new("GraphqlSoupDocument:00000000-0000-0000-0000-000000000001")+ .unwrap(),+ id: uuid::Uuid::from_u128(1),+ owner: "macro|[email protected]".into(),+ project_id: None,+ file_type: file_type.map(str::to_owned),+ created_at: chrono::DateTime::from_timestamp(0, 0).unwrap(),+ updated_at: chrono::DateTime::from_timestamp(0, 0).unwrap(),+ }+}++#[test]+fn raw_file_types_survive_snapshots_and_patches() {+ for file_type in [+ None,+ Some(""),+ Some("doc"),+ Some("custom.ext"),+ Some("pdf"),+ Some("PDF"),+ Some(".pdf"),+ Some("日本語"),+ ] {+ let input = document_input(file_type);+ let supplement =+ SoupCacheProjectionSupplement::document(input.record_key.clone(), false, false, vec![]);+ let mut document = compose_soup_flat_v3(input, None, Some(&supplement)).unwrap();+ let values = document+ .exact_facts+ .iter()+ .filter(|fact| fact.attribute == vocabulary::file_type())+ .map(|fact| fact.value.clone())+ .collect::<Vec<_>>();+ let expected = file_type+ .map(|value| ExactValue::utf8(value).unwrap())+ .into_iter()+ .collect::<Vec<_>>();+ assert_eq!(values, expected);+ document.profile = vocabulary::profile_v4();+ validate_soup_flat_v4(&document).unwrap();+ let patch = patch_direct_fields(DirectProjectionPatchInput {+ record_key: document.record_key,+ kind: SoupFlatEntityKind::Document,+ owner: None,+ project_id: None,+ file_type: Some(file_type.map(str::to_owned)),+ created_at: None,+ updated_at: None,+ })+ .unwrap();+ let predicate_index::OptimisticProjectionMutation::Patch { exact, .. } = patch else {+ panic!("direct fields produce a patch");+ };+ assert_eq!(exact.len(), 1);+ assert_eq!(exact[0].attribute, vocabulary::file_type());+ assert_eq!(exact[0].values, expected);+ }+}++#[test]+fn file_type_size_utf8_and_cardinality_are_still_validated() {+ let oversized = "a".repeat(predicate_index::MAX_EXACT_VALUE_BYTES + 1);+ assert!(project_direct_fields(document_input(Some(&oversized))).is_err());+ let input = document_input(Some("doc"));+ let supplement =+ SoupCacheProjectionSupplement::document(input.record_key.clone(), false, false, vec![]);+ let mut document = compose_soup_flat_v3(input, None, Some(&supplement)).unwrap();+ document.profile = vocabulary::profile_v4();+ document.exact_facts.push(ExactFact {+ attribute: vocabulary::file_type(),+ value: ExactValue::new([0xff]).unwrap(),+ });+ assert_eq!(+ validate_soup_flat_v4(&document),+ Err(ProfileValidationError::InvalidValue("file-type"))+ );+ document.exact_facts.last_mut().unwrap().value = ExactValue::utf8("other").unwrap();+ assert_eq!(+ validate_soup_flat_v4(&document),+ Err(ProfileValidationError::Duplicate("file-type"))+ );+}++#[test]+fn notification_value_shape_is_still_validated() {+ let mut document = project();+ document.exact_facts.push(ExactFact {+ attribute: vocabulary::notification_seen(),+ value: ExactValue::utf8("not-a-uuid").unwrap(),+ });+ assert_eq!(+ validate_soup_flat_v4(&document),+ Err(ProfileValidationError::InvalidValue("notification-id")),+ );+}diff --git a/crates/soup_filter_projection/src/test.rs b/crates/soup_filter_projection/src/test.rsindex ca02f87e6b..efe52f70ea 100644--- a/crates/soup_filter_projection/src/test.rs+++ b/crates/soup_filter_projection/src/test.rs@@ -106,7 +106,7 @@ fn optimistic_direct_projection_and_patch_share_authoritative_vocabulary() {}).unwrap();assert!(projection.exact_facts.iter().any(|fact| {- fact.attribute == vocabulary::file_type() && fact.value == ExactValue::utf8("md").unwrap()+ fact.attribute == vocabulary::file_type() && fact.value == ExactValue::utf8(".MD").unwrap()}));let patch = patch_direct_fields(DirectProjectionPatchInput {