Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

feat(search_processing_service): properties kafka (#5319)

Will Hutchinson35e5b4126965
diff --git a/crates/properties/src/domain/ports.rs b/crates/properties/src/domain/ports.rs
index 8a2c875ddb..6fcb4152bd 100644
--- a/crates/properties/src/domain/ports.rs
+++ b/crates/properties/src/domain/ports.rs
@@ -407,19 +407,3 @@ pub trait NotificationService: Send + Sync + 'static {
notification: TaskAssignedNotification<'a>,
) -> impl Future<Output = Result<(), Self::Err>> + Send;
}
-
-/// Port for keeping an entity's indexed properties in sync after a mutation.
-///
-/// Mirrors the per-domain `*SearchIndexer` ports (e.g. `CallSearchIndexer`):
-/// the domain calls it on a write and an SQS-backed adapter in the composition
-/// root publishes the upsert. `dyn`-compatible (boxed future) so it can be an
-/// optional collaborator on the service without adding a generic parameter.
-pub trait PropertySearchIndexer: Send + Sync + std::fmt::Debug {
- /// Enqueue an upsert of the entity's indexed properties. Best-effort —
- /// callers log and continue on error.
- fn enqueue_upsert(
- &self,
- entity_id: String,
- entity_type: EntityType,
- ) -> std::pin::Pin<Box<dyn std::future::Future<Output = anyhow::Result<()>> + Send>>;
-}
diff --git a/crates/properties/src/domain/service_impl/mod.rs b/crates/properties/src/domain/service_impl/mod.rs
index 6a918b1727..b85ade4a44 100644
--- a/crates/properties/src/domain/service_impl/mod.rs
+++ b/crates/properties/src/domain/service_impl/mod.rs
@@ -29,8 +29,6 @@ use models_properties::{EntityReference, EntityType};
use system_properties::SystemPropertyKey;
use uuid::Uuid;
-use std::sync::Arc;
-
use super::error::PropertiesErr;
use super::events::{
EntityPropertiesClearedMetadata, EntityPropertyDeletedMetadata, EntityPropertyUpdatedMetadata,
@@ -44,29 +42,15 @@ use super::model::{
PropertyTargetKey, ResolvedPropertySubject, TagScope, TagSet, UpdatePropertyOptionOutcome,
ViewReceipt,
};
-use super::ports::{NotificationService, PermissionService, PropertiesRepo, PropertySearchIndexer};
+use super::ports::{NotificationService, PermissionService, PropertiesRepo};
use super::service::{PropertiesService, TeamReceipt, team_id_from_receipt};
use helpers::{
extract_option_ids_from_property_value, is_property_applicable_to, retain_caller_visible_tags,
};
-/// Entity types whose search index denormalizes property values, i.e. whose
-/// property mutations must enqueue a search reindex.
-fn is_search_indexed(entity_type: EntityType) -> bool {
- matches!(
- entity_type,
- EntityType::Task
- | EntityType::Document
- | EntityType::Thread
- | EntityType::Chat
- | EntityType::Project
- | EntityType::CallRecord
- )
-}
-
/// Implementation of [`PropertiesService`] using repository, permission,
-/// notification, search-indexing, and event-publishing ports.
+/// notification, and event-publishing ports.
#[derive(Debug)]
pub struct PropertiesServiceImpl<R, P, N, B = NoopMacroEventBroker>
where
@@ -78,7 +62,6 @@ where
repository: R,
permission_service: Option<P>,
notification_service: Option<N>,
- search_indexer: Option<Arc<dyn PropertySearchIndexer>>,
event_broker: B,
}
@@ -98,7 +81,6 @@ where
repository,
permission_service,
notification_service,
- search_indexer: None,
event_broker: NoopMacroEventBroker,
}
}
@@ -120,18 +102,10 @@ where
repository: self.repository,
permission_service: self.permission_service,
notification_service: self.notification_service,
- search_indexer: self.search_indexer,
event_broker,
}
}
- /// Attach a search-reindex publisher so property mutations refresh the
- /// search index. Builder-style so existing constructions are unaffected.
- pub fn with_search_indexer(mut self, search_indexer: Arc<dyn PropertySearchIndexer>) -> Self {
- self.search_indexer = Some(search_indexer);
- self
- }
-
/// Publish a property lifecycle event without coupling broker availability
/// to a completed mutation.
fn publish_property_event(&self, event: PropertyMacroEvent) {
@@ -317,24 +291,6 @@ where
.ok_or_else(|| PropertiesErr::Validation("Missing property target".to_string()))
}
- /// Best-effort publish of a property reindex for entity types whose
- /// search index denormalizes property values. Logs and continues on
- /// failure so a missed reindex never fails the mutation itself.
- async fn enqueue_property_upsert(&self, entity_id: &str, entity_type: EntityType) {
- let Some(search_indexer) = self.search_indexer.as_ref() else {
- return;
- };
- if !is_search_indexed(entity_type) {
- return;
- }
- if let Err(error) = search_indexer
- .enqueue_upsert(entity_id.to_string(), entity_type)
- .await
- {
- tracing::warn!(error = ?error, entity_id = %entity_id, "failed to enqueue search reindex for property change");
- }
- }
-
/// Fetch a property definition, ensuring it exists, isn't a system
/// property, and is owned by the caller (their user property or a property
/// of their team).
@@ -617,7 +573,6 @@ where
.await
.map_err(anyhow::Error::from)?;
- self.enqueue_property_upsert(entity_id, entity_type).await;
self.publish_property_event(Self::entity_property_updated_event(
&property,
&property_value,
@@ -686,8 +641,6 @@ where
.await
.map_err(anyhow::Error::from)?;
- self.enqueue_property_upsert(access.entity_id(), subject.storage_entity_type)
- .await;
self.publish_property_event(Self::entity_property_updated_event(
&mutation.property,
&mutation.value,
@@ -724,8 +677,6 @@ where
.await
.map_err(anyhow::Error::from)?;
- self.enqueue_property_upsert(access.entity_id(), subject.storage_entity_type)
- .await;
if let Some(mutation) = mutation {
self.publish_property_event(Self::entity_property_updated_event(
&mutation.property,
@@ -798,8 +749,6 @@ where
.await
.map_err(anyhow::Error::from)?;
- self.enqueue_property_upsert(access.entity_id(), entity_type)
- .await;
self.publish_entity_property_selection_events(&selections, access);
Ok(selections)
@@ -887,8 +836,6 @@ where
continue;
}
- // Convert the repo error to a Send type up front so the raw
- // `R::Err` is never held across the reindex await below.
let update_result = self
.repository
.bulk_update_entity_property_options(
@@ -905,7 +852,6 @@ where
.last()
.map(|selection| selection.option_ids.clone())
.unwrap_or_default();
- self.enqueue_property_upsert(entity_id, entity_type).await;
self.publish_entity_property_selection_events(&selections, &access[index]);
outcomes[index] = Some(EntityOptionUpdateOutcome::Applied { option_ids });
}
diff --git a/crates/properties/src/lib.rs b/crates/properties/src/lib.rs
index 468e896301..640882676b 100644
--- a/crates/properties/src/lib.rs
+++ b/crates/properties/src/lib.rs
@@ -21,9 +21,7 @@ pub use domain::model::{
EditReceipt, EntityPropertiesKey, EntityPropertyInfo, PropertyAccessReceiptExt,
PropertyOptionInfo, PropertyTargetKey, ViewReceipt, canonical_entity_type,
};
-pub use domain::ports::{
- NotificationService, PermissionService, PropertiesRepo, PropertySearchIndexer,
-};
+pub use domain::ports::{NotificationService, PermissionService, PropertiesRepo};
pub use domain::service::{PropertiesService, TeamReceipt};
pub use domain::service_impl::PropertiesServiceImpl;
#[cfg(feature = "outbound")]
diff --git a/crates/sqs_client/src/search/document.rs b/crates/sqs_client/src/search/document.rs
index e827705594..045c71b772 100644
--- a/crates/sqs_client/src/search/document.rs
+++ b/crates/sqs_client/src/search/document.rs
@@ -2,16 +2,6 @@
use model::document::{BackfillSearchDocumentInformation, FileType};
use s3_key::CONVERTED_DOCUMENT_FILE_NAME;
-/// A request to refresh only the indexed properties of a document (e.g. after
-/// a status / assignee / tag change), without re-extracting its content.
-#[derive(serde::Serialize, serde::Deserialize, PartialEq, Eq, Debug)]
-pub struct DocumentPropertiesUpdate {
- /// The document (entity) whose indexed properties should be refreshed.
- pub document_id: String,
- /// The property entity type the values are keyed under (e.g. "TASK").
- pub entity_type: String,
-}
-
/// Search text extractor message
#[derive(serde::Serialize, serde::Deserialize, PartialEq, Eq, Debug)]
pub struct SearchExtractorMessage {
diff --git a/crates/sqs_client/src/search/mod.rs b/crates/sqs_client/src/search/mod.rs
index 59ceb16699..a3bd10ffbe 100644
--- a/crates/sqs_client/src/search/mod.rs
+++ b/crates/sqs_client/src/search/mod.rs
@@ -4,7 +4,7 @@ use crate::{
call::{CallRecordMessage, RemoveCallRecord},
channel::ChannelMessageUpdate,
chat::ChatMessage,
- document::{DocumentPropertiesUpdate, SearchExtractorMessage},
+ document::SearchExtractorMessage,
email::{EmailLinkMessage, EmailMessage, EmailThreadBatchMessage, EmailThreadMessage},
project::UpsertProject,
},
@@ -71,7 +71,6 @@ pub enum SearchQueueMessage {
// Document
ExtractDocumentText(SearchExtractorMessage),
ExtractSync(SearchExtractorMessage),
- UpdateDocumentProperties(DocumentPropertiesUpdate),
// Chat
/// SQS backfill work-queue contract for reconciling a chat message.
ChatMessage(ChatMessage),
@@ -98,7 +97,6 @@ impl PrimaryId for SearchQueueMessage {
match self {
SearchQueueMessage::ExtractDocumentText(message) => message.document_id.clone(),
SearchQueueMessage::ExtractSync(message) => message.document_id.clone(),
- SearchQueueMessage::UpdateDocumentProperties(message) => message.document_id.clone(),
// The message id keeps entries unique within an SQS batch.
SearchQueueMessage::ChatMessage(message) => message.message_id.clone(),
SearchQueueMessage::ExtractEmailMessage(message)
@@ -128,7 +126,6 @@ impl SearchQueueMessage {
// Document
SearchQueueMessage::ExtractDocumentText(_) => Operation::ExtractText,
SearchQueueMessage::ExtractSync(_) => Operation::ExtractSync,
- SearchQueueMessage::UpdateDocumentProperties(_) => Operation::UpdateMetadata,
// Chat
SearchQueueMessage::ChatMessage(_) => Operation::ExtractText,
// Email
diff --git a/services/document_storage_service/src/main.rs b/services/document_storage_service/src/main.rs
index ebb12c4026..ec849b1883 100644
--- a/services/document_storage_service/src/main.rs
+++ b/services/document_storage_service/src/main.rs
@@ -382,12 +382,7 @@ async fn main() -> anyhow::Result<()> {
Some(permission_checker),
Some(notification_service),
)
- .with_event_broker(macro_event_broker.clone())
- .with_search_indexer(Arc::new(
- crate::service::property_search_indexer::SqsPropertySearchIndexer::new(
- sqs_client.clone(),
- ),
- )),
+ .with_event_broker(macro_event_broker.clone()),
);
// Create the channel list service used by soup.
diff --git a/services/document_storage_service/src/service/mod.rs b/services/document_storage_service/src/service/mod.rs
index f4d84f1703..167fbce172 100644
--- a/services/document_storage_service/src/service/mod.rs
+++ b/services/document_storage_service/src/service/mod.rs
@@ -3,6 +3,5 @@ pub mod conn_gateway;
pub mod delete_document_worker;
pub mod document_event_publisher;
pub mod entity_mutation;
-pub mod property_search_indexer;
pub mod s3;
pub mod soup_favorites_reader;
diff --git a/services/document_storage_service/src/service/property_search_indexer.rs b/services/document_storage_service/src/service/property_search_indexer.rs
deleted file mode 100644
index 8ccaf4f105..0000000000
--- a/services/document_storage_service/src/service/property_search_indexer.rs
+++ /dev/null
@@ -1,40 +0,0 @@
-//! Adapter that publishes property search-index upserts to the search queue.
-
-use models_properties::EntityType;
-use properties::PropertySearchIndexer;
-use sqs_client::SQS;
-use sqs_client::search::SearchQueueMessage;
-use sqs_client::search::document::DocumentPropertiesUpdate;
-
-/// Publishes an upsert of an entity's indexed properties to the shared search
-/// event queue so the search-processing service refreshes them after a write.
-#[derive(Debug)]
-pub struct SqsPropertySearchIndexer {
- sqs: SQS,
-}
-
-impl SqsPropertySearchIndexer {
- pub fn new(sqs: SQS) -> Self {
- Self { sqs }
- }
-}
-
-impl PropertySearchIndexer for SqsPropertySearchIndexer {
- fn enqueue_upsert(
- &self,
- entity_id: String,
- entity_type: EntityType,
- ) -> std::pin::Pin<Box<dyn std::future::Future<Output = anyhow::Result<()>> + Send>> {
- let sqs = self.sqs.clone();
- Box::pin(async move {
- sqs.send_message_to_search_event_queue(SearchQueueMessage::UpdateDocumentProperties(
- DocumentPropertiesUpdate {
- document_id: entity_id,
- entity_type: entity_type.to_string(),
- },
- ))
- .await?;
- Ok(())
- })
- }
-}
diff --git a/services/search_processing_service/Cargo.toml b/services/search_processing_service/Cargo.toml
index 1e7075884e..841d2e9b5b 100644
--- a/services/search_processing_service/Cargo.toml
+++ b/services/search_processing_service/Cargo.toml
@@ -15,7 +15,9 @@ processing = [
"dep:kafka_util",
"dep:macro_event_broker",
"dep:macro_event_topics",
+ "dep:models_properties",
"dep:projects",
+ "dep:properties",
"dep:rdkafka",
"dep:rootcause",
"dep:tokio-retry",
@@ -24,8 +26,8 @@ service = [
"dep:documents",
"dep:macro_event_broker",
"macro_db_client",
- "models_properties",
- "properties",
+ "dep:models_properties",
+ "dep:properties",
]
pdf = ["dep:pdfium-render"]
diff --git a/services/search_processing_service/src/domain/models.rs b/services/search_processing_service/src/domain/models.rs
index 1193133af7..7dd0e8474a 100644
--- a/services/search_processing_service/src/domain/models.rs
+++ b/services/search_processing_service/src/domain/models.rs
@@ -1,4 +1,5 @@
use chrono::{DateTime, Utc};
+use models_properties::EntityType;
use serde::{Deserialize, Serialize};
use sqs_client::search::SearchQueueMessage;
use thiserror::Error;
@@ -31,12 +32,40 @@ impl SourcePage {
}
}
+/// One page of entities whose denormalized properties must be reindexed.
+///
+/// A page has one entity type so the source parses and validates that type
+/// once, while the indexer receives typed work items. `rows_consumed` stays
+/// separate from the number of IDs so offset pagination follows source rows.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct PropertySourcePage {
+ /// IDs of the entities to reindex.
+ pub entity_ids: Vec<String>,
+ /// Property entity type shared by every ID in this page.
+ pub entity_type: EntityType,
+ /// Number of source rows covered by this page.
+ pub rows_consumed: usize,
+}
+
+impl PropertySourcePage {
+ /// Construct the end-of-source page for an entity type.
+ pub fn empty(entity_type: EntityType) -> Self {
+ Self {
+ entity_ids: Vec::new(),
+ entity_type,
+ rows_consumed: 0,
+ }
+ }
+}
+
#[derive(Debug, Error)]
pub enum BackfillError {
#[error("failed reading backfill source")]
Source(#[source] anyhow::Error),
#[error("failed publishing to search event queue")]
Publish(#[source] anyhow::Error),
+ #[error("failed reindexing entity properties")]
+ Reindex(#[source] anyhow::Error),
}
#[derive(Debug, Clone, Copy, Default, Deserialize, PartialEq, Eq)]
@@ -193,10 +222,9 @@ pub struct EmailBackfillRequest {
pub batch_size: Option<usize>,
}
-/// Property-only backfill: re-enqueue a property update message for every
-/// entity of one type that has property rows, refreshing the denormalized
-/// `properties` field without re-extracting content. Used after adding the
-/// field to an index's mapping.
+/// Property-only backfill: directly reindex every entity of one type that has
+/// property rows, refreshing the denormalized `properties` field without
+/// re-extracting content. Used after adding the field to an index's mapping.
#[derive(Debug, Clone, Deserialize)]
pub struct PropertiesBackfillRequest {
/// The property entity type to backfill (e.g. "thread").
diff --git a/services/search_processing_service/src/domain/ports.rs b/services/search_processing_service/src/domain/ports.rs
index ed4eedee0c..ef8dc868e0 100644
--- a/services/search_processing_service/src/domain/ports.rs
+++ b/services/search_processing_service/src/domain/ports.rs
@@ -1,12 +1,14 @@
//! Outbound trait contracts for the backfill domain.
//!
-//! Two ports:
+//! Three ports:
//!
//! - [`BackfillSource`] — entity-aware reader. One method per searchable
-//! entity, each producing a [`SourcePage`] (messages + rows consumed) for
-//! a given offset. The orchestrator drives sources through these methods.
+//! entity, producing either a queue-backed [`SourcePage`] or a typed
+//! [`PropertySourcePage`] for a given pagination position.
//! - [`SearchEventPublisher`] — entity-agnostic batch publisher onto the
//! search-event queue.
+//! - [`PropertyBackfillIndexer`] — directly reindexes one typed entity's
+//! denormalized properties.
//!
//! Splitting reads (source) from the queue write (publisher) keeps each
//! adapter single-concern and lets the application-level pagination loop be
@@ -14,13 +16,14 @@
use std::future::Future;
+use models_properties::EntityType;
use sqs_client::search::SearchQueueMessage;
use super::models::{
BackfillError, CallBackfillCursor, CallBackfillRequest, ChannelBackfillRequest,
ChatBackfillCursor, ChatBackfillRequest, DocumentBackfillCursor, DocumentBackfillRequest,
EmailBackfillRequest, ProjectBackfillCursor, ProjectBackfillRequest, PropertiesBackfillRequest,
- SourcePage,
+ PropertySourcePage, SourcePage,
};
/// Publishes batches of search-event messages.
@@ -31,8 +34,18 @@ pub trait SearchEventPublisher: Send + Sync + 'static {
) -> impl Future<Output = Result<(), BackfillError>> + Send;
}
-/// Source of backfill messages across every searchable entity. The
-/// orchestrator's `drain_source` loop calls one of these per request.
+/// Directly reindexes denormalized properties for one typed entity.
+pub trait PropertyBackfillIndexer: Send + Sync + 'static {
+ /// Refetch and overwrite the indexed properties for an entity.
+ fn reindex(
+ &self,
+ entity_id: &str,
+ entity_type: EntityType,
+ ) -> impl Future<Output = Result<(), BackfillError>> + Send;
+}
+
+/// Source of backfill work across every searchable entity. The orchestrator
+/// calls one of these methods per request.
///
/// `rows_consumed` on each [`SourcePage`] is the unit the orchestrator
/// advances by; `messages` is what gets handed to the publisher. Sources
@@ -92,7 +105,8 @@ pub trait BackfillSource: Send + Sync + 'static {
&self,
req: &PropertiesBackfillRequest,
offset: usize,
- ) -> impl Future<Output = Result<SourcePage, BackfillError>> + Send;
+ ) -> impl Future<Output = Result<PropertySourcePage, BackfillError>> + Send;
+
/// Projects paginate by keyset cursor (mirroring documents): each call
/// passes the cursor of the last row from the previous page (or `None`
/// for the first page), and the implementation returns the page plus
diff --git a/services/search_processing_service/src/domain/service.rs b/services/search_processing_service/src/domain/service.rs
index 52cabb640d..fe54092708 100644
--- a/services/search_processing_service/src/domain/service.rs
+++ b/services/search_processing_service/src/domain/service.rs
@@ -1,24 +1,25 @@
//! Application-level backfill service.
//!
//! [`BackfillService`] is the inbound contract the HTTP layer talks to. The
-//! [`BackfillOrchestrator`] holds one [`BackfillSource`] (which knows about
-//! every entity type) and one [`SearchEventPublisher`], and runs the shared
-//! paginate-and-publish loop that drains a source onto the publisher. The
-//! loop lives here (in the domain) so it can be tested with in-memory fakes
-//! — adapters stay single-concern.
+//! [`BackfillOrchestrator`] holds a [`BackfillSource`], a
+//! [`SearchEventPublisher`] for queue-backed families, and a
+//! [`PropertyBackfillIndexer`] for direct property indexing. The orchestration
+//! loops live here so they can be tested with in-memory fakes while adapters
+//! stay single-concern.
use std::future::Future;
use std::sync::Arc;
+use futures::{StreamExt, TryStreamExt, stream};
use tokio_util::sync::CancellationToken;
use super::jobs::JobProgress;
use super::models::{
BackfillError, BackfillReceipt, CallBackfillRequest, ChannelBackfillRequest,
ChatBackfillRequest, DocumentBackfillRequest, EmailBackfillRequest, ProjectBackfillRequest,
- PropertiesBackfillRequest, SourcePage,
+ PropertiesBackfillRequest, PropertySourcePage, SourcePage,
};
-use super::ports::{BackfillSource, SearchEventPublisher};
+use super::ports::{BackfillSource, PropertyBackfillIndexer, SearchEventPublisher};
/// Drive a source by repeatedly calling `fetch(cursor)`, publishing each
/// page's messages, and stopping when the source reports zero rows
@@ -112,18 +113,24 @@ pub trait BackfillService: Send + Sync + 'static {
) -> impl Future<Output = Result<BackfillReceipt, BackfillError>> + Send;
}
-pub struct BackfillOrchestrator<S, P> {
+pub struct BackfillOrchestrator<S, P, I> {
source: S,
publisher: P,
+ property_indexer: I,
}
-impl<S, P> BackfillOrchestrator<S, P>
+impl<S, P, I> BackfillOrchestrator<S, P, I>
where
S: BackfillSource,
P: SearchEventPublisher,
+ I: PropertyBackfillIndexer,
{
- pub fn new(source: S, publisher: P) -> Self {
- Self { source, publisher }
+ pub fn new(source: S, publisher: P, property_indexer: I) -> Self {
+ Self {
+ source,
+ publisher,
+ property_indexer,
+ }
}
}
@@ -167,10 +174,58 @@ where
Ok(BackfillReceipt { enqueued })
}
-impl<S, P> BackfillService for BackfillOrchestrator<S, P>
+/// Drain typed property pages with a bounded number of direct reindexes.
+/// Cancellation is observed only at page boundaries so each started page is
+/// either completed and counted or fails without updating progress.
+async fn drain_property_source<Fut, I>(
+ indexer: &I,
+ progress: &JobProgress,
+ cancel: &CancellationToken,
+ fetch: impl Fn(usize) -> Fut,
+) -> Result<BackfillReceipt, BackfillError>
+where
+ Fut: Future<Output = Result<PropertySourcePage, BackfillError>>,
+ I: PropertyBackfillIndexer,
+{
+ let mut offset = 0usize;
+ let mut enqueued = 0usize;
+
+ loop {
+ if cancel.is_cancelled() {
+ tracing::info!(enqueued, "property backfill cancelled between pages");
+ break;
+ }
+
+ let page = fetch(offset).await?;
+ if page.rows_consumed == 0 {
+ break;
+ }
+
+ let PropertySourcePage {
+ entity_ids,
+ entity_type,
+ rows_consumed,
+ } = page;
+
+ stream::iter(entity_ids)
+ .map(|entity_id| async move { indexer.reindex(&entity_id, entity_type).await })
+ .buffer_unordered(20)
+ .try_collect::<Vec<()>>()
+ .await?;
+
+ enqueued += rows_consumed;
+ offset += rows_consumed;
+ progress.add(rows_consumed).await;
+ }
+
+ Ok(BackfillReceipt { enqueued })
+}
+
+impl<S, P, I> BackfillService for BackfillOrchestrator<S, P, I>
where
S: BackfillSource,
P: SearchEventPublisher,
+ I: PropertyBackfillIndexer,
{
async fn backfill_calls(
&self,
@@ -238,7 +293,7 @@ where
progress: Arc<JobProgress>,
cancel: CancellationToken,
) -> Result<BackfillReceipt, BackfillError> {
- drain_source(&self.publisher, &progress, &cancel, |offset| {
+ drain_property_source(&self.property_indexer, &progress, &cancel, |offset| {
self.source.fetch_entity_properties(&req, offset)
})
.await
diff --git a/services/search_processing_service/src/domain/service/test.rs b/services/search_processing_service/src/domain/service/test.rs
index b4ab729921..058047e0a3 100644
--- a/services/search_processing_service/src/domain/service/test.rs
+++ b/services/search_processing_service/src/domain/service/test.rs
@@ -1,12 +1,20 @@
-use std::sync::Mutex;
-
+use std::{
+ collections::VecDeque,
+ sync::{
+ Mutex,
+ atomic::{AtomicUsize, Ordering},
+ },
+ time::Duration,
+};
+
+use models_properties::EntityType;
use sqs_client::search::{SearchQueueMessage, call::CallRecordMessage};
use tokio_util::sync::CancellationToken;
use super::*;
use crate::domain::jobs::JobProgress;
-use crate::domain::models::{CallBackfillRequest, SourcePage};
-use crate::domain::ports::SearchEventPublisher;
+use crate::domain::models::{CallBackfillRequest, PropertySourcePage, SourcePage};
+use crate::domain::ports::{PropertyBackfillIndexer, SearchEventPublisher};
/// Programmable fake fetch closure. `drain_source` takes any
/// `Fn(usize) -> Future<SourcePage>`, so the test fakes don't need to
@@ -478,3 +486,288 @@ async fn cancel_between_pages_stops_drain_after_current_page() {
assert_eq!(source.observed_offsets(), vec![0]);
assert_eq!(*publisher.seen.lock().unwrap(), vec![5]);
}
+
+struct FakePropertySource {
+ pages: Mutex<VecDeque<PropertySourcePage>>,
+ offsets: Mutex<Vec<usize>>,
+ entity_type: EntityType,
+}
+
+impl FakePropertySource {
+ fn new(entity_type: EntityType, pages: Vec<PropertySourcePage>) -> Self {
+ Self {
+ pages: Mutex::new(pages.into_iter().collect()),
+ offsets: Mutex::new(Vec::new()),
+ entity_type,
+ }
+ }
+
+ async fn fetch_page(&self, offset: usize) -> Result<PropertySourcePage, BackfillError> {
+ self.offsets.lock().unwrap().push(offset);
+ Ok(self
+ .pages
+ .lock()
+ .unwrap()
+ .pop_front()
+ .unwrap_or_else(|| PropertySourcePage::empty(self.entity_type)))
+ }
+
+ fn observed_offsets(&self) -> Vec<usize> {
+ self.offsets.lock().unwrap().clone()
+ }
+}
+
+#[derive(Default)]
+struct RecordingPropertyIndexer {
+ entities: Mutex<Vec<(String, EntityType)>>,
+}
+
+impl RecordingPropertyIndexer {
+ fn indexed_count(&self) -> usize {
+ self.entities.lock().unwrap().len()
+ }
+
+ fn indexed_entities(&self) -> Vec<(String, EntityType)> {
+ self.entities.lock().unwrap().clone()
+ }
+}
+
+impl PropertyBackfillIndexer for RecordingPropertyIndexer {
+ async fn reindex(&self, entity_id: &str, entity_type: EntityType) -> Result<(), BackfillError> {
+ self.entities
+ .lock()
+ .unwrap()
+ .push((entity_id.to_owned(), entity_type));
+ Ok(())
+ }
+}
+
+fn property_page(
+ entity_type: EntityType,
+ entity_ids: impl IntoIterator<Item = impl Into<String>>,
+ rows_consumed: usize,
+) -> PropertySourcePage {
+ PropertySourcePage {
+ entity_ids: entity_ids.into_iter().map(Into::into).collect(),
+ entity_type,
+ rows_consumed,
+ }
+}
+
+#[tokio::test]
+async fn property_backfill_drains_multiple_pages() {
+ let entity_type = EntityType::Document;
+ let source = FakePropertySource::new(
+ entity_type,
+ vec![
+ property_page(entity_type, ["one", "two"], 2),
+ property_page(entity_type, ["three"], 1),
+ ],
+ );
+ let indexer = RecordingPropertyIndexer::default();
+ let (progress, cancel) = detached();
+
+ let receipt = drain_property_source(&indexer, &progress, &cancel, |offset| {
+ source.fetch_page(offset)
+ })
+ .await
+ .unwrap();
+
+ assert_eq!(receipt.enqueued, 3);
+ assert_eq!(
+ indexer.indexed_entities(),
+ vec![
+ ("one".to_owned(), EntityType::Document),
+ ("two".to_owned(), EntityType::Document),
+ ("three".to_owned(), EntityType::Document),
+ ]
+ );
+ assert_eq!(source.observed_offsets(), vec![0, 2, 3]);
+}
+
+#[tokio::test]
+async fn property_backfill_advances_offset_by_rows_consumed() {
+ let entity_type = EntityType::Thread;
+ let source = FakePropertySource::new(
+ entity_type,
+ vec![
+ property_page(entity_type, ["one", "two"], 100),
+ property_page(entity_type, ["three"], 40),
+ ],
+ );
+ let indexer = RecordingPropertyIndexer::default();
+ let (progress, cancel) = detached();
+
+ let receipt = drain_property_source(&indexer, &progress, &cancel, |offset| {
+ source.fetch_page(offset)
+ })
+ .await
+ .unwrap();
+
+ assert_eq!(receipt.enqueued, 140);
+ assert_eq!(indexer.indexed_count(), 3);
+ assert_eq!(source.observed_offsets(), vec![0, 100, 140]);
+}
+
+#[tokio::test]
+async fn property_backfill_updates_progress_after_successful_page() {
+ let entity_type = EntityType::Project;
+ let source = FakePropertySource::new(
+ entity_type,
+ vec![property_page(entity_type, ["one", "two"], 7)],
+ );
+ let indexer = RecordingPropertyIndexer::default();
+ let (progress, cancel) = detached();
+
+ let receipt = drain_property_source(&indexer, &progress, &cancel, |offset| {
+ source.fetch_page(offset)
+ })
+ .await
+ .unwrap();
+
+ assert_eq!(receipt.enqueued, 7);
+ assert_eq!(progress.local_count(), 7);
+}
+
+#[tokio::test]
+async fn property_backfill_empty_page_indexes_nothing() {
+ let entity_type = EntityType::Task;
+ let source = FakePropertySource::new(entity_type, Vec::new());
+ let indexer = RecordingPropertyIndexer::default();
+ let (progress, cancel) = detached();
+
+ let receipt = drain_property_source(&indexer, &progress, &cancel, |offset| {
+ source.fetch_page(offset)
+ })
+ .await
+ .unwrap();
+
+ assert_eq!(receipt.enqueued, 0);
+ assert_eq!(progress.local_count(), 0);
+ assert_eq!(indexer.indexed_count(), 0);
+ assert_eq!(source.observed_offsets(), vec![0]);
+}
+
+struct CancellingPropertyIndexer {
+ cancel: CancellationToken,
+ indexed: AtomicUsize,
+}
+
+impl PropertyBackfillIndexer for CancellingPropertyIndexer {
+ async fn reindex(
+ &self,
+ _entity_id: &str,
+ _entity_type: EntityType,
+ ) -> Result<(), BackfillError> {
+ self.indexed.fetch_add(1, Ordering::Relaxed);
+ self.cancel.cancel();
+ Ok(())
+ }
+}
+
+#[tokio::test]
+async fn property_backfill_cancellation_stops_between_pages() {
+ let entity_type = EntityType::Chat;
+ let source = FakePropertySource::new(
+ entity_type,
+ vec![
+ property_page(entity_type, ["indexed"], 1),
+ property_page(entity_type, ["not-indexed"], 1),
+ ],
+ );
+ let (progress, cancel) = detached();
+ let indexer = CancellingPropertyIndexer {
+ cancel: cancel.clone(),
+ indexed: AtomicUsize::new(0),
+ };
+
+ let receipt = drain_property_source(&indexer, &progress, &cancel, |offset| {
+ source.fetch_page(offset)
+ })
+ .await
+ .unwrap();
+
+ assert_eq!(receipt.enqueued, 1);
+ assert_eq!(progress.local_count(), 1);
+ assert_eq!(indexer.indexed.load(Ordering::Relaxed), 1);
+ assert_eq!(source.observed_offsets(), vec![0]);
+}
+
+struct FailingPropertyIndexer;
+
+impl PropertyBackfillIndexer for FailingPropertyIndexer {
+ async fn reindex(
+ &self,
+ _entity_id: &str,
+ _entity_type: EntityType,
+ ) -> Result<(), BackfillError> {
+ Err(BackfillError::Reindex(anyhow::anyhow!("reindex failed")))
+ }
+}
+
+#[tokio::test]
+async fn property_backfill_reindex_failure_propagates_without_progress() {
+ let entity_type = EntityType::Channel;
+ let source = FakePropertySource::new(
+ entity_type,
+ vec![property_page(entity_type, ["failure"], 1)],
+ );
+ let indexer = FailingPropertyIndexer;
+ let (progress, cancel) = detached();
+
+ let error = drain_property_source(&indexer, &progress, &cancel, |offset| {
+ source.fetch_page(offset)
+ })
+ .await
+ .unwrap_err();
+
+ assert!(matches!(error, BackfillError::Reindex(_)));
+ assert_eq!(progress.local_count(), 0);
+ assert_eq!(source.observed_offsets(), vec![0]);
+}
+
+#[derive(Default)]
+struct ConcurrencyTrackingPropertyIndexer {
+ active: AtomicUsize,
+ maximum_active: AtomicUsize,
+ completed: AtomicUsize,
+}
+
+impl PropertyBackfillIndexer for ConcurrencyTrackingPropertyIndexer {
+ async fn reindex(
+ &self,
+ _entity_id: &str,
+ _entity_type: EntityType,
+ ) -> Result<(), BackfillError> {
+ let active = self.active.fetch_add(1, Ordering::SeqCst) + 1;
+ self.maximum_active.fetch_max(active, Ordering::SeqCst);
+
+ tokio::time::sleep(Duration::from_millis(5)).await;
+
+ self.active.fetch_sub(1, Ordering::SeqCst);
+ self.completed.fetch_add(1, Ordering::SeqCst);
+ Ok(())
+ }
+}
+
+#[tokio::test]
+async fn property_backfill_limits_reindexes_to_twenty_simultaneous_operations() {
+ let entity_type = EntityType::Document;
+ let entity_ids = (0..45).map(|index| format!("entity-{index}"));
+ let source = FakePropertySource::new(
+ entity_type,
+ vec![property_page(entity_type, entity_ids, 45)],
+ );
+ let indexer = ConcurrencyTrackingPropertyIndexer::default();
+ let (progress, cancel) = detached();
+
+ let receipt = drain_property_source(&indexer, &progress, &cancel, |offset| {
+ source.fetch_page(offset)
+ })
+ .await
+ .unwrap();
+
+ assert_eq!(receipt.enqueued, 45);
+ assert_eq!(indexer.completed.load(Ordering::SeqCst), 45);
+ assert_eq!(indexer.maximum_active.load(Ordering::SeqCst), 20);
+}
diff --git a/services/search_processing_service/src/inbound/kafka_consumer.rs b/services/search_processing_service/src/inbound/kafka_consumer.rs
index a0f920a058..289d4e4439 100644
--- a/services/search_processing_service/src/inbound/kafka_consumer.rs
+++ b/services/search_processing_service/src/inbound/kafka_consumer.rs
@@ -7,8 +7,8 @@
//! offsets are already committed.
//!
//! Per-entity event mapping and processing live in the [`call`], [`channel`],
-//! [`chat`], [`document`], and [`project`] submodules; this module owns the poll
-//! loop, worker, retry policy, and commit semantics.
+//! [`chat`], [`document`], [`project`], and [`property`] submodules; this module
+//! owns the poll loop, worker, retry policy, and commit semantics.
#![allow(clippy::enum_variant_names)]
@@ -18,6 +18,7 @@ mod chat;
mod context;
mod document;
mod project;
+mod property;
#[cfg(test)]
mod test;
@@ -30,6 +31,7 @@ use documents::domain::events::DocumentMacroEvent;
use kafka_util::{GroupName, KafkaEventConsumer};
use macro_event_broker::{KafkaConsumerAdapter, MacroEventCollection, MacroEventConsumerService};
use projects::domain::events::ProjectMacroEvent;
+use properties::domain::events::PropertyMacroEvent;
use rdkafka::{
consumer::CommitMode,
message::{BorrowedMessage, Message as _},
@@ -41,6 +43,7 @@ use tokio_retry::{Retry, strategy::ExponentialBackoff};
use self::{
call::process_call_event, channel::process_channel_event, chat::process_chat_event,
document::process_document_event, project::process_project_event,
+ property::process_property_event,
};
pub(crate) use self::context::KafkaProcessingContext;
@@ -64,6 +67,7 @@ macro_event_broker::declare_topics!(
ChatMacroEvent,
DocumentMacroEvent,
ProjectMacroEvent,
+ PropertyMacroEvent,
);
/// Maximum number of decoded events waiting for the sequential worker.
@@ -189,6 +193,9 @@ async fn process_event(
DeclaredMacroEvent::ProjectMacroEvent(event) => {
process_project_event(db, opensearch_client, event, partition, offset).await
}
+ DeclaredMacroEvent::PropertyMacroEvent(event) => {
+ process_property_event(db, opensearch_client, event, partition, offset).await
+ }
}
}
diff --git a/services/search_processing_service/src/inbound/kafka_consumer/property.rs b/services/search_processing_service/src/inbound/kafka_consumer/property.rs
new file mode 100644
index 0000000000..6ed10061dd
--- /dev/null
+++ b/services/search_processing_service/src/inbound/kafka_consumer/property.rs
@@ -0,0 +1,143 @@
+//! Maps property lifecycle events to search-index actions and processes them.
+
+use macro_event_broker::MacroEvent as _;
+use models_properties::EntityType;
+use opensearch_client::OpensearchClient;
+use properties::domain::events::{PropertyMacroEvent, PropertyTopicEvent};
+use sqlx::PgPool;
+
+use super::{EventOutcome, MAX_PROCESSING_ATTEMPTS, PROCESSING_RETRY_BASE_DELAY, retry_processing};
+use crate::process::properties::process_entity_property_update;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(super) enum PropertyIndexAction<'a> {
+ Reindex {
+ entity_id: &'a str,
+ entity_type: EntityType,
+ },
+ Ignore,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(super) struct PropertyEventDescription<'a> {
+ pub(super) action: PropertyIndexAction<'a>,
+ pub(super) event_type: &'static str,
+}
+
+pub(super) fn describe_property_event(event: &PropertyTopicEvent) -> PropertyEventDescription<'_> {
+ match event {
+ PropertyTopicEvent::Created(_) => PropertyEventDescription {
+ action: PropertyIndexAction::Ignore,
+ event_type: "property.created",
+ },
+ PropertyTopicEvent::Deleted(_) => PropertyEventDescription {
+ action: PropertyIndexAction::Ignore,
+ event_type: "property.deleted",
+ },
+ PropertyTopicEvent::OptionCreated(_) => PropertyEventDescription {
+ action: PropertyIndexAction::Ignore,
+ event_type: "property_option.created",
+ },
+ PropertyTopicEvent::OptionUpdated(_) => PropertyEventDescription {
+ action: PropertyIndexAction::Ignore,
+ event_type: "property_option.updated",
+ },
+ PropertyTopicEvent::OptionDeleted(_) => PropertyEventDescription {
+ action: PropertyIndexAction::Ignore,
+ event_type: "property_option.deleted",
+ },
+ PropertyTopicEvent::EntityPropertyUpdated(metadata) => PropertyEventDescription {
+ action: PropertyIndexAction::Reindex {
+ entity_id: &metadata.entity_id,
+ entity_type: metadata.entity_type,
+ },
+ event_type: "entity_property.updated",
+ },
+ PropertyTopicEvent::EntityPropertyDeleted(metadata) => PropertyEventDescription {
+ action: PropertyIndexAction::Reindex {
+ entity_id: &metadata.entity_id,
+ entity_type: metadata.entity_type,
+ },
+ event_type: "entity_property.deleted",
+ },
+ PropertyTopicEvent::EntityPropertiesCleared(metadata) => PropertyEventDescription {
+ action: PropertyIndexAction::Reindex {
+ entity_id: &metadata.entity_id,
+ entity_type: metadata.entity_type,
+ },
+ event_type: "entity_properties.cleared",
+ },
+ }
+}
+
+pub(super) async fn process_property_event(
+ db: &PgPool,
+ opensearch_client: &OpensearchClient,
+ event: &PropertyMacroEvent,
+ partition: i32,
+ offset: i64,
+) -> EventOutcome {
+ let description = describe_property_event(&event.event().event);
+ let PropertyIndexAction::Reindex {
+ entity_id,
+ entity_type,
+ } = description.action
+ else {
+ tracing::trace!(
+ event_type = description.event_type,
+ partition,
+ offset,
+ "ignoring property event without a search-index action"
+ );
+ return EventOutcome::Ignored;
+ };
+
+ let result = retry_processing(|attempt| async move {
+ tracing::trace!(
+ entity_id,
+ entity_type = %entity_type,
+ event_type = description.event_type,
+ partition,
+ offset,
+ attempt,
+ "processing property search-index event"
+ );
+ process_entity_property_update(opensearch_client, db, entity_id, entity_type)
+ .await
+ .inspect_err(|error| {
+ if attempt < MAX_PROCESSING_ATTEMPTS {
+ let retry_delay =
+ PROCESSING_RETRY_BASE_DELAY * 2u32.pow(attempt.saturating_sub(1));
+ tracing::warn!(
+ error = ?error,
+ entity_id,
+ entity_type = %entity_type,
+ event_type = description.event_type,
+ partition,
+ offset,
+ attempt,
+ delay_secs = retry_delay.as_secs(),
+ "property search-index processing failed, retrying"
+ );
+ }
+ })
+ })
+ .await;
+
+ match result {
+ Ok(()) => EventOutcome::Indexed,
+ Err(error) => {
+ tracing::error!(
+ error = ?error,
+ entity_id,
+ entity_type = %entity_type,
+ event_type = description.event_type,
+ partition,
+ offset,
+ attempts = MAX_PROCESSING_ATTEMPTS,
+ "dropping property event after processing retries were exhausted"
+ );
+ EventOutcome::Dropped
+ }
+ }
+}
diff --git a/services/search_processing_service/src/inbound/kafka_consumer/test.rs b/services/search_processing_service/src/inbound/kafka_consumer/test.rs
index 54336ba4cd..cf8daa25e0 100644
--- a/services/search_processing_service/src/inbound/kafka_consumer/test.rs
+++ b/services/search_processing_service/src/inbound/kafka_consumer/test.rs
@@ -31,14 +31,22 @@ use documents::domain::events::{
use macro_event_broker::{Event, EventBrokerError, MacroEvent as _, MessageParts};
use macro_event_topics::{
MacroCallsTopic, MacroChannelsTopic, MacroChatsTopic, MacroDocumentsTopic, MacroProjectsTopic,
- Topic as _,
+ MacroPropertiesTopic, Topic as _,
};
use macro_user_id::user_id::MacroUserIdStr;
use model::document::FileType;
+use models_properties::{
+ DataType, EntityType, PropertyOwner, service::property_option::PropertyOptionValue,
+};
use projects::domain::events::{
ProjectCreatedMetadata, ProjectDeletedMetadata, ProjectPermanentlyDeletedMetadata,
ProjectRestoredMetadata, ProjectTopicEvent, ProjectUpdatedMetadata, ProjectUploadedMetadata,
};
+use properties::domain::events::{
+ EntityPropertiesClearedMetadata, EntityPropertyDeletedMetadata, EntityPropertyUpdatedMetadata,
+ PropertyCreatedMetadata, PropertyDeletedMetadata, PropertyOptionCreatedMetadata,
+ PropertyOptionDeletedMetadata, PropertyOptionUpdatedMetadata, PropertyTopicEvent,
+};
use uuid::Uuid;
use super::{
@@ -52,6 +60,7 @@ use super::{
project::{
ProjectEventDescription, ProjectIndexAction, collect_project_ids, describe_project_event,
},
+ property::{PropertyEventDescription, PropertyIndexAction, describe_property_event},
*,
};
@@ -67,6 +76,10 @@ const PROJECT_ID: &str = "project-root";
const CHILD_PROJECT_ID: &str = "project-child";
const PARENT_PROJECT_ID: &str = "project-parent";
const NEW_PARENT_PROJECT_ID: &str = "project-new-parent";
+const PROPERTY_ENTITY_ID: &str = "property-entity-id";
+const PROPERTY_DEFINITION_ID: Uuid = Uuid::from_u128(4);
+const PROPERTY_OPTION_ID: Uuid = Uuid::from_u128(5);
+const ENTITY_PROPERTY_ID: Uuid = Uuid::from_u128(6);
struct TestMessage {
topic: &'static str,
@@ -742,6 +755,130 @@ fn project_event_cases() -> Vec<(ProjectTopicEvent, ProjectEventDescription<'sta
]
}
+fn property_event_cases() -> Vec<(PropertyTopicEvent, PropertyEventDescription<'static>)> {
+ let actor_user_id = Some(user_id());
+
+ vec![
+ (
+ PropertyTopicEvent::Created(PropertyCreatedMetadata {
+ property_definition_id: PROPERTY_DEFINITION_ID,
+ actor_user_id: actor_user_id.clone(),
+ owner: PropertyOwner::System,
+ display_name: "Status".to_string(),
+ data_type: DataType::String,
+ is_multi_select: false,
+ specific_entity_type: None,
+ created_at: Utc::now(),
+ }),
+ PropertyEventDescription {
+ action: PropertyIndexAction::Ignore,
+ event_type: "property.created",
+ },
+ ),
+ (
+ PropertyTopicEvent::Deleted(PropertyDeletedMetadata {
+ property_definition_id: PROPERTY_DEFINITION_ID,
+ actor_user_id: actor_user_id.clone(),
+ owner: PropertyOwner::System,
+ display_name: "Status".to_string(),
+ data_type: DataType::String,
+ }),
+ PropertyEventDescription {
+ action: PropertyIndexAction::Ignore,
+ event_type: "property.deleted",
+ },
+ ),
+ (
+ PropertyTopicEvent::OptionCreated(PropertyOptionCreatedMetadata {
+ option_id: PROPERTY_OPTION_ID,
+ property_definition_id: PROPERTY_DEFINITION_ID,
+ actor_user_id: actor_user_id.clone(),
+ value: PropertyOptionValue::String("Open".to_string()),
+ color: None,
+ display_order: 0,
+ }),
+ PropertyEventDescription {
+ action: PropertyIndexAction::Ignore,
+ event_type: "property_option.created",
+ },
+ ),
+ (
+ PropertyTopicEvent::OptionUpdated(PropertyOptionUpdatedMetadata {
+ option_id: PROPERTY_OPTION_ID,
+ property_definition_id: PROPERTY_DEFINITION_ID,
+ actor_user_id: actor_user_id.clone(),
+ value: PropertyOptionValue::String("Done".to_string()),
+ color: Some("#ffffff".to_string()),
+ display_order: 1,
+ }),
+ PropertyEventDescription {
+ action: PropertyIndexAction::Ignore,
+ event_type: "property_option.updated",
+ },
+ ),
+ (
+ PropertyTopicEvent::OptionDeleted(PropertyOptionDeletedMetadata {
+ option_id: PROPERTY_OPTION_ID,
+ property_definition_id: PROPERTY_DEFINITION_ID,
+ actor_user_id: actor_user_id.clone(),
+ value: PropertyOptionValue::String("Done".to_string()),
+ }),
+ PropertyEventDescription {
+ action: PropertyIndexAction::Ignore,
+ event_type: "property_option.deleted",
+ },
+ ),
+ (
+ PropertyTopicEvent::EntityPropertyUpdated(EntityPropertyUpdatedMetadata {
+ entity_property_id: ENTITY_PROPERTY_ID,
+ entity_id: PROPERTY_ENTITY_ID.to_string(),
+ entity_type: EntityType::Document,
+ property_definition_id: PROPERTY_DEFINITION_ID,
+ actor_user_id: actor_user_id.clone(),
+ value: None,
+ updated_at: Utc::now(),
+ }),
+ PropertyEventDescription {
+ action: PropertyIndexAction::Reindex {
+ entity_id: PROPERTY_ENTITY_ID,
+ entity_type: EntityType::Document,
+ },
+ event_type: "entity_property.updated",
+ },
+ ),
+ (
+ PropertyTopicEvent::EntityPropertyDeleted(EntityPropertyDeletedMetadata {
+ entity_property_id: ENTITY_PROPERTY_ID,
+ entity_id: PROPERTY_ENTITY_ID.to_string(),
+ entity_type: EntityType::Chat,
+ property_definition_id: PROPERTY_DEFINITION_ID,
+ actor_user_id: actor_user_id.clone(),
+ }),
+ PropertyEventDescription {
+ action: PropertyIndexAction::Reindex {
+ entity_id: PROPERTY_ENTITY_ID,
+ entity_type: EntityType::Chat,
+ },
+ event_type: "entity_property.deleted",
+ },
+ ),
+ (
+ PropertyTopicEvent::EntityPropertiesCleared(EntityPropertiesClearedMetadata {
+ entity_id: PROPERTY_ENTITY_ID.to_string(),
+ entity_type: EntityType::Thread,
+ actor_user_id,
+ }),
+ PropertyEventDescription {
+ action: PropertyIndexAction::Reindex {
+ entity_id: PROPERTY_ENTITY_ID,
+ entity_type: EntityType::Thread,
+ },
+ event_type: "entity_properties.cleared",
+ },
+ ),
+ ]
+}
+
#[test]
fn subscribes_to_declared_search_processing_topics_with_durable_group() {
assert_eq!(
@@ -755,6 +892,7 @@ fn subscribes_to_declared_search_processing_topics_with_durable_group() {
assert!(topics.contains(&MacroChatsTopic::TOPIC_STR));
assert!(topics.contains(&MacroDocumentsTopic::TOPIC_STR));
assert!(topics.contains(&MacroProjectsTopic::TOPIC_STR));
+ assert!(topics.contains(&MacroPropertiesTopic::TOPIC_STR));
}
#[test]
@@ -914,6 +1052,18 @@ fn maps_all_project_lifecycle_events_to_reconciliation_actions() {
}
}
+#[test]
+fn maps_all_property_lifecycle_events_to_index_actions() {
+ let cases = property_event_cases();
+ assert_eq!(cases.len(), 8);
+
+ for (event, expected) in cases {
+ let serialized = serde_json::to_value(&event).expect("serializable property event");
+ assert_eq!(serialized["event_type"], expected.event_type);
+ assert_eq!(describe_property_event(&event), expected);
+ }
+}
+
#[test]
fn project_id_collection_is_stable_and_drops_missing_or_empty_ids() {
assert_eq!(
@@ -995,6 +1145,27 @@ fn project_envelope_decodes_round_trip_with_string_key() {
assert_eq!(decoded_event.event().event, event);
}
+#[test]
+fn property_envelope_decodes_round_trip_with_entity_key() {
+ let event = PropertyTopicEvent::EntityPropertiesCleared(EntityPropertiesClearedMetadata {
+ entity_id: PROPERTY_ENTITY_ID.to_string(),
+ entity_type: EntityType::Document,
+ actor_user_id: Some(user_id()),
+ });
+ let message = encoded_message(
+ MacroPropertiesTopic::TOPIC_STR,
+ PROPERTY_ENTITY_ID,
+ Event::new(event.clone()),
+ );
+
+ let decoded = DeclaredMacroEvent::decode(&message).expect("decodable property event");
+ let DeclaredMacroEvent::PropertyMacroEvent(decoded_event) = decoded else {
+ panic!("expected property event");
+ };
+ assert_eq!(decoded_event.key(), PROPERTY_ENTITY_ID);
+ assert_eq!(decoded_event.event().event, event);
+}
+
#[test]
fn exact_macro_documents_envelopes_decode_into_document_events() {
let cases: Vec<(&[u8], Event<DocumentTopicEvent>)> = vec![
@@ -1151,6 +1322,39 @@ async fn unsupported_project_schema_message_is_commit_safe() {
));
}
+#[tokio::test]
+async fn unsupported_property_schema_message_is_commit_safe() {
+ let event = PropertyTopicEvent::EntityPropertiesCleared(EntityPropertiesClearedMetadata {
+ entity_id: PROPERTY_ENTITY_ID.to_string(),
+ entity_type: EntityType::Document,
+ actor_user_id: Some(user_id()),
+ });
+ let message = encoded_message(
+ MacroPropertiesTopic::TOPIC_STR,
+ PROPERTY_ENTITY_ID,
+ Event::with_schema_version(event, 2),
+ );
+ let decoded = attach_event_coordinates(DeclaredMacroEvent::decode(&message), 6, 50);
+ let (sender, mut receiver) = mpsc::channel(1);
+
+ match handoff_decoded(&sender, decoded).await {
+ HandoffOutcome::MalformedRecord(EventBrokerError::UnsupportedSchemaVersion {
+ topic,
+ expected,
+ actual,
+ }) => {
+ assert_eq!(topic, MacroPropertiesTopic::TOPIC_STR);
+ assert_eq!(expected, 1);
+ assert_eq!(actual, 2);
+ }
+ outcome => panic!("expected malformed property record, got {outcome:?}"),
+ }
+ assert!(matches!(
+ receiver.try_recv(),
+ Err(mpsc::error::TryRecvError::Empty)
+ ));
+}
+
#[tokio::test]
async fn unsupported_channel_schema_message_is_commit_safe() {
let event = ChannelTopicEvent::Deleted(ChannelDeletedMetadata {
diff --git a/services/search_processing_service/src/main.rs b/services/search_processing_service/src/main.rs
index 5bdd81317f..c823171a9a 100644
--- a/services/search_processing_service/src/main.rs
+++ b/services/search_processing_service/src/main.rs
@@ -7,7 +7,10 @@ use crate::{
api::context::{ApiContext, AuthorizationService},
config::DatabaseUrlReadonly,
domain::{jobs::BackfillJobs, service::BackfillOrchestrator},
- outbound::{publisher::SqsSearchEventPublisher, source::PgBackfillSource},
+ outbound::{
+ property_search_indexer::DirectPropertyBackfillIndexer, publisher::SqsSearchEventPublisher,
+ source::PgBackfillSource,
+ },
process::{context::SearchProcessingContext, worker::run_search_processing_workers},
};
use anyhow::Context;
@@ -35,10 +38,11 @@ mod outbound;
mod parsers;
mod process;
-/// Concrete [`BackfillOrchestrator`] wired to the production Postgres source
-/// and the SQS publisher. Lives in the wiring module so the domain stays
-/// agnostic of which adapters back it.
-pub type BackfillServiceImpl = BackfillOrchestrator<PgBackfillSource, SqsSearchEventPublisher>;
+/// Concrete [`BackfillOrchestrator`] wired to the production Postgres source,
+/// SQS publisher, and direct property indexer. Lives in the wiring module so
+/// the domain stays agnostic of which adapters back it.
+pub type BackfillServiceImpl =
+ BackfillOrchestrator<PgBackfillSource, SqsSearchEventPublisher, DirectPropertyBackfillIndexer>;
/// Resolve a read-replica macrodb URL and
/// connect a small pool. Returns `None` when the replica URL is missing,
@@ -211,6 +215,7 @@ async fn main() -> anyhow::Result<()> {
let backfill_service = Arc::new(BackfillOrchestrator::new(
PgBackfillSource::new(backfill_db, config.backfill_page_sizes()?),
SqsSearchEventPublisher::new(sqs_client.clone()),
+ DirectPropertyBackfillIndexer::new(db.clone(), opensearch_client.clone()),
));
let shutdown_token = CancellationToken::new();
diff --git a/services/search_processing_service/src/outbound/mod.rs b/services/search_processing_service/src/outbound/mod.rs
index 7d27ca1d18..278cb05e09 100644
--- a/services/search_processing_service/src/outbound/mod.rs
+++ b/services/search_processing_service/src/outbound/mod.rs
@@ -1,2 +1,3 @@
+pub mod property_search_indexer;
pub mod publisher;
pub mod source;
diff --git a/services/search_processing_service/src/outbound/property_search_indexer.rs b/services/search_processing_service/src/outbound/property_search_indexer.rs
new file mode 100644
index 0000000000..c28707dd39
--- /dev/null
+++ b/services/search_processing_service/src/outbound/property_search_indexer.rs
@@ -0,0 +1,33 @@
+use std::sync::Arc;
+
+use models_properties::EntityType;
+use opensearch_client::OpensearchClient;
+use sqlx::PgPool;
+
+use crate::domain::models::BackfillError;
+use crate::domain::ports::PropertyBackfillIndexer;
+use crate::process::properties::process_entity_property_update;
+
+/// Direct property backfill adapter backed by the primary database and
+/// OpenSearch.
+pub struct DirectPropertyBackfillIndexer {
+ db: PgPool,
+ opensearch_client: Arc<OpensearchClient>,
+}
+
+impl DirectPropertyBackfillIndexer {
+ pub fn new(db: PgPool, opensearch_client: Arc<OpensearchClient>) -> Self {
+ Self {
+ db,
+ opensearch_client,
+ }
+ }
+}
+
+impl PropertyBackfillIndexer for DirectPropertyBackfillIndexer {
+ async fn reindex(&self, entity_id: &str, entity_type: EntityType) -> Result<(), BackfillError> {
+ process_entity_property_update(&self.opensearch_client, &self.db, entity_id, entity_type)
+ .await
+ .map_err(BackfillError::Reindex)
+ }
+}
diff --git a/services/search_processing_service/src/outbound/source.rs b/services/search_processing_service/src/outbound/source.rs
index 694906bbae..c2c77cab4b 100644
--- a/services/search_processing_service/src/outbound/source.rs
+++ b/services/search_processing_service/src/outbound/source.rs
@@ -6,7 +6,7 @@ use models_properties::EntityType;
use sqlx::PgPool;
use sqs_client::search::{
SearchQueueMessage, call::CallRecordMessage, channel::ChannelMessageUpdate, chat::ChatMessage,
- document::DocumentPropertiesUpdate, email::EmailThreadBatchMessage, project::UpsertProject,
+ email::EmailThreadBatchMessage, project::UpsertProject,
};
use crate::config::BackfillPageSizes;
@@ -14,15 +14,15 @@ use crate::domain::models::{
BackfillError, CallBackfillCursor, CallBackfillRequest, ChannelBackfillRequest,
ChatBackfillCursor, ChatBackfillRequest, DocumentBackfillCursor, DocumentBackfillRequest,
EmailBackfillRequest, ProjectBackfillCursor, ProjectBackfillRequest, PropertiesBackfillRequest,
- SourcePage,
+ PropertySourcePage, SourcePage,
};
use crate::domain::ports::BackfillSource;
const DEFAULT_EMAIL_BATCH_SIZE: usize = 50;
/// Page size for the properties backfill's distinct-entity-id scan. A fixed
-/// value rather than a config knob: property rows are few and one message is
-/// enqueued per entity.
+/// value rather than a config knob: property rows are few and each entity is
+/// reindexed directly.
const PROPERTIES_PAGE_SIZE: usize = 5000;
/// Postgres-backed [`BackfillSource`] for every search-indexed entity. One
@@ -339,7 +339,7 @@ impl BackfillSource for PgBackfillSource {
&self,
req: &PropertiesBackfillRequest,
offset: usize,
- ) -> Result<SourcePage, BackfillError> {
+ ) -> Result<PropertySourcePage, BackfillError> {
let entity_type = EntityType::from_str(&req.entity_type)
.map_err(|e| BackfillError::Source(anyhow::Error::new(e)))?;
@@ -354,18 +354,9 @@ impl BackfillSource for PgBackfillSource {
.map_err(BackfillError::Source)?;
let rows_consumed = entity_ids.len();
- let messages: Vec<SearchQueueMessage> = entity_ids
- .into_iter()
- .map(|entity_id| {
- SearchQueueMessage::UpdateDocumentProperties(DocumentPropertiesUpdate {
- document_id: entity_id,
- entity_type: entity_type.to_string(),
- })
- })
- .collect();
-
- Ok(SourcePage {
- messages,
+ Ok(PropertySourcePage {
+ entity_ids,
+ entity_type,
rows_consumed,
})
}
diff --git a/services/search_processing_service/src/process/mod.rs b/services/search_processing_service/src/process/mod.rs
index 4350fc1935..9e62159a80 100644
--- a/services/search_processing_service/src/process/mod.rs
+++ b/services/search_processing_service/src/process/mod.rs
@@ -5,7 +5,7 @@ pub mod context;
pub(crate) mod document;
mod email;
pub(crate) mod project;
-mod properties;
+pub(crate) mod properties;
mod user;
pub mod worker;
@@ -101,10 +101,6 @@ pub async fn process_message(
)
.await?;
}
- SearchQueueMessage::UpdateDocumentProperties(message) => {
- properties::process_entity_property_update(&ctx.opensearch_client, &ctx.db, &message)
- .await?;
- }
SearchQueueMessage::ChatMessage(message) => {
chat::insert_chat_message(&ctx.opensearch_client, &ctx.db, &message).await?;
}
diff --git a/services/search_processing_service/src/process/properties.rs b/services/search_processing_service/src/process/properties.rs
index 1299152570..b8c4baa32c 100644
--- a/services/search_processing_service/src/process/properties.rs
+++ b/services/search_processing_service/src/process/properties.rs
@@ -1,12 +1,9 @@
-use std::str::FromStr;
-
use anyhow::Context;
use models_properties::EntityType;
use opensearch_client::{OpensearchClient, upsert::properties::IndexedProperty};
use properties::outbound::entity_properties_get_query::{
IndexedEntityProperty, get_entity_properties_for_index,
};
-use sqs_client::search::document::DocumentPropertiesUpdate;
/// Refresh only the indexed `properties` of an entity after a property
/// mutation, without re-extracting its content. Fetches once, then routes to
@@ -15,12 +12,9 @@ use sqs_client::search::document::DocumentPropertiesUpdate;
pub async fn process_entity_property_update(
opensearch_client: &OpensearchClient,
db: &sqlx::Pool<sqlx::Postgres>,
- message: &DocumentPropertiesUpdate,
+ entity_id: &str,
+ entity_type: EntityType,
) -> anyhow::Result<()> {
- let entity_id = &message.document_id;
- let entity_type = EntityType::from_str(&message.entity_type)
- .with_context(|| format!("invalid entity_type '{}'", message.entity_type))?;
-
let rows = get_entity_properties_for_index(db, entity_id, entity_type)
.await
.context("failed to fetch properties for reindex")?;