Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(agent): index agents in search (#6376)

Eric Hayes39063404e6e0
diff --git a/.github/workspace-dep-closures.json b/.github/workspace-dep-closures.json
index c54d44c1f9..562e88453b 100644
--- a/.github/workspace-dep-closures.json
+++ b/.github/workspace-dep-closures.json
@@ -6889,12 +6889,14 @@
"crates/agent",
"crates/agent_fold",
"crates/agent_runtime_protocol",
+ "crates/agent_session",
"crates/ai_toolset",
"crates/ai_usage",
"crates/attachment",
"crates/authentication_service_client",
"crates/bot_id",
"crates/bot_token",
+ "crates/bots",
"crates/broadcast",
"crates/calendar_events",
"crates/call",
diff --git a/.sqlx/query-b23b0d4c6d6dd5782fd789eeab824fc417b8594a2d8d721fb2127f232e31c9b2.json b/.sqlx/query-b23b0d4c6d6dd5782fd789eeab824fc417b8594a2d8d721fb2127f232e31c9b2.json
new file mode 100644
index 0000000000..c8d375d07a
--- /dev/null
+++ b/.sqlx/query-b23b0d4c6d6dd5782fd789eeab824fc417b8594a2d8d721fb2127f232e31c9b2.json
@@ -0,0 +1,22 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "SELECT pg_advisory_xact_lock(hashtextextended('agent-session-search:' || $1::text, 0))",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "pg_advisory_xact_lock",
+ "type_info": "Void"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Text"
+ ]
+ },
+ "nullable": [
+ null
+ ]
+ },
+ "hash": "b23b0d4c6d6dd5782fd789eeab824fc417b8594a2d8d721fb2127f232e31c9b2"
+}
diff --git a/.sqlx/query-cda7bd1c8001e386048e00031ccc28b2517b4ee95134aeac2f66aeeedddcd7f7.json b/.sqlx/query-cda7bd1c8001e386048e00031ccc28b2517b4ee95134aeac2f66aeeedddcd7f7.json
new file mode 100644
index 0000000000..398f451b9f
--- /dev/null
+++ b/.sqlx/query-cda7bd1c8001e386048e00031ccc28b2517b4ee95134aeac2f66aeeedddcd7f7.json
@@ -0,0 +1,22 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "SELECT id FROM agent_session WHERE ($1::uuid IS NULL OR id > $1) ORDER BY id LIMIT 100",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "id",
+ "type_info": "Uuid"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Uuid"
+ ]
+ },
+ "nullable": [
+ false
+ ]
+ },
+ "hash": "cda7bd1c8001e386048e00031ccc28b2517b4ee95134aeac2f66aeeedddcd7f7"
+}
diff --git a/Cargo.lock b/Cargo.lock
index 9a7c664ced..34704c48ce 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -14466,6 +14466,8 @@ dependencies = [
name = "search_processing_service"
version = "0.1.0"
dependencies = [
+ "agent_fold",
+ "agent_session",
"anyhow",
"aws-sdk-dynamodb",
"aws-sdk-s3",
diff --git a/crates/agent_fold/src/domain/model.rs b/crates/agent_fold/src/domain/model.rs
index 1d407e61f3..8d07e22b43 100644
--- a/crates/agent_fold/src/domain/model.rs
+++ b/crates/agent_fold/src/domain/model.rs
@@ -30,6 +30,8 @@ mod part;
mod permission;
/// The agent's plan.
mod plan;
+/// Plain text derived from renderable messages for search.
+mod search;
/// Turn-level facts derived from the fold.
mod signal;
/// Delegated agents.
diff --git a/crates/agent_fold/src/domain/model/search.rs b/crates/agent_fold/src/domain/model/search.rs
new file mode 100644
index 0000000000..fd3d23c3a1
--- /dev/null
+++ b/crates/agent_fold/src/domain/model/search.rs
@@ -0,0 +1,335 @@
+//! Searchable text derived from the same vocabulary a transcript renders.
+//!
+//! This module deliberately starts from [`FoldedMessage`], never from ACP
+//! frames. The fold is the boundary that decides which protocol material is
+//! user-visible; search merely flattens that already-decided representation.
+
+use serde_json::Value;
+
+use super::{
+ AnsweredField, AnsweredValue, Control, ControlOutcome, ElicitationOutcome,
+ ElicitationPropertySchema, ElicitationRequest, ElicitationSchema, FoldedMessage, MessagePart,
+ PermissionOutcome, StopReason, SubagentResult, ToolDetail, UserToolOutcome,
+};
+
+#[cfg(test)]
+mod test;
+
+impl FoldedMessage {
+ /// Flatten the content a transcript exposes into text suitable for full-text search.
+ ///
+ /// Stable UI labels such as "Permission requested" are omitted because they
+ /// would make every message match the same boilerplate. Variable content—the
+ /// prose, paths, commands, outputs, questions, answers, and errors a person can
+ /// actually recognize—is retained in render order. Nested subagent parts are
+ /// traversed recursively.
+ #[must_use]
+ pub fn searchable_text(&self) -> String {
+ let mut text = SearchText::default();
+ for part in self.parts.iter() {
+ text.part(part);
+ }
+ if let Some(StopReason::Failed { message }) = &self.stop {
+ text.push(message);
+ }
+ text.finish()
+ }
+}
+
+#[derive(Default)]
+struct SearchText {
+ value: String,
+}
+
+impl SearchText {
+ fn finish(self) -> String {
+ self.value
+ }
+
+ fn push(&mut self, value: &str) {
+ if value.trim().is_empty() {
+ return;
+ }
+ if !self.value.is_empty() {
+ self.value.push('\n');
+ }
+ self.value.push_str(value);
+ }
+
+ fn json(&mut self, value: &Value) {
+ if let Ok(value) = serde_json::to_string_pretty(value) {
+ self.push(&value);
+ }
+ }
+
+ fn part(&mut self, part: &MessagePart) {
+ match part {
+ MessagePart::Text { text } | MessagePart::Thought { text } => self.push(text),
+ MessagePart::ToolUse { name, detail, .. } => {
+ self.push(name.display());
+ self.tool(detail);
+ }
+ MessagePart::Permission {
+ options, outcome, ..
+ } => {
+ if let PermissionOutcome::Selected { option_id } = outcome
+ && let Some(option) = options.iter().find(|option| option.id == *option_id)
+ {
+ self.push(&option.name);
+ }
+ }
+ MessagePart::Control { control, outcome } => {
+ if let Control::SetModel { model } = control {
+ self.push(model);
+ }
+ if let ControlOutcome::Rejected { message } = outcome {
+ self.push(message);
+ }
+ }
+ MessagePart::Plan { entries } => {
+ for entry in entries {
+ self.push(&entry.content);
+ }
+ }
+ MessagePart::Elicitation {
+ message,
+ request,
+ outcome,
+ reported,
+ tool_outcome,
+ ..
+ } => {
+ self.push(message);
+ self.elicitation_request(request);
+ self.elicitation_outcome(outcome);
+ if let Some(reported) = reported {
+ self.answers(reported);
+ }
+ if let Some(outcome) = tool_outcome {
+ self.user_tool_outcome(outcome);
+ }
+ }
+ }
+ }
+
+ fn tool(&mut self, detail: &ToolDetail) {
+ match detail {
+ ToolDetail::Terminal {
+ command, output, ..
+ } => {
+ if let Some(command) = command {
+ self.push(command);
+ }
+ if let Some(output) = output {
+ self.push(output.as_str());
+ }
+ }
+ ToolDetail::Edit { diffs } => {
+ for diff in diffs {
+ self.push(&diff.path.to_string_lossy());
+ if let Some(old_text) = &diff.old_text {
+ self.push(old_text);
+ }
+ self.push(&diff.new_text);
+ }
+ }
+ ToolDetail::Read { paths }
+ | ToolDetail::Delete { paths }
+ | ToolDetail::Move { paths } => {
+ for path in paths {
+ self.push(&path.to_string_lossy());
+ }
+ }
+ ToolDetail::Search { paths, output } => {
+ for path in paths {
+ self.push(&path.to_string_lossy());
+ }
+ if let Some(output) = output {
+ self.push(output);
+ }
+ }
+ ToolDetail::Fetch { output } | ToolDetail::Think { output } => {
+ if let Some(output) = output {
+ self.push(output);
+ }
+ }
+ ToolDetail::Other { input, output, .. } => {
+ if let Some(input) = input {
+ self.json(input);
+ }
+ if let Some(output) = output {
+ self.push(output);
+ }
+ }
+ ToolDetail::Macro {
+ input,
+ output,
+ error,
+ } => {
+ self.json(input);
+ if let Some(error) = error {
+ self.push(error);
+ } else if let Some(output) = output {
+ self.json(output);
+ }
+ }
+ ToolDetail::UserTool { input, outcome } => {
+ self.json(input);
+ self.user_tool_outcome(outcome);
+ }
+ ToolDetail::Subagent {
+ title,
+ agent_type,
+ description,
+ prompt,
+ children,
+ result,
+ ..
+ } => {
+ self.push(title);
+ if let Some(agent_type) = agent_type {
+ self.push(agent_type);
+ }
+ if let Some(description) = description {
+ self.push(description);
+ }
+ if let Some(prompt) = prompt {
+ self.push(prompt);
+ }
+ for child in children {
+ self.part(child);
+ }
+ if let Some(result) = result {
+ self.subagent_result(result);
+ }
+ }
+ }
+ }
+
+ fn subagent_result(&mut self, result: &SubagentResult) {
+ if let Some(text) = &result.text {
+ self.push(text);
+ }
+ if let Some(error) = &result.error {
+ self.push(error);
+ }
+ if let Some(model) = &result.model {
+ self.push(model);
+ }
+ }
+
+ fn elicitation_request(&mut self, request: &ElicitationRequest) {
+ match request {
+ ElicitationRequest::Form { schema } => self.elicitation_schema(schema),
+ ElicitationRequest::Url { url, .. } => self.push(url),
+ ElicitationRequest::UserTool {
+ tool,
+ draft,
+ schema,
+ } => {
+ self.push(tool);
+ self.json(draft);
+ self.elicitation_schema(schema);
+ }
+ ElicitationRequest::Unrecognized { mode, .. } => self.push(mode),
+ }
+ }
+
+ fn elicitation_schema(&mut self, schema: &ElicitationSchema) {
+ if let Some(title) = &schema.title {
+ self.push(title);
+ }
+ if let Some(description) = &schema.description {
+ self.push(description);
+ }
+ for property in &schema.properties {
+ if let Some(title) = &property.title {
+ self.push(title);
+ } else {
+ self.push(&property.name);
+ }
+ if let Some(description) = &property.description {
+ self.push(description);
+ }
+ match &property.schema {
+ ElicitationPropertySchema::String { options, .. }
+ | ElicitationPropertySchema::MultiSelect { options, .. } => {
+ for option in options {
+ self.push(option.title.as_deref().unwrap_or(&option.value));
+ if let Some(description) = &option.description {
+ self.push(description);
+ }
+ }
+ }
+ ElicitationPropertySchema::Unrecognized { type_name, .. } => {
+ self.push(type_name);
+ }
+ ElicitationPropertySchema::Number { .. }
+ | ElicitationPropertySchema::Integer { .. }
+ | ElicitationPropertySchema::Boolean { .. } => {}
+ }
+ }
+ }
+
+ fn elicitation_outcome(&mut self, outcome: &ElicitationOutcome) {
+ match outcome {
+ ElicitationOutcome::Accepted { answers } => self.answers(answers),
+ ElicitationOutcome::Errored { message } => self.push(message),
+ ElicitationOutcome::Pending
+ | ElicitationOutcome::Declined
+ | ElicitationOutcome::Cancelled
+ | ElicitationOutcome::Completed
+ | ElicitationOutcome::Unrecognized => {}
+ }
+ }
+
+ fn answers(&mut self, answers: &[AnsweredField]) {
+ for answer in answers {
+ self.push(&answer.label);
+ match &answer.value {
+ AnsweredValue::Text { text }
+ | AnsweredValue::Number { text }
+ | AnsweredValue::Custom { text } => self.push(text),
+ AnsweredValue::Boolean { checked } => {
+ self.push(if *checked { "Yes" } else { "No" });
+ }
+ AnsweredValue::Choice { choice } => {
+ self.push(choice.title.as_deref().unwrap_or(&choice.value));
+ }
+ AnsweredValue::Choices { choices } => {
+ for choice in choices {
+ self.push(choice.title.as_deref().unwrap_or(&choice.value));
+ }
+ }
+ AnsweredValue::Unrecognized { raw } => self.json(raw),
+ }
+ }
+ }
+
+ fn user_tool_outcome(&mut self, outcome: &UserToolOutcome) {
+ match outcome {
+ UserToolOutcome::Sent {
+ message_id,
+ thread_id,
+ } => {
+ self.push(message_id);
+ self.push(thread_id);
+ }
+ UserToolOutcome::Draft {
+ draft_id,
+ thread_id,
+ } => {
+ self.push(draft_id);
+ if let Some(thread_id) = thread_id {
+ self.push(thread_id);
+ }
+ }
+ UserToolOutcome::Completed { result } => self.json(result),
+ UserToolOutcome::Failed { message } => self.push(message),
+ UserToolOutcome::Pending
+ | UserToolOutcome::Edited
+ | UserToolOutcome::Rejected
+ | UserToolOutcome::Unrecognized => {}
+ }
+ }
+}
diff --git a/crates/agent_fold/src/domain/model/search/test.rs b/crates/agent_fold/src/domain/model/search/test.rs
new file mode 100644
index 0000000000..fa192624d4
--- /dev/null
+++ b/crates/agent_fold/src/domain/model/search/test.rs
@@ -0,0 +1,124 @@
+use std::path::PathBuf;
+
+use non_empty::NonEmpty;
+use serde_json::json;
+
+use super::*;
+use crate::domain::model::{
+ Author, FileDiff, PlanEntry, PlanEntryPriority, PlanEntryStatus, ToolName, ToolStatus,
+ ToolUseId, TurnId,
+};
+
+fn message(parts: Vec<MessagePart>) -> FoldedMessage {
+ FoldedMessage {
+ id: TurnId(4),
+ author: Author::Agent,
+ request_id: None,
+ parts: NonEmpty::new(parts).expect("test message has parts"),
+ stop: None,
+ }
+}
+
+#[test]
+fn searchable_text_follows_render_order_and_nested_parts() {
+ let projected = message(vec![
+ MessagePart::Text {
+ text: "I found the issue.".to_owned(),
+ },
+ MessagePart::Plan {
+ entries: vec![PlanEntry {
+ content: "Repair the session index".to_owned(),
+ priority: PlanEntryPriority::High,
+ status: PlanEntryStatus::InProgress,
+ }],
+ },
+ MessagePart::ToolUse {
+ id: ToolUseId("tool-1".to_owned()),
+ name: ToolName::native("Agent"),
+ status: ToolStatus::Completed,
+ detail: ToolDetail::Subagent {
+ title: "Inspect indexing".to_owned(),
+ agent_type: Some("explore".to_owned()),
+ description: None,
+ prompt: Some("Trace the authoritative data path".to_owned()),
+ background: false,
+ children: vec![MessagePart::ToolUse {
+ id: ToolUseId("tool-2".to_owned()),
+ name: ToolName::native("Edit"),
+ status: ToolStatus::Completed,
+ detail: ToolDetail::Edit {
+ diffs: vec![FileDiff {
+ path: PathBuf::from("src/search.rs"),
+ old_text: Some("raw ACP".to_owned()),
+ new_text: "folded message".to_owned(),
+ }],
+ },
+ }],
+ result: Some(Box::new(SubagentResult {
+ text: Some("The fold is canonical.".to_owned()),
+ model: Some("test-model".to_owned()),
+ ..SubagentResult::default()
+ })),
+ },
+ },
+ ])
+ .searchable_text();
+
+ assert_eq!(
+ projected,
+ [
+ "I found the issue.",
+ "Repair the session index",
+ "Agent",
+ "Inspect indexing",
+ "explore",
+ "Trace the authoritative data path",
+ "Edit",
+ "src/search.rs",
+ "raw ACP",
+ "folded message",
+ "The fold is canonical.",
+ "test-model",
+ ]
+ .join("\n")
+ );
+}
+
+#[test]
+fn searchable_text_includes_folded_json_and_visible_failure() {
+ let mut message = message(vec![MessagePart::ToolUse {
+ id: ToolUseId("tool-1".to_owned()),
+ name: ToolName::native("CreateDocument"),
+ status: ToolStatus::Failed,
+ detail: ToolDetail::Macro {
+ input: json!({ "name": "Search notes" }),
+ output: None,
+ error: Some("document creation failed".to_owned()),
+ },
+ }]);
+ message.stop = Some(StopReason::Failed {
+ message: "runtime disconnected".to_owned(),
+ });
+
+ assert_eq!(
+ message.searchable_text(),
+ [
+ "CreateDocument",
+ "{\n \"name\": \"Search notes\"\n}",
+ "document creation failed",
+ "runtime disconnected",
+ ]
+ .join("\n")
+ );
+}
+
+#[test]
+fn searchable_text_omits_non_semantic_permission_boilerplate() {
+ let message = message(vec![MessagePart::Permission {
+ tool_call: ToolUseId("tool-1".to_owned()),
+ options: Vec::new(),
+ outcome: PermissionOutcome::Pending,
+ }]);
+
+ assert_eq!(message.searchable_text(), "");
+}
diff --git a/crates/agent_session/src/domain/search.rs b/crates/agent_session/src/domain/search.rs
index e7c6ca9a6e..366e0229f9 100644
--- a/crates/agent_session/src/domain/search.rs
+++ b/crates/agent_session/src/domain/search.rs
@@ -1,5 +1,7 @@
//! Metadata needed to present a searchable session without loading its ACP log.
+pub mod indexing;
+
use std::{future::Future, pin::Pin};
use chrono::{DateTime, Utc};
diff --git a/crates/agent_session/src/domain/search/indexing.rs b/crates/agent_session/src/domain/search/indexing.rs
new file mode 100644
index 0000000000..adaa74b09f
--- /dev/null
+++ b/crates/agent_session/src/domain/search/indexing.rs
@@ -0,0 +1,80 @@
+//! Internal search projection reads. ACP remains the source of every message.
+
+use agent_fold::domain::{model::FoldedMessage, ports::FoldedMessageRepo};
+use macro_uuid::Uuid;
+use rootcause::Report;
+
+use super::{AgentSessionSearchMetadata, AgentSessionSearchMetadataRepo};
+use crate::domain::model::AgentSessionId;
+
+/// Persistence capabilities used only by the trusted search indexer.
+pub trait SearchIndexingRepo: AgentSessionSearchMetadataRepo {
+ /// A cross-process lease released when dropped.
+ type Lease: Send;
+
+ /// Serialize a session's live indexing and backfill across all workers.
+ fn lock(&self, id: AgentSessionId) -> impl Future<Output = Result<Self::Lease, Report>> + Send;
+
+ /// Walk existing sessions by UUID, returning at most 100 IDs.
+ fn page(&self, after: Option<Uuid>) -> impl Future<Output = Result<Vec<Uuid>, Report>> + Send;
+}
+
+/// A session's current metadata and messages derived from its effective ACP log.
+pub struct SearchSnapshot {
+ /// Current database metadata; missing sessions have no snapshot.
+ pub metadata: AgentSessionSearchMetadata,
+ /// The same folded messages that the transcript renders.
+ pub messages: Vec<FoldedMessage>,
+}
+
+/// Domain API for trusted background indexing, separate from user-facing reads.
+pub trait SearchSnapshotService: Send + Sync + 'static {
+ /// Lease kept alive until the index operation finishes.
+ type Lease: Send;
+ /// Acquire the session's indexing lease before loading its current snapshot.
+ fn lock(&self, id: AgentSessionId) -> impl Future<Output = Result<Self::Lease, Report>> + Send;
+ /// Missing/deleted sessions return `None`; storage errors remain errors.
+ fn snapshot(
+ &self,
+ id: AgentSessionId,
+ ) -> impl Future<Output = Result<Option<SearchSnapshot>, Report>> + Send;
+ /// Enumerate a bounded page for a full repair/backfill.
+ fn page(&self, after: Option<Uuid>) -> impl Future<Output = Result<Vec<Uuid>, Report>> + Send;
+}
+
+/// Composes the owning repository and fold service without exposing storage to callers.
+pub struct SearchSnapshotServiceImpl<R, F> {
+ repo: R,
+ folds: F,
+}
+
+impl<R, F> SearchSnapshotServiceImpl<R, F> {
+ /// Assemble the metadata and folded-message capabilities.
+ pub fn new(repo: R, folds: F) -> Self {
+ Self { repo, folds }
+ }
+}
+
+impl<R, F> SearchSnapshotService for SearchSnapshotServiceImpl<R, F>
+where
+ R: SearchIndexingRepo,
+ F: FoldedMessageRepo + Send + Sync + 'static,
+{
+ type Lease = R::Lease;
+
+ async fn lock(&self, id: AgentSessionId) -> Result<Self::Lease, Report> {
+ self.repo.lock(id).await
+ }
+
+ async fn snapshot(&self, id: AgentSessionId) -> Result<Option<SearchSnapshot>, Report> {
+ let Some(metadata) = self.repo.search_metadata(&[id.as_uuid()]).await?.pop() else {
+ return Ok(None);
+ };
+ let messages = self.folds.messages(id).await?;
+ Ok(Some(SearchSnapshot { metadata, messages }))
+ }
+
+ async fn page(&self, after: Option<Uuid>) -> Result<Vec<Uuid>, Report> {
+ self.repo.page(after).await
+ }
+}
diff --git a/crates/agent_session/src/outbound/postgres/search.rs b/crates/agent_session/src/outbound/postgres/search.rs
index c614a60694..70a77223f1 100644
--- a/crates/agent_session/src/outbound/postgres/search.rs
+++ b/crates/agent_session/src/outbound/postgres/search.rs
@@ -8,6 +8,55 @@ use crate::{
outbound::postgres::PgAgentSessionRepo,
};
+/// Indexing reads use the primary pool; leases use a separate pool so waiting
+/// index workers cannot consume the connections needed to read their snapshots.
+pub struct PgSearchIndexingRepo {
+ pool: sqlx::PgPool,
+ lock_pool: sqlx::PgPool,
+}
+
+impl PgSearchIndexingRepo {
+ /// The two pools must be independent and connect to the same primary DB.
+ pub fn new(pool: sqlx::PgPool, lock_pool: sqlx::PgPool) -> Self {
+ Self { pool, lock_pool }
+ }
+}
+
+impl AgentSessionSearchMetadataRepo for PgSearchIndexingRepo {
+ async fn search_metadata(&self, ids: &[uuid::Uuid]) -> Result<Vec<AgentSessionSearchMetadata>> {
+ Ok(search_metadata(&self.pool, ids)
+ .await
+ .map_err(anyhow::Error::from)?)
+ }
+}
+
+impl crate::domain::search::indexing::SearchIndexingRepo for PgSearchIndexingRepo {
+ type Lease = sqlx::Transaction<'static, sqlx::Postgres>;
+
+ async fn lock(
+ &self,
+ id: crate::domain::model::AgentSessionId,
+ ) -> Result<Self::Lease, rootcause::Report> {
+ let mut transaction = self.lock_pool.begin().await?;
+ sqlx::query!(
+ "SELECT pg_advisory_xact_lock(hashtextextended('agent-session-search:' || $1::text, 0))",
+ id.to_string(),
+ )
+ .execute(&mut *transaction)
+ .await?;
+ Ok(transaction)
+ }
+
+ async fn page(&self, after: Option<uuid::Uuid>) -> Result<Vec<uuid::Uuid>, rootcause::Report> {
+ Ok(sqlx::query_scalar!(
+ "SELECT id FROM agent_session WHERE ($1::uuid IS NULL OR id > $1) ORDER BY id LIMIT 100",
+ after,
+ )
+ .fetch_all(&self.pool)
+ .await?)
+ }
+}
+
/// Fetch current metadata for already authorized session IDs.
#[tracing::instrument(err, skip(pool, ids))]
async fn search_metadata(
diff --git a/crates/agent_session/src/outbound/postgres/test/search.rs b/crates/agent_session/src/outbound/postgres/test/search.rs
index e7c1901f53..6a42b58428 100644
--- a/crates/agent_session/src/outbound/postgres/test/search.rs
+++ b/crates/agent_session/src/outbound/postgres/test/search.rs
@@ -3,6 +3,105 @@ use crate::domain::search::{
AgentSessionSearchMetadataService as _, AgentSessionSearchMetadataServiceImpl,
};
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn search_snapshot_folds_persisted_acp_instead_of_indexing_protocol_frames(pool: PgPool) {
+ use crate::domain::search::indexing::{SearchSnapshotService, SearchSnapshotServiceImpl};
+ use crate::outbound::postgres::search::PgSearchIndexingRepo;
+ use agent_fold::domain::{model::Author, service::FoldedMessageService};
+ use serde_json::json;
+
+ let repo = PgAgentSessionRepo::new(pool.clone());
+ let bot = create_test_bot(&pool).await;
+ let session = create_session(&repo, new_session(bot, None, None)).await;
+ for content in [
+ json!({"direction":"to_runtime", "content": {"type":"acp", "jsonrpc":"2.0", "id":"prompt-1", "method":"session/prompt", "params": {
+ "sessionId":"hidden-protocol-session", "prompt":[{"type":"text", "text":"find cobalt"}]
+ }}}),
+ json!({"direction":"to_server", "content": {"type":"acp", "jsonrpc":"2.0", "method":"session/update", "params": {
+ "sessionId":"hidden-protocol-session", "update":{"sessionUpdate":"agent_message_chunk", "content":{"type":"text", "text":"found "}}
+ }}}),
+ json!({"direction":"to_server", "content": {"type":"acp", "jsonrpc":"2.0", "method":"session/update", "params": {
+ "sessionId":"hidden-protocol-session", "update":{"sessionUpdate":"agent_message_chunk", "content":{"type":"text", "text":"quartz"}}
+ }}}),
+ json!({"direction":"to_server", "content": {"type":"acp", "jsonrpc":"2.0", "id":"prompt-1", "result":{"stopReason":"end_turn"}}}),
+ ] {
+ AgentSessionLogRepo::create(
+ &repo,
+ AgentSessionLog {
+ agent_session_id: session.id,
+ user_id: Some(user_id(OWNER)),
+ content: serde_json::from_value(content).unwrap(),
+ },
+ )
+ .await
+ .unwrap();
+ }
+ let locks = sqlx::postgres::PgPoolOptions::new()
+ .max_connections(1)
+ .connect_lazy_with((*pool.connect_options()).clone());
+ let service = SearchSnapshotServiceImpl::new(
+ PgSearchIndexingRepo::new(pool, locks),
+ FoldedMessageService::new(repo.clone()),
+ );
+ let snapshot = service.snapshot(session.id).await.unwrap().unwrap();
+ assert_eq!(snapshot.metadata.id, session.id.as_uuid());
+ assert_eq!(snapshot.messages.len(), 2);
+ assert!(matches!(snapshot.messages[0].author, Author::User { .. }));
+ assert_eq!(snapshot.messages[0].searchable_text(), "find cobalt");
+ assert!(matches!(snapshot.messages[1].author, Author::Agent));
+ assert_eq!(snapshot.messages[1].searchable_text(), "found quartz");
+
+ AgentSessionRepo::delete(&repo, session.id).await.unwrap();
+ assert!(service.snapshot(session.id).await.unwrap().is_none());
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn indexing_pages_current_sessions_and_serializes_snapshot_leases(pool: PgPool) {
+ use crate::domain::search::indexing::SearchIndexingRepo;
+ use crate::outbound::postgres::search::PgSearchIndexingRepo;
+ use sqlx::postgres::PgPoolOptions;
+ use std::time::Duration;
+
+ let repo = PgAgentSessionRepo::new(pool.clone());
+ let bot = create_test_bot(&pool).await;
+ let first = create_session(&repo, new_session(bot, None, None)).await;
+ let second = create_session(&repo, new_session(bot, None, None)).await;
+ let lock_pool = PgPoolOptions::new()
+ .max_connections(3)
+ .connect_lazy_with((*pool.connect_options()).clone());
+ let indexing = PgSearchIndexingRepo::new(pool, lock_pool);
+ let mut ids = vec![first.id.as_uuid(), second.id.as_uuid()];
+ ids.sort();
+ assert_eq!(indexing.page(None).await.unwrap(), ids);
+ assert_eq!(indexing.page(Some(ids[0])).await.unwrap(), vec![ids[1]]);
+ assert!(indexing.page(Some(ids[1])).await.unwrap().is_empty());
+
+ let lease = indexing.lock(first.id).await.unwrap();
+ let mut waiting = Box::pin(indexing.lock(first.id));
+ assert!(
+ tokio::time::timeout(Duration::from_millis(100), &mut waiting)
+ .await
+ .is_err()
+ );
+ // A different session is independent, even while the first lease is held.
+ let other = tokio::time::timeout(Duration::from_secs(5), indexing.lock(second.id))
+ .await
+ .unwrap()
+ .unwrap();
+ drop(other);
+ drop(lease);
+ let _lease = tokio::time::timeout(Duration::from_secs(5), waiting)
+ .await
+ .unwrap()
+ .unwrap();
+
+ AgentSessionRepo::delete(&repo, first.id).await.unwrap();
+ assert_eq!(
+ indexing.page(None).await.unwrap(),
+ vec![second.id.as_uuid()]
+ );
+}
+
#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
async fn search_metadata_is_scoped_current_and_omits_deleted_sessions(pool: PgPool) {
let repo = PgAgentSessionRepo::new(pool.clone());
diff --git a/crates/agent_session/src/testing.rs b/crates/agent_session/src/testing.rs
index 227545a1c6..e222605d6a 100644
--- a/crates/agent_session/src/testing.rs
+++ b/crates/agent_session/src/testing.rs
@@ -93,21 +93,25 @@ impl InMemoryAgentSessionRepo {
/// Seed log entries, in the order they should be read back.
///
- /// Each is stamped as it lands, the way the real table's `created_at`
- /// default does.
+ /// Give seeded frames strictly increasing timestamps: an in-memory loop
+ /// can outrun the clock, and UUIDv7 suffixes do not preserve insertion order
+ /// when the production `(created_at, id)` reader breaks timestamp ties.
pub fn extend_log(&self, entries: impl IntoIterator<Item = AgentSessionLog>) {
let mut logs = self
.logs
.lock()
.expect("in-memory log store is not poisoned");
for entry in entries {
- logs.entry(entry.agent_session_id)
- .or_default()
- .push(StoredAgentSessionLog {
- id: macro_uuid::generate_uuid_v7(),
- created_at: chrono::Utc::now(),
- entry,
- });
+ let rows = logs.entry(entry.agent_session_id).or_default();
+ let now = chrono::Utc::now();
+ let created_at = rows.last().map_or(now, |last| {
+ now.max(last.created_at + chrono::Duration::microseconds(1))
+ });
+ rows.push(StoredAgentSessionLog {
+ id: macro_uuid::generate_uuid_v7(),
+ created_at,
+ entry,
+ });
}
}
}
diff --git a/crates/agent_session/src/testing/test.rs b/crates/agent_session/src/testing/test.rs
index 578d40b52f..58b3dee322 100644
--- a/crates/agent_session/src/testing/test.rs
+++ b/crates/agent_session/src/testing/test.rs
@@ -2,6 +2,36 @@ use super::*;
use crate::domain::model::Message;
use agent_runtime_protocol::domain::schema::v0::SystemEvent;
+#[test]
+fn seeded_frames_remain_ordered_when_the_clock_has_not_advanced() {
+ let repo = InMemoryAgentSessionRepo::new();
+ let session = AgentSessionId::new();
+ let entry = AgentSessionLog {
+ agent_session_id: session,
+ user_id: None,
+ content: Message::ToServer(ToServerMessage::Event {
+ event: SystemEvent::AcpReady,
+ }),
+ };
+ // Put the existing seed ahead of the wall clock to exercise the tie/backward
+ // clock path deterministically, rather than hoping two calls share a tick.
+ repo.logs.lock().unwrap().insert(
+ session,
+ vec![StoredAgentSessionLog {
+ id: Uuid::from_u128(1),
+ created_at: chrono::Utc::now() + chrono::Duration::seconds(60),
+ entry: entry.clone(),
+ }],
+ );
+ repo.extend_log([entry.clone(), entry]);
+ let rows = repo.logs.lock().unwrap();
+ assert!(
+ rows[&session]
+ .windows(2)
+ .all(|pair| pair[0].created_at < pair[1].created_at)
+ );
+}
+
#[tokio::test]
async fn effective_history_orders_timestamps_and_uuid_ties_before_selecting_boundary() {
let repo = InMemoryAgentSessionRepo::new();
diff --git a/crates/opensearch_client/src/agent_session.rs b/crates/opensearch_client/src/agent_session.rs
new file mode 100644
index 0000000000..672d61dc6c
--- /dev/null
+++ b/crates/opensearch_client/src/agent_session.rs
@@ -0,0 +1,63 @@
+use crate::{OpensearchClient, Result, delete, upsert};
+
+pub use upsert::agent_session::{
+ AgentSessionMessageAuthor, AgentSessionMessageDocument, ReconcileAgentSessionArgs,
+};
+
+/// Delete-by-query may report partial failure in an otherwise successful HTTP response.
+pub(crate) async fn check_delete_response(
+ response: opensearch::http::response::Response,
+ method: &str,
+) -> Result<()> {
+ let status = response.status_code();
+ let body: serde_json::Value = response.json().await.map_err(|error| {
+ crate::error::OpensearchClientError::DeserializationFailed {
+ details: error.to_string(),
+ method: Some(method.to_owned()),
+ }
+ })?;
+ if !status.is_success() || delete_failed(&body) {
+ return Err(crate::error::OpensearchClientError::Unknown {
+ details: body.to_string(),
+ method: Some(method.to_owned()),
+ });
+ }
+ Ok(())
+}
+
+fn delete_failed(body: &serde_json::Value) -> bool {
+ body["timed_out"].as_bool() == Some(true)
+ || body["version_conflicts"]
+ .as_u64()
+ .is_some_and(|count| count > 0)
+ || body["failures"]
+ .as_array()
+ .is_some_and(|items| !items.is_empty())
+}
+
+#[cfg(test)]
+mod test;
+
+impl OpensearchClient {
+ /// Replace the searchable projection of one agent session with an
+ /// authoritative folded snapshot.
+ #[tracing::instrument(skip(self, args), err)]
+ pub async fn reconcile_agent_session(
+ &self,
+ args: &ReconcileAgentSessionArgs,
+ index_override: Option<&str>,
+ ) -> Result<()> {
+ upsert::agent_session::reconcile_agent_session(&self.inner, args, index_override).await
+ }
+
+ /// Delete the parent and every folded-message child for an agent session.
+ #[tracing::instrument(skip(self), err)]
+ pub async fn delete_agent_session(
+ &self,
+ agent_session_id: &str,
+ index_override: Option<&str>,
+ ) -> Result<()> {
+ delete::agent_session::delete_agent_session(&self.inner, agent_session_id, index_override)
+ .await
+ }
+}
diff --git a/crates/opensearch_client/src/agent_session/test.rs b/crates/opensearch_client/src/agent_session/test.rs
new file mode 100644
index 0000000000..115c7e8387
--- /dev/null
+++ b/crates/opensearch_client/src/agent_session/test.rs
@@ -0,0 +1,14 @@
+use super::delete_failed;
+use serde_json::json;
+
+#[test]
+fn partial_deletions_are_errors_even_with_http_success() {
+ assert!(!delete_failed(
+ &json!({"timed_out": false, "version_conflicts": 0, "failures": []})
+ ));
+ assert!(delete_failed(&json!({"timed_out": true})));
+ assert!(delete_failed(&json!({"version_conflicts": 1})));
+ assert!(delete_failed(
+ &json!({"failures": [{"reason": "shard unavailable"}]})
+ ));
+}
diff --git a/crates/opensearch_client/src/delete/agent_session.rs b/crates/opensearch_client/src/delete/agent_session.rs
new file mode 100644
index 0000000000..bdf1e858ed
--- /dev/null
+++ b/crates/opensearch_client/src/delete/agent_session.rs
@@ -0,0 +1,28 @@
+use models_opensearch::SearchIndex;
+
+use crate::{Result, error::OpensearchClientError};
+
+/// Remove a session parent and all of its message children.
+pub(crate) async fn delete_agent_session(
+ client: &opensearch::OpenSearch,
+ agent_session_id: &str,
+ index_override: Option<&str>,
+) -> Result<()> {
+ let index = index_override.unwrap_or(SearchIndex::AgentSessions.as_ref());
+ let routing = [agent_session_id];
+ let response = client
+ .delete_by_query(opensearch::DeleteByQueryParts::Index(&[index]))
+ .routing(&routing)
+ .body(serde_json::json!({
+ "query": { "term": { "agent_session_id": agent_session_id } }
+ }))
+ .refresh(true)
+ .send()
+ .await
+ .map_err(|error| OpensearchClientError::Unknown {
+ details: error.to_string(),
+ method: Some("delete_agent_session".to_owned()),
+ })?;
+
+ crate::agent_session::check_delete_response(response, "delete_agent_session").await
+}
diff --git a/crates/opensearch_client/src/delete/mod.rs b/crates/opensearch_client/src/delete/mod.rs
index 3635d4ae88..5cdc151b3d 100644
--- a/crates/opensearch_client/src/delete/mod.rs
+++ b/crates/opensearch_client/src/delete/mod.rs
@@ -1,3 +1,4 @@
+pub mod agent_session;
pub mod calendar_event;
pub mod call_record;
pub mod channel;
diff --git a/crates/opensearch_client/src/lib.rs b/crates/opensearch_client/src/lib.rs
index e39fc543dd..1eee949c79 100644
--- a/crates/opensearch_client/src/lib.rs
+++ b/crates/opensearch_client/src/lib.rs
@@ -5,6 +5,7 @@ pub mod search;
pub mod unified;
pub mod upsert;
+pub mod agent_session;
pub mod calendar_event;
pub mod call_record;
pub mod channel_message;
diff --git a/crates/opensearch_client/src/upsert.rs b/crates/opensearch_client/src/upsert.rs
index ef561bc25c..274be235c1 100644
--- a/crates/opensearch_client/src/upsert.rs
+++ b/crates/opensearch_client/src/upsert.rs
@@ -1,3 +1,4 @@
+pub mod agent_session;
pub mod calendar_event;
pub mod call_record;
pub mod channel_message;
diff --git a/crates/opensearch_client/src/upsert/agent_session.rs b/crates/opensearch_client/src/upsert/agent_session.rs
new file mode 100644
index 0000000000..a1f53bd669
--- /dev/null
+++ b/crates/opensearch_client/src/upsert/agent_session.rs
@@ -0,0 +1,246 @@
+use models_opensearch::SearchIndex;
+use opensearch::BulkParts;
+use serde::Serialize;
+
+use crate::{Result, date_format::EpochMillis, error::OpensearchClientError};
+
+#[cfg(test)]
+mod test;
+
+const PARENT_RELATION: &str = "agent_session";
+const CHILD_RELATION: &str = "message";
+
+/// The two author sides of a folded conversation.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, strum::Display)]
+#[serde(rename_all = "lowercase")]
+#[strum(serialize_all = "lowercase")]
+pub enum AgentSessionMessageAuthor {
+ /// A user's prompt.
+ User,
+ /// The agent's reply.
+ Agent,
+}
+
+/// One folded message in an authoritative agent-session projection.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct AgentSessionMessageDocument {
+ /// Stable turn number assigned by `agent_fold`.
+ pub turn: u32,
+ /// Stable author side (`user` or `agent`).
+ pub author: AgentSessionMessageAuthor,
+ /// Attributed user for a user-authored prompt, when known.
+ pub author_user_id: Option<String>,
+ /// Text flattened from the folded, renderable message vocabulary.
+ pub content: String,
+}
+
+/// Complete searchable projection of one agent session.
+#[derive(Debug, Clone)]
+pub struct ReconcileAgentSessionArgs {
+ /// Agent-session UUID.
+ pub agent_session_id: String,
+ /// User-facing session name.
+ pub name: String,
+ /// Owner recorded by the authoritative session row.
+ pub owner_id: String,
+ /// Agent persona behind the session.
+ pub bot_id: String,
+ /// Originating thread, when the session was opened from one.
+ pub thread_id: Option<String>,
+ /// Message that opened the session, when any.
+ pub originating_message_id: Option<String>,
+ /// Session creation time.
+ pub created_at_millis: EpochMillis,
+ /// Latest persisted session metadata time.
+ pub updated_at_millis: EpochMillis,
+ /// Opaque identity of this complete projection. Every document written by
+ /// one reconcile carries it so obsolete children can be pruned.
+ pub projection_generation: String,
+ /// Every currently folded message, oldest first.
+ pub messages: Vec<AgentSessionMessageDocument>,
+}
+
+#[derive(Serialize)]
+struct ParentDocument<'a> {
+ agent_session_id: &'a str,
+ projection_generation: &'a str,
+ name: &'a str,
+ owner_id: &'a str,
+ bot_id: &'a str,
+ thread_id: Option<&'a str>,
+ originating_message_id: Option<&'a str>,
+ created_at_millis: EpochMillis,
+ updated_at_millis: EpochMillis,
+ agent_session_relation: &'static str,
+}
+
+#[derive(Serialize)]
+struct ChildDocument<'a> {
+ agent_session_id: &'a str,
+ projection_generation: &'a str,
+ message_turn: u32,
+ author: AgentSessionMessageAuthor,
+ author_user_id: Option<&'a str>,
+ content: &'a str,
+ agent_session_relation: ChildRelation<'a>,
+}
+
+#[derive(Serialize)]
+struct ChildRelation<'a> {
+ name: &'static str,
+ parent: &'a str,
+}
+
+fn resolve_destination(index_override: Option<&str>) -> &str {
+ index_override.unwrap_or(SearchIndex::AgentSessions.as_ref())
+}
+
+fn child_id(agent_session_id: &str, message: &AgentSessionMessageDocument) -> String {
+ format!("{agent_session_id}:{}:{}", message.turn, message.author)
+}
+
+fn parent_document(args: &ReconcileAgentSessionArgs) -> ParentDocument<'_> {
+ ParentDocument {
+ agent_session_id: &args.agent_session_id,
+ projection_generation: &args.projection_generation,
+ name: &args.name,
+ owner_id: &args.owner_id,
+ bot_id: &args.bot_id,
+ thread_id: args.thread_id.as_deref(),
+ originating_message_id: args.originating_message_id.as_deref(),
+ created_at_millis: args.created_at_millis,
+ updated_at_millis: args.updated_at_millis,
+ agent_session_relation: PARENT_RELATION,
+ }
+}
+
+fn child_document<'a>(
+ args: &'a ReconcileAgentSessionArgs,
+ message: &'a AgentSessionMessageDocument,
+) -> ChildDocument<'a> {
+ ChildDocument {
+ agent_session_id: &args.agent_session_id,
+ projection_generation: &args.projection_generation,
+ message_turn: message.turn,
+ author: message.author,
+ author_user_id: message.author_user_id.as_deref(),
+ content: &message.content,
+ agent_session_relation: ChildRelation {
+ name: CHILD_RELATION,
+ parent: &args.agent_session_id,
+ },
+ }
+}
+
+fn serialization_error(method: &str, error: impl std::fmt::Display) -> OpensearchClientError {
+ OpensearchClientError::DeserializationFailed {
+ details: error.to_string(),
+ method: Some(method.to_owned()),
+ }
+}
+
+async fn response_error(
+ response: opensearch::http::response::Response,
+ method: &str,
+) -> Result<()> {
+ let status_code = response.status_code();
+ if status_code.is_success() {
+ return Ok(());
+ }
+ let body = response
+ .text()
+ .await
+ .map_err(|error| serialization_error(method, error))?;
+ tracing::error!(%status_code, %body, "agent-session OpenSearch operation failed");
+ Err(OpensearchClientError::Unknown {
+ details: body,
+ method: Some(method.to_owned()),
+ })
+}
+
+/// Write the current parent and children, then remove documents from older
+/// projections. Callers serialize reconciles by session id.
+pub(crate) async fn reconcile_agent_session(
+ client: &opensearch::OpenSearch,
+ args: &ReconcileAgentSessionArgs,
+ index_override: Option<&str>,
+) -> Result<()> {
+ let index = resolve_destination(index_override);
+ let routing = args.agent_session_id.as_str();
+
+ let parent = client
+ .index(opensearch::IndexParts::IndexId(
+ index,
+ &args.agent_session_id,
+ ))
+ .routing(routing)
+ .body(parent_document(args))
+ .refresh(opensearch::params::Refresh::WaitFor)
+ .send()
+ .await
+ .map_err(|error| serialization_error("reconcile_agent_session_parent", error))?;
+ response_error(parent, "reconcile_agent_session_parent").await?;
+
+ if !args.messages.is_empty() {
+ let mut body = Vec::with_capacity(args.messages.len() * 2);
+ for message in &args.messages {
+ body.push(
+ serde_json::to_string(&serde_json::json!({
+ "index": {
+ "_id": child_id(&args.agent_session_id, message),
+ "routing": routing,
+ }
+ }))
+ .map_err(|error| serialization_error("reconcile_agent_session_bulk", error))?,
+ );
+ body.push(
+ serde_json::to_string(&child_document(args, message))
+ .map_err(|error| serialization_error("reconcile_agent_session_bulk", error))?,
+ );
+ }
+
+ let response = client
+ .bulk(BulkParts::Index(index))
+ .body(body)
+ .refresh(opensearch::params::Refresh::WaitFor)
+ .send()
+ .await
+ .map_err(|error| serialization_error("reconcile_agent_session_bulk", error))?;
+ let status_code = response.status_code();
+ if !status_code.is_success() {
+ return response_error(response, "reconcile_agent_session_bulk").await;
+ }
+ let response_body: serde_json::Value = response
+ .json()
+ .await
+ .map_err(|error| serialization_error("reconcile_agent_session_bulk", error))?;
+ if response_body["errors"].as_bool() == Some(true) {
+ tracing::error!(body=?response_body, "agent-session bulk reconcile had failed items");
+ return Err(OpensearchClientError::Unknown {
+ details: response_body.to_string(),
+ method: Some("reconcile_agent_session_bulk".to_owned()),
+ });
+ }
+ }
+
+ let prune_query = serde_json::json!({
+ "query": {
+ "bool": {
+ "filter": [{ "term": { "agent_session_id": &args.agent_session_id } }],
+ "must_not": [{
+ "term": { "projection_generation": &args.projection_generation }
+ }]
+ }
+ }
+ });
+ let routing_values = [routing];
+ let response = client
+ .delete_by_query(opensearch::DeleteByQueryParts::Index(&[index]))
+ .routing(&routing_values)
+ .body(prune_query)
+ .refresh(true)
+ .send()
+ .await
+ .map_err(|error| serialization_error("reconcile_agent_session_prune", error))?;
+ crate::agent_session::check_delete_response(response, "reconcile_agent_session_prune").await
+}
diff --git a/crates/opensearch_client/src/upsert/agent_session/test.rs b/crates/opensearch_client/src/upsert/agent_session/test.rs
new file mode 100644
index 0000000000..4ebf4579f5
--- /dev/null
+++ b/crates/opensearch_client/src/upsert/agent_session/test.rs
@@ -0,0 +1,139 @@
+use serde_json::json;
+
+use super::*;
+
+fn args() -> ReconcileAgentSessionArgs {
+ ReconcileAgentSessionArgs {
+ agent_session_id: "session-1".to_owned(),
+ name: "Investigate indexing".to_owned(),
+ owner_id: "user-1".to_owned(),
+ bot_id: "bot-1".to_owned(),
+ thread_id: Some("thread-1".to_owned()),
+ originating_message_id: None,
+ created_at_millis: EpochMillis::new(10).unwrap(),
+ updated_at_millis: EpochMillis::new(20).unwrap(),
+ projection_generation: "generation-1".to_owned(),
+ messages: vec![AgentSessionMessageDocument {
+ turn: 3,
+ author: AgentSessionMessageAuthor::Agent,
+ author_user_id: None,
+ content: "Folded output".to_owned(),
+ }],
+ }
+}
+
+#[test]
+fn parent_and_child_documents_share_session_and_generation() {
+ let args = args();
+
+ assert_eq!(
+ serde_json::to_value(parent_document(&args)).unwrap(),
+ json!({
+ "agent_session_id": "session-1",
+ "projection_generation": "generation-1",
+ "name": "Investigate indexing",
+ "owner_id": "user-1",
+ "bot_id": "bot-1",
+ "thread_id": "thread-1",
+ "originating_message_id": null,
+ "created_at_millis": 10,
+ "updated_at_millis": 20,
+ "agent_session_relation": "agent_session",
+ })
+ );
+ assert_eq!(
+ serde_json::to_value(child_document(&args, &args.messages[0])).unwrap(),
+ json!({
+ "agent_session_id": "session-1",
+ "projection_generation": "generation-1",
+ "message_turn": 3,
+ "author": "agent",
+ "author_user_id": null,
+ "content": "Folded output",
+ "agent_session_relation": {
+ "name": "message",
+ "parent": "session-1",
+ },
+ })
+ );
+ assert_eq!(
+ child_id("session-1", &args.messages[0]),
+ "session-1:3:agent"
+ );
+}
+
+#[test]
+fn index_override_selects_a_physical_backfill_index() {
+ assert_eq!(resolve_destination(None), "agent_sessions");
+ assert_eq!(
+ resolve_destination(Some("agent_sessions_v2")),
+ "agent_sessions_v2"
+ );
+}
+
+/// Uses only a uniquely named scratch index; never changes application indices.
+#[tokio::test]
+#[ignore = "requires local OpenSearch at http://localhost:9200"]
+async fn empty_index_accepts_both_authors_and_prunes_replaced_transcripts() {
+ use crate::OpensearchClient;
+ use opensearch::{SearchParts, indices::IndicesDeleteParts};
+
+ let client =
+ OpensearchClient::new("http://localhost:9200".into(), "".into(), "".into()).unwrap();
+ let index = format!("agent-session-indexing-test-{}", uuid::Uuid::new_v4());
+ client.ensure_index_exists(&index, json!({
+ "settings": {"number_of_shards": 1, "number_of_replicas": 0, "refresh_interval": "100ms"},
+ "mappings": {"properties": {
+ "agent_session_id": {"type": "keyword"},
+ "projection_generation": {"type": "keyword"},
+ "content": {"type": "text"},
+ "agent_session_relation": {"type": "join", "relations": {"agent_session": "message"}}
+ }}
+ })).await.unwrap();
+
+ let result: anyhow::Result<()> = async {
+ let mut projection = args();
+ projection.messages[0].content = "quartzresponse".into();
+ projection.messages.push(AgentSessionMessageDocument {
+ turn: 3, author: AgentSessionMessageAuthor::User, author_user_id: Some("user-1".into()),
+ content: "cobaltquestion".into(),
+ });
+ client.reconcile_agent_session(&projection, Some(&index)).await?;
+ // Both terms must join back to the same parent, across different authors.
+ let hits: serde_json::Value = client.inner.search(SearchParts::Index(&[&index]))
+ .body(json!({"query": {"bool": {"must": [
+ {"has_child": {"type": "message", "query": {"match": {"content": "quartzresponse"}}}},
+ {"has_child": {"type": "message", "query": {"match": {"content": "cobaltquestion"}}}}
+ ]}}})).send().await?.error_for_status_code()?.json().await?;
+ assert_eq!(hits["hits"]["total"]["value"], 1);
+
+ // History replacement removes the old user child and replaces agent text.
+ projection.projection_generation = "generation-2".into();
+ projection.messages.truncate(1);
+ projection.messages[0].content = "newhistory".into();
+ client.reconcile_agent_session(&projection, Some(&index)).await?;
+ client.reconcile_agent_session(&projection, Some(&index)).await?;
+ let hits: serde_json::Value = client.inner.search(SearchParts::Index(&[&index]))
+ .body(json!({"query": {"match_all": {}}})).send().await?.error_for_status_code()?.json().await?;
+ assert_eq!(hits["hits"]["total"]["value"], 2);
+ let content: Vec<_> = hits["hits"]["hits"].as_array().unwrap().iter()
+ .filter_map(|hit| hit["_source"]["content"].as_str()).collect();
+ assert_eq!(content, vec!["newhistory"]);
+
+ client.delete_agent_session(&projection.agent_session_id, Some(&index)).await?;
+ let hits: serde_json::Value = client.inner.search(SearchParts::Index(&[&index]))
+ .body(json!({"query": {"match_all": {}}})).send().await?.error_for_status_code()?.json().await?;
+ assert_eq!(hits["hits"]["total"]["value"], 0);
+ Ok(())
+ }.await;
+ client
+ .inner
+ .indices()
+ .delete(IndicesDeleteParts::Index(&[&index]))
+ .send()
+ .await
+ .unwrap()
+ .error_for_status_code()
+ .unwrap();
+ result.unwrap();
+}
diff --git a/services/search_processing_service/Cargo.toml b/services/search_processing_service/Cargo.toml
index 4752326b19..91f9df4da5 100644
--- a/services/search_processing_service/Cargo.toml
+++ b/services/search_processing_service/Cargo.toml
@@ -42,6 +42,8 @@ name = "search_processing_service_doppler_config"
path = "src/doppler_config.rs"
[dependencies]
+agent_session = { path = "../../crates/agent_session", default-features = false, features = ["outbound"] }
+agent_fold = { path = "../../crates/agent_fold" }
anyhow = { workspace = true }
aws-sdk-dynamodb = { workspace = true }
aws-sdk-s3 = { workspace = true }
diff --git a/services/search_processing_service/README.md b/services/search_processing_service/README.md
index f623cab7fb..8293b1045d 100644
--- a/services/search_processing_service/README.md
+++ b/services/search_processing_service/README.md
@@ -3,8 +3,9 @@
This service owns the search-event pipeline:
- **Live calls**: the `search-processing-service` Kafka consumer reads call lifecycle events from `macro.calls` and indexes them into OpenSearch.
-- **Other live entities**: SQS workers drain `SEARCH_EVENT_QUEUE` for all other search entities.
-- **Backfills**: internal HTTP endpoints enqueue every indexable record of a given entity type onto SQS. Call backfills remain on SQS alongside all other entity backfills.
+- **Agent sessions**: the same consumer subscribes to `macro.agent_session_lifecycle`, rereads the primary database, and indexes messages produced by `agent_fold`. Both user prompts and agent replies are searchable; raw ACP envelopes are not indexed. Agent-session backfills use this same direct indexing path, not SQS.
+- **Other live entities**: Kafka handlers and legacy SQS workers drain events for the remaining search entities.
+- **Backfills**: internal HTTP endpoints start tracked jobs. Most enqueue records onto SQS; agent sessions and properties index directly.
## Architecture
@@ -23,7 +24,64 @@ The orchestrator is the single inbound contract for HTTP handlers. Swapping an a
## Ingestion Ownership and Event Mapping
-Kafka is used only for live call indexing. SQS remains responsible for live events from other search entities and for every backfill, including call backfills. The SQS call message contract and handler remain available so `POST /internal/backfill/calls` can enqueue call records with the existing backfill path.
+Kafka subscriptions are declared in `src/inbound/kafka_consumer.rs`, including agent-session lifecycle events. SQS remains available for legacy producers and backfills other than agent sessions and direct property indexing. The SQS call message contract and handler remain available so `POST /internal/backfill/calls` can enqueue call records with the existing backfill path.
+
+### Agent-session indexing and rollout
+
+There is no agent-session indexing feature flag. The deployed search-processing
+binary subscribes to the existing `macro.agent_session_lifecycle` topic. Events
+are invalidation hints: every event rereads current metadata and the effective
+ACP log through the owning agent-session/fold services. It never indexes the
+event's excerpt as a substitute for the transcript. Completed turns, questions,
+renames, and deletions therefore converge on current persisted state.
+
+Live events and backfills hold the same per-session PostgreSQL advisory lease
+through the OpenSearch write. Lease connections have their own pool so waiting
+workers cannot exhaust the pool used to load snapshots. Each reconcile writes
+the parent and both authors' folded messages, refreshes them, then removes
+documents from older projection generations. Missing sessions delete their
+entire projection, even when triggered by an older event.
+
+For rollout:
+
+1. Verify the `agent_sessions` alias exists with the join mapping defined in
+ `infra/stacks/opensearch/helpers/scripts/create_indices.ts`. Do not reset
+ unrelated indices or recreate an existing index just to deploy this consumer.
+2. Deploy search-processing and confirm its consumer group is subscribed to
+ `macro.agent_session_lifecycle`. The topic and shared MSK permissions already
+ exist; the publisher does not need a deployment for this change.
+3. Call the internally authenticated `POST /internal/backfill/agent-sessions`
+ with `{}` to repair all existing sessions. It returns `202` and `{"job_id":
+ "..."}`; poll `GET /internal/backfill/{job_id}` until completed. Its `enqueued`
+ progress field counts sessions reconciled directly, not SQS messages.
+ Retained Kafka history alone is not a complete historical backfill.
+4. Verify a unique phrase from a new user prompt and a completed agent reply
+ both return the session. Also verify a rename and deletion.
+
+Use `{"agent_session_ids": ["<uuid>"]}` to repair selected sessions, including
+removing a known deleted session's documents. `index_override` can target a
+scratch/replacement index with the same mapping. A full scan walks existing rows
+in bounded UUID pages and does not use `modified_at`: ACP log changes do not
+necessarily update that timestamp. It cannot discover already-deleted IDs.
+
+The shared consumer commits on in-memory handoff and drops events after exhausted
+retries. Monitor dropped-event logs and use this backfill for recovery. Backfills
+are not automatically scheduled or started by deployment.
+
+Regression checks, from the repository root inside Nix, with `SQLX_OFFLINE` unset:
+
+```bash
+cargo test -p agent_fold
+cargo test -p agent_session search
+cargo test -p search_processing_service --no-default-features --features processing,service
+cargo test -p opensearch_client
+# With local OpenSearch running on localhost:9200; creates/deletes only a scratch index:
+cargo test -p opensearch_client empty_index_accepts_both_authors -- --ignored
+```
+
+The empty-index check covers parent/child joins across both authors, idempotent
+replays, transcript replacement, stale-child removal, and deletion. Do not rely
+on a previously populated local index to validate ingestion wiring.
The `macro.calls` lifecycle events map to search actions as follows:
@@ -40,12 +98,11 @@ Upserts read the current call state from Postgres and overwrite the indexed repr
### Kafka Delivery Contract
-- The durable consumer group is `search-processing-service`. It subscribes only to `macro.calls`.
-- The poll loop hands decoded events to a bounded 128-message channel. Sending waits when the channel is full, which stops polling and committing until the worker catches up.
-- Exactly one worker drains the channel sequentially. This preserves partition/key ordering so an upsert cannot be processed after a later delete for the same call. Do not add naive concurrent workers; any future parallelism must consistently shard by `call_id`.
-- An offset is committed asynchronously immediately after a successful in-memory handoff, not after OpenSearch processing. This commit-after-handoff design creates a loss window: a process or host crash can lose already committed events still in the 128-message buffer. If the worker channel closes before handoff, the offset is left uncommitted for redelivery.
+- The durable consumer group is `search-processing-service`. Its subscriptions are declared in `src/inbound/kafka_consumer.rs`.
+- The poll loop shards decoded events by entity ordering key across ten sequential workers, each with a bounded 16-message channel. Sending waits when the selected channel is full, stopping polling and committing until that worker catches up. Events for the same entity stay ordered.
+- An offset is committed asynchronously immediately after a successful in-memory handoff, not after OpenSearch processing. This commit-after-handoff design creates a loss window: a process or host crash can lose committed events still in those buffers. If the worker channel closes before handoff, the offset is left uncommitted for redelivery.
- Malformed, keyless, and unsupported-schema records are logged and committed without handoff so a poison record cannot wedge a partition. Commit failures are logged and may cause duplicate delivery, which is safe because full upserts and per-call deletes are idempotent.
-- Processing gets five total attempts. Failures are retried after 1, 2, 4, and 8 seconds; after the fifth failure, the event is logged with its call ID, event type, partition, offset, and final error, then dropped so later events can continue.
+- Processing gets three total attempts, with retries after 1 and 2 seconds. Exhausted events are logged and dropped so later events can continue.
## Running Locally
@@ -119,10 +176,11 @@ Use `POST /internal/backfill/calls` after a rollback, a dropped upsert, or a sus
## Backfill HTTP Routes
-Every search-indexed entity has a POST endpoint on sps's internal surface. They all share the same response shape (`{"enqueued": <usize>}`), share internal-auth via the `x-internal-auth-key` header, and accept a per-entity JSON filter in the request body.
+Every search-indexed entity has a POST endpoint on sps's internal surface. They return `202 Accepted` with `{"job_id": "..."}`, share internal-auth via the `x-internal-auth-key` header, and accept a per-entity JSON filter in the request body. Poll `GET /internal/backfill/{job_id}` for status and the `enqueued` progress count.
| Entity | Route | Body (all fields optional) |
|---|---|---|
+| Agent sessions | `POST /internal/backfill/agent-sessions` | `{"agent_session_ids": ["<uuid>"], "index_override": "agent_sessions_v2"}` — `{}` = all existing sessions, indexed directly |
| Calls | `POST /internal/backfill/calls` | `{"call_ids": ["<uuid>"]}` — empty = all archived calls |
| Chats | `POST /internal/backfill/chats` | `{"chat_ids": [...], "user_ids": [...]}` |
| Channels | `POST /internal/backfill/channels` | `{}` |
diff --git a/services/search_processing_service/src/api/context.rs b/services/search_processing_service/src/api/context.rs
index 24b5dd085b..3cb1d2a579 100644
--- a/services/search_processing_service/src/api/context.rs
+++ b/services/search_processing_service/src/api/context.rs
@@ -15,6 +15,7 @@ pub(crate) type SpsEventBroker = MacroEventBrokerService<KafkaEventPublisher, Ta
#[derive(Clone, FromRef)]
pub(crate) struct ApiContext {
+ pub agent_session_indexer: Arc<crate::AgentSessionIndexer>,
pub db: sqlx::Pool<sqlx::Postgres>,
pub opensearch_client: Arc<opensearch_client::OpensearchClient>,
pub authorization_state: MacroAuthorizationState<AuthorizationService>,
diff --git a/services/search_processing_service/src/api/internal/backfill.rs b/services/search_processing_service/src/api/internal/backfill.rs
index 32d552309b..afefbeec85 100644
--- a/services/search_processing_service/src/api/internal/backfill.rs
+++ b/services/search_processing_service/src/api/internal/backfill.rs
@@ -36,6 +36,7 @@ use crate::domain::service::BackfillService;
pub fn router() -> Router<ApiContext> {
Router::new()
+ .route("/agent-sessions", post(agent_sessions))
.route("/calls", post(calls))
.route("/chats", post(chats))
.route("/channels", post(channels))
@@ -52,6 +53,24 @@ struct AcceptedReceipt {
job_id: JobId,
}
+#[tracing::instrument(skip(service, jobs, _internal_authorization, req))]
+async fn agent_sessions(
+ State(service): State<Arc<crate::AgentSessionIndexer>>,
+ State(jobs): State<BackfillJobs>,
+ _internal_authorization: MacroAuthorizationExtractor<AuthorizationService, InternalOnly>,
+ extract::Json(req): extract::Json<
+ crate::domain::agent_session_index::AgentSessionBackfillRequest,
+ >,
+) -> Response {
+ spawn_backfill(
+ service,
+ jobs,
+ "agent-sessions",
+ move |svc, progress, cancel| async move { svc.backfill(req, progress, cancel).await },
+ )
+ .await
+}
+
#[tracing::instrument(skip(service, jobs, _internal_authorization, req))]
async fn calls(
State(service): State<Arc<BackfillServiceImpl>>,
@@ -212,15 +231,16 @@ async fn status(
/// to the worker future the caller built, spawn it, and return
/// `202 Accepted` with the job id. The worker future captures the request
/// body so the HTTP body reference doesn't outlive the handler.
-async fn spawn_backfill<F, Fut>(
- service: Arc<BackfillServiceImpl>,
+async fn spawn_backfill<S, F, Fut>(
+ service: Arc<S>,
jobs: BackfillJobs,
entity: &'static str,
run: F,
) -> Response
where
+ S: Send + Sync + 'static,
F: FnOnce(
- Arc<BackfillServiceImpl>,
+ Arc<S>,
Arc<crate::domain::jobs::JobProgress>,
tokio_util::sync::CancellationToken,
) -> Fut
diff --git a/services/search_processing_service/src/domain/agent_session_index.rs b/services/search_processing_service/src/domain/agent_session_index.rs
new file mode 100644
index 0000000000..7b9addd6b2
--- /dev/null
+++ b/services/search_processing_service/src/domain/agent_session_index.rs
@@ -0,0 +1,128 @@
+//! Reconcile the search projection from the owning agent-session domain service.
+
+use std::{collections::HashSet, sync::Arc};
+
+use agent_session::domain::{
+ model::AgentSessionId,
+ search::indexing::{SearchSnapshot, SearchSnapshotService},
+};
+use rootcause::Report;
+use serde::Deserialize;
+use tokio_util::sync::CancellationToken;
+use uuid::Uuid;
+
+use super::{
+ jobs::JobProgress,
+ models::{BackfillError, BackfillReceipt},
+};
+
+#[cfg(test)]
+mod test;
+
+/// Index operations over a folded snapshot; no ACP parsing belongs in the adapter.
+pub trait AgentSessionSearchIndex: Send + Sync + 'static {
+ fn reconcile(
+ &self,
+ snapshot: SearchSnapshot,
+ index_override: Option<&str>,
+ ) -> impl Future<Output = Result<(), Report>> + Send;
+ fn delete(
+ &self,
+ id: AgentSessionId,
+ index_override: Option<&str>,
+ ) -> impl Future<Output = Result<(), Report>> + Send;
+}
+
+/// Repair selected sessions, or scan every existing session when IDs are omitted.
+/// A full scan deliberately does not filter on metadata timestamps: log appends
+/// need not change `agent_session.modified_at`.
+#[derive(Debug, Default, Deserialize)]
+#[serde(default, deny_unknown_fields)]
+pub struct AgentSessionBackfillRequest {
+ pub agent_session_ids: Vec<Uuid>,
+ pub index_override: Option<String>,
+}
+
+/// Owns serialization, authoritative rereads, and deletion of missing sessions.
+pub struct AgentSessionIndexService<S, I> {
+ source: S,
+ index: I,
+}
+
+impl<S, I> AgentSessionIndexService<S, I> {
+ pub fn new(source: S, index: I) -> Self {
+ Self { source, index }
+ }
+}
+
+impl<S: SearchSnapshotService, I: AgentSessionSearchIndex> AgentSessionIndexService<S, I> {
+ /// Events are hints to reread, so delayed events cannot resurrect deleted sessions.
+ #[tracing::instrument(err, skip(self, index_override))]
+ pub async fn reconcile(
+ &self,
+ id: AgentSessionId,
+ index_override: Option<&str>,
+ ) -> Result<(), Report> {
+ let _lease = self.source.lock(id).await?;
+ match self.source.snapshot(id).await? {
+ Some(snapshot) => self.index.reconcile(snapshot, index_override).await,
+ None => self.index.delete(id, index_override).await,
+ }
+ }
+
+ /// Backfill and live events use the same service and cross-process session lease.
+ pub async fn backfill(
+ &self,
+ req: AgentSessionBackfillRequest,
+ progress: Arc<JobProgress>,
+ cancel: CancellationToken,
+ ) -> Result<BackfillReceipt, BackfillError> {
+ let mut enqueued = 0;
+ if !req.agent_session_ids.is_empty() {
+ let mut seen = HashSet::new();
+ for id in req.agent_session_ids {
+ if cancel.is_cancelled() {
+ break;
+ }
+ if !seen.insert(id) {
+ continue;
+ }
+ self.reconcile(
+ AgentSessionId::new_from_uuid(id),
+ req.index_override.as_deref(),
+ )
+ .await
+ .map_err(|e| BackfillError::Reindex(anyhow::anyhow!("{e:#}")))?;
+ enqueued += 1;
+ progress.add(1).await;
+ }
+ } else {
+ let mut after = None;
+ while !cancel.is_cancelled() {
+ let ids = self
+ .source
+ .page(after)
+ .await
+ .map_err(|e| BackfillError::Source(anyhow::anyhow!("{e:#}")))?;
+ if ids.is_empty() {
+ break;
+ }
+ for id in ids {
+ if cancel.is_cancelled() {
+ break;
+ }
+ self.reconcile(
+ AgentSessionId::new_from_uuid(id),
+ req.index_override.as_deref(),
+ )
+ .await
+ .map_err(|e| BackfillError::Reindex(anyhow::anyhow!("{e:#}")))?;
+ after = Some(id);
+ enqueued += 1;
+ progress.add(1).await;
+ }
+ }
+ }
+ Ok(BackfillReceipt { enqueued })
+ }
+}
diff --git a/services/search_processing_service/src/domain/agent_session_index/test.rs b/services/search_processing_service/src/domain/agent_session_index/test.rs
new file mode 100644
index 0000000000..88094fc88b
--- /dev/null
+++ b/services/search_processing_service/src/domain/agent_session_index/test.rs
@@ -0,0 +1,122 @@
+use super::*;
+use agent_session::domain::search::AgentSessionSearchMetadata;
+use std::sync::{
+ Mutex,
+ atomic::{AtomicBool, Ordering},
+};
+
+#[derive(Default)]
+struct State {
+ exists: AtomicBool,
+ locked: AtomicBool,
+ writes: Mutex<Vec<Option<Uuid>>>,
+}
+
+struct Source(Arc<State>);
+struct Index(Arc<State>);
+struct Lease(Arc<State>);
+impl Drop for Lease {
+ fn drop(&mut self) {
+ self.0.locked.store(false, Ordering::SeqCst);
+ }
+}
+
+impl SearchSnapshotService for Source {
+ type Lease = Lease;
+ async fn lock(&self, _: AgentSessionId) -> Result<Lease, Report> {
+ assert!(!self.0.locked.swap(true, Ordering::SeqCst));
+ Ok(Lease(self.0.clone()))
+ }
+ async fn snapshot(&self, id: AgentSessionId) -> Result<Option<SearchSnapshot>, Report> {
+ assert!(self.0.locked.load(Ordering::SeqCst));
+ Ok(self
+ .0
+ .exists
+ .load(Ordering::SeqCst)
+ .then(|| SearchSnapshot {
+ metadata: AgentSessionSearchMetadata {
+ id: id.as_uuid(),
+ name: "searchable title".into(),
+ owner_id: macro_user_id::user_id::MacroUserIdStr::try_from(
+ "macro|[email protected]".to_string(),
+ )
+ .unwrap(),
+ bot_id: Uuid::from_u128(2),
+ created_at: chrono::Utc::now(),
+ updated_at: chrono::Utc::now(),
+ },
+ messages: vec![],
+ }))
+ }
+ async fn page(&self, after: Option<Uuid>) -> Result<Vec<Uuid>, Report> {
+ Ok(if after.is_none() {
+ vec![Uuid::from_u128(1)]
+ } else {
+ vec![]
+ })
+ }
+}
+impl AgentSessionSearchIndex for Index {
+ async fn reconcile(&self, snapshot: SearchSnapshot, _: Option<&str>) -> Result<(), Report> {
+ assert!(self.0.locked.load(Ordering::SeqCst));
+ self.0
+ .writes
+ .lock()
+ .unwrap()
+ .push(Some(snapshot.metadata.id));
+ Ok(())
+ }
+ async fn delete(&self, _: AgentSessionId, _: Option<&str>) -> Result<(), Report> {
+ assert!(self.0.locked.load(Ordering::SeqCst));
+ self.0.writes.lock().unwrap().push(None);
+ Ok(())
+ }
+}
+
+#[tokio::test]
+async fn delayed_event_removes_deleted_session_and_holds_lease_through_index_write() {
+ let state = Arc::new(State::default());
+ let service = AgentSessionIndexService::new(Source(state.clone()), Index(state.clone()));
+ let id = AgentSessionId::new_from_uuid(Uuid::from_u128(1));
+ state.exists.store(true, Ordering::SeqCst);
+ service.reconcile(id, None).await.unwrap();
+ state.exists.store(false, Ordering::SeqCst);
+ service.reconcile(id, None).await.unwrap();
+ assert_eq!(
+ *state.writes.lock().unwrap(),
+ vec![Some(id.as_uuid()), None]
+ );
+ assert!(!state.locked.load(Ordering::SeqCst));
+}
+
+#[tokio::test]
+async fn backfill_deduplicates_ids_and_full_scan_stops_at_end() {
+ let state = Arc::new(State::default());
+ state.exists.store(true, Ordering::SeqCst);
+ let service = AgentSessionIndexService::new(Source(state.clone()), Index(state.clone()));
+ let id = Uuid::from_u128(1);
+ let progress = Arc::new(JobProgress::detached());
+ let receipt = service
+ .backfill(
+ AgentSessionBackfillRequest {
+ agent_session_ids: vec![id, id],
+ index_override: None,
+ },
+ progress.clone(),
+ CancellationToken::new(),
+ )
+ .await
+ .unwrap();
+ assert_eq!(receipt.enqueued, 1);
+ assert_eq!(progress.local_count(), 1);
+ let receipt = service
+ .backfill(
+ AgentSessionBackfillRequest::default(),
+ progress,
+ CancellationToken::new(),
+ )
+ .await
+ .unwrap();
+ assert_eq!(receipt.enqueued, 1);
+ assert_eq!(state.writes.lock().unwrap().len(), 2);
+}
diff --git a/services/search_processing_service/src/domain/mod.rs b/services/search_processing_service/src/domain/mod.rs
index 2761b811a6..275d68ef6c 100644
--- a/services/search_processing_service/src/domain/mod.rs
+++ b/services/search_processing_service/src/domain/mod.rs
@@ -7,6 +7,7 @@
//! own mapping to [`sqs_client::search::SearchQueueMessage`] — but the public
//! surface (orchestration + HTTP routes) stays uniform.
+pub mod agent_session_index;
pub mod jobs;
pub mod models;
pub mod ports;
diff --git a/services/search_processing_service/src/inbound/kafka_consumer.rs b/services/search_processing_service/src/inbound/kafka_consumer.rs
index ba5f13885c..405d052183 100644
--- a/services/search_processing_service/src/inbound/kafka_consumer.rs
+++ b/services/search_processing_service/src/inbound/kafka_consumer.rs
@@ -16,6 +16,7 @@
#![allow(clippy::enum_variant_names)]
+mod agent_session;
mod calendar_event;
mod call;
mod channel;
@@ -36,6 +37,7 @@ use std::{
time::Duration,
};
+use ::agent_session::domain::events::AgentSessionLifecycleMacroEvent;
use ::call::domain::events::CallMacroEvent;
use ::chat::domain::events::ChatMacroEvent;
use ::email::domain::events::EmailMacroEvent;
@@ -83,6 +85,7 @@ type SearchProcessingKafkaConsumer =
macro_event_broker::declare_topics!(
DeclaredMacroEvent:
+ AgentSessionLifecycleMacroEvent,
CalendarMacroEvent,
CallMacroEvent,
ChannelMacroEvent,
@@ -180,6 +183,7 @@ where
/// pool.
fn ordering_key(event: &DeclaredMacroEvent) -> Cow<'_, str> {
match event {
+ DeclaredMacroEvent::AgentSessionLifecycleMacroEvent(event) => Cow::Borrowed(event.key()),
DeclaredMacroEvent::CalendarMacroEvent(event) => Cow::Borrowed(event.key()),
DeclaredMacroEvent::CallMacroEvent(event) => Cow::Borrowed(event.key()),
DeclaredMacroEvent::ChannelMacroEvent(event) => Cow::Borrowed(event.key()),
@@ -268,6 +272,9 @@ async fn process_event(
let opensearch_client = context.opensearch_client.as_ref();
match event {
+ DeclaredMacroEvent::AgentSessionLifecycleMacroEvent(event) => {
+ agent_session::process_agent_session_event(context, event).await
+ }
DeclaredMacroEvent::CallMacroEvent(event) => {
process_call_event(db, opensearch_client, event, partition, offset).await
}
diff --git a/services/search_processing_service/src/inbound/kafka_consumer/agent_session.rs b/services/search_processing_service/src/inbound/kafka_consumer/agent_session.rs
new file mode 100644
index 0000000000..3d668b7453
--- /dev/null
+++ b/services/search_processing_service/src/inbound/kafka_consumer/agent_session.rs
@@ -0,0 +1,26 @@
+//! Lifecycle events trigger an authoritative reread, including deletion.
+
+use ::agent_session::domain::events::AgentSessionLifecycleMacroEvent;
+use macro_event_broker::MacroEvent as _;
+
+use super::{EventOutcome, KafkaProcessingContext, retry_processing};
+
+pub(super) async fn process_agent_session_event(
+ context: &KafkaProcessingContext,
+ event: &AgentSessionLifecycleMacroEvent,
+) -> EventOutcome {
+ let id = event.event().event.session_id();
+ match retry_processing(|attempt| async move {
+ context.agent_session_indexer.reconcile(id, None).await.inspect_err(|error| {
+ tracing::error!(error=?error, session_id=%id, attempt, "agent session indexing failed");
+ })
+ })
+ .await
+ {
+ Ok(()) => EventOutcome::Indexed,
+ Err(error) => {
+ tracing::error!(error=?error, session_id=%id, "agent session indexing exhausted retries; repair with backfill");
+ EventOutcome::Dropped
+ }
+ }
+}
diff --git a/services/search_processing_service/src/inbound/kafka_consumer/context.rs b/services/search_processing_service/src/inbound/kafka_consumer/context.rs
index dc1bc8f914..7b0f89f09b 100644
--- a/services/search_processing_service/src/inbound/kafka_consumer/context.rs
+++ b/services/search_processing_service/src/inbound/kafka_consumer/context.rs
@@ -8,6 +8,7 @@ use sqlx::PgPool;
/// Shared dependencies used to process search-index events from Kafka.
#[derive(Clone)]
pub(crate) struct KafkaProcessingContext {
+ pub(crate) agent_session_indexer: Arc<crate::AgentSessionIndexer>,
pub(crate) db: PgPool,
pub(crate) opensearch_client: Arc<OpensearchClient>,
pub(crate) s3_client: Arc<S3>,
diff --git a/services/search_processing_service/src/inbound/kafka_consumer/test.rs b/services/search_processing_service/src/inbound/kafka_consumer/test.rs
index aa55a24228..58761c5913 100644
--- a/services/search_processing_service/src/inbound/kafka_consumer/test.rs
+++ b/services/search_processing_service/src/inbound/kafka_consumer/test.rs
@@ -1270,6 +1270,7 @@ fn subscribes_to_declared_search_processing_topics_with_durable_group() {
"search-processing-service"
);
let topics = DeclaredMacroEvent::topics();
+ assert!(topics.contains(&macro_event_topics::MacroAgentSessionLifecycleTopic::TOPIC_STR));
assert_eq!(MacroChatsTopic::TOPIC_STR, "macro.chats");
assert!(topics.contains(&MacroCallsTopic::TOPIC_STR));
assert!(topics.contains(&MacroChannelsTopic::TOPIC_STR));
@@ -1512,6 +1513,41 @@ fn email_envelope_decodes_round_trip() {
assert_eq!(decoded_event.event().event, event);
}
+#[test]
+fn agent_session_lifecycle_decodes_and_routes_by_session() {
+ use ::agent_session::domain::{events::*, model::AgentSessionId};
+ use macro_event_topics::MacroAgentSessionLifecycleTopic;
+
+ let id = AgentSessionId::new_from_uuid(Uuid::from_u128(10));
+ let identity = SessionIdentity {
+ session_id: id,
+ session_name: "Indexed session".into(),
+ bot_id: serde_json::from_value(serde_json::json!(Uuid::from_u128(2))).unwrap(),
+ bot_name: "Agent".into(),
+ owner_id: user_id(),
+ origin: None,
+ audience: vec![],
+ };
+ for event in [
+ AgentSessionLifecycleEvent::Renamed(SessionRenamedMetadata {
+ identity: identity.clone(),
+ }),
+ AgentSessionLifecycleEvent::Deleted(SessionDeletedMetadata { identity }),
+ ] {
+ let message = encoded_message(
+ MacroAgentSessionLifecycleTopic::TOPIC_STR,
+ id,
+ Event::new(event.clone()),
+ );
+ let decoded = DeclaredMacroEvent::decode(&message).unwrap();
+ assert_eq!(ordering_key(&decoded), id.to_string());
+ let DeclaredMacroEvent::AgentSessionLifecycleMacroEvent(decoded) = decoded else {
+ panic!("expected agent-session lifecycle event");
+ };
+ assert_eq!(decoded.event().event, event);
+ }
+}
+
#[test]
fn chat_envelope_decodes_round_trip() {
let event = ChatTopicEvent::MessageSent(ChatMessageSentMetadata {
diff --git a/services/search_processing_service/src/main.rs b/services/search_processing_service/src/main.rs
index 23b45680b1..a64e38b5a6 100644
--- a/services/search_processing_service/src/main.rs
+++ b/services/search_processing_service/src/main.rs
@@ -47,6 +47,17 @@ mod process;
pub type BackfillServiceImpl =
BackfillOrchestrator<PgBackfillSource, SqsSearchEventPublisher, DirectPropertyBackfillIndexer>;
+/// Production wiring for the authoritative agent-session search projection.
+pub type AgentSessionIndexer = domain::agent_session_index::AgentSessionIndexService<
+ agent_session::domain::search::indexing::SearchSnapshotServiceImpl<
+ agent_session::outbound::postgres::search::PgSearchIndexingRepo,
+ agent_fold::domain::service::FoldedMessageService<
+ agent_session::outbound::postgres::PgAgentSessionRepo,
+ >,
+ >,
+ outbound::agent_session_search::OpenSearchAgentSessionIndex,
+>;
+
/// Resolve a read-replica macrodb URL and
/// connect a small pool. Returns `None` when the replica URL is missing,
/// blank. Failures
@@ -216,6 +227,22 @@ async fn main() -> anyhow::Result<()> {
let sqs_client = Arc::new(sqs_client);
+ let session_repo = agent_session::outbound::postgres::PgAgentSessionRepo::new(db.clone());
+ // Waiting advisory locks must not exhaust the pool used to read ACP logs.
+ let session_lock_pool = PgPoolOptions::new()
+ .max_connections(10)
+ .connect_lazy(config.database_url.as_ref())?;
+ let agent_session_indexer = Arc::new(AgentSessionIndexer::new(
+ agent_session::domain::search::indexing::SearchSnapshotServiceImpl::new(
+ agent_session::outbound::postgres::search::PgSearchIndexingRepo::new(
+ db.clone(),
+ session_lock_pool,
+ ),
+ agent_fold::domain::service::FoldedMessageService::new(session_repo),
+ ),
+ outbound::agent_session_search::OpenSearchAgentSessionIndex(opensearch_client.clone()),
+ ));
+
let backfill_service = Arc::new(BackfillOrchestrator::new(
PgBackfillSource::new(backfill_db, config.backfill_page_sizes()?),
SqsSearchEventPublisher::new(sqs_client.clone()),
@@ -263,6 +290,7 @@ async fn main() -> anyhow::Result<()> {
run_search_processing_workers(search_processing_context, config.worker_count);
let kafka_processing_context = KafkaProcessingContext {
+ agent_session_indexer: agent_session_indexer.clone(),
db: db.clone(),
opensearch_client: opensearch_client.clone(),
s3_client: s3_client.clone(),
@@ -292,6 +320,7 @@ async fn main() -> anyhow::Result<()> {
let api_result = api::setup_and_serve(
ApiContext {
+ agent_session_indexer,
db,
authorization_state,
opensearch_client,
diff --git a/services/search_processing_service/src/outbound/agent_session_search.rs b/services/search_processing_service/src/outbound/agent_session_search.rs
new file mode 100644
index 0000000000..daf440bfc4
--- /dev/null
+++ b/services/search_processing_service/src/outbound/agent_session_search.rs
@@ -0,0 +1,70 @@
+//! OpenSearch adapter for the agent-session indexing domain port.
+
+use agent_fold::domain::model::Author;
+use agent_session::domain::{model::AgentSessionId, search::indexing::SearchSnapshot};
+use opensearch_client::{
+ OpensearchClient,
+ agent_session::{
+ AgentSessionMessageAuthor, AgentSessionMessageDocument, ReconcileAgentSessionArgs,
+ },
+};
+use rootcause::Report;
+use std::sync::Arc;
+
+use crate::domain::agent_session_index::AgentSessionSearchIndex;
+
+pub struct OpenSearchAgentSessionIndex(pub Arc<OpensearchClient>);
+
+impl AgentSessionSearchIndex for OpenSearchAgentSessionIndex {
+ async fn reconcile(
+ &self,
+ snapshot: SearchSnapshot,
+ index_override: Option<&str>,
+ ) -> Result<(), Report> {
+ let metadata = snapshot.metadata;
+ let args = ReconcileAgentSessionArgs {
+ agent_session_id: metadata.id.to_string(),
+ name: metadata.name,
+ owner_id: metadata.owner_id.to_string(),
+ bot_id: metadata.bot_id.to_string(),
+ thread_id: None,
+ originating_message_id: None,
+ created_at_millis: opensearch_client::date_format::EpochMillis::new(
+ metadata.created_at.timestamp_millis(),
+ )?,
+ updated_at_millis: opensearch_client::date_format::EpochMillis::new(
+ metadata.updated_at.timestamp_millis(),
+ )?,
+ // The session lease serializes writers; every reconciliation gets
+ // its own generation to remove obsolete children after a refold.
+ projection_generation: macro_uuid::Uuid::now_v7().to_string(),
+ messages: snapshot
+ .messages
+ .into_iter()
+ .map(|message| AgentSessionMessageDocument {
+ turn: message.id.0,
+ author: match &message.author {
+ Author::User { .. } => AgentSessionMessageAuthor::User,
+ Author::Agent => AgentSessionMessageAuthor::Agent,
+ },
+ author_user_id: match &message.author {
+ Author::User { user_id } => user_id.as_ref().map(ToString::to_string),
+ Author::Agent => None,
+ },
+ content: message.searchable_text(),
+ })
+ .collect(),
+ };
+ Ok(self
+ .0
+ .reconcile_agent_session(&args, index_override)
+ .await?)
+ }
+
+ async fn delete(&self, id: AgentSessionId, index_override: Option<&str>) -> Result<(), Report> {
+ Ok(self
+ .0
+ .delete_agent_session(&id.to_string(), index_override)
+ .await?)
+ }
+}
diff --git a/services/search_processing_service/src/outbound/mod.rs b/services/search_processing_service/src/outbound/mod.rs
index 278cb05e09..3e8932a8a4 100644
--- a/services/search_processing_service/src/outbound/mod.rs
+++ b/services/search_processing_service/src/outbound/mod.rs
@@ -1,3 +1,4 @@
+pub mod agent_session_search;
pub mod property_search_indexer;
pub mod publisher;
pub mod source;