Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
feat(local): run search indexing in run_local (#4431)
gbirman331c1e341f72
diff --git a/docker-compose-databases.yml b/docker-compose-databases.ymlindex 341565b3f8..18933fe09b 100644--- a/docker-compose-databases.yml+++ b/docker-compose-databases.yml@@ -35,7 +35,9 @@ services:networks:- databasessearch:- image: opensearchproject/opensearch:latest+ build:+ context: ./infra/local/opensearch+ image: macro-local-opensearch:devenvironment:cluster.name: searchdiscovery.type: single-nodediff --git a/docker-compose.yml b/docker-compose.ymlindex 2c401ffe2c..d5870cc129 100644--- a/docker-compose.yml+++ b/docker-compose.yml@@ -143,11 +143,13 @@ services:# VITE_SYNC_SERVICE_HOST=sync-service-playground.macroverse.workers.dev VITE_LOCAL_SERVERS=ALL bun run dev# if you do so- OVERRIDE_SYNC_SERVICE_URL=http://sync-service:8787+ - OVERRIDE_LEXICAL_SERVICE_URL=http://lexical-service:8096depends_on:- redis- connection_gateway- authentication-service- sync_service+ - lexical_servicenetworks:databases:services:@@ -261,19 +263,15 @@ services:# Search indexing worker# ============================================================================search_processing_service:- profiles:- - processors- <<: *common-env- # Pinned because search_processing_service/pdfium-lib/linux/libpdfium.so is amd64-only;- # on Apple Silicon this forces QEMU emulation for build and runtime.- platform: linux/amd64- build:- context: ./rust/cloud-storage- dockerfile: Dockerfile.search_processing_service.dev+ <<: [*common-env, *rust-services-image]+ command: ["/app/out/search_processing_service"]ports:- "8092:8080"+ environment:+ - OVERRIDE_LEXICAL_SERVICE_URL=http://lexical-service:8096depends_on:- email_service+ - lexical_servicenetworks:databases:services:diff --git a/infra/local/opensearch/Dockerfile b/infra/local/opensearch/Dockerfilenew file mode 100644index 0000000000..013327267e--- /dev/null+++ b/infra/local/opensearch/Dockerfile@@ -0,0 +1,9 @@+# Local OpenSearch with the analysis-icu plugin baked in. The channels index+# mapping uses a custom `content_text` analyzer whose `icu_folding` filter comes+# from analysis-icu, which the stock image does not ship. Baking it at build time+# (vs installing on every container start) keeps startup fast and offline-safe.+#+# Pinned to 3.5 to match the AWS dev/prod domain engine version+# (infra/stacks/opensearch: `OpenSearch_3.5`).+FROM opensearchproject/opensearch:3.5.0+RUN /usr/share/opensearch/bin/opensearch-plugin install --batch analysis-icudiff --git a/infra/stacks/opensearch/helpers/scripts/create_call_records_v2.ts b/infra/stacks/opensearch/helpers/scripts/create_call_records_v2.tsdeleted file mode 100644index 2a485d8ef0..0000000000--- a/infra/stacks/opensearch/helpers/scripts/create_call_records_v2.ts+++ /dev/null@@ -1,89 +0,0 @@-/**- * Creates `call_records_v2` with the parent/child join mapping used for- * the multi-term AND search migration. Does NOT touch the `call_records`- * alias — `call_records_v2` is created idle so the- * search_processing_service backfill can populate it (with- * `index_override: "call_records_v2"`) while production reads/writes- * continue to flow through `call_records` -> `call_records_v1`.- *- * Swap the alias separately, once backfill is caught up and the new- * search query path is deployed.- *- * Usage:- * bun scripts/create_call_records_v2.ts- *- * Idempotent — safe to re-run; if `call_records_v2` already exists- * nothing happens.- */-require('dotenv').config();--import { client } from '../client';-import { SHARD_SETTINGS, SLOWLOG_SETTINGS } from '../constants';--const INDEX = 'call_records_v2';-const RELATION_PARENT = 'call';-const RELATION_CHILD = 'segment';--const BODY = {- settings: {- ...SHARD_SETTINGS,- ...SLOWLOG_SETTINGS,- refresh_interval: '2s',- },- mappings: {- dynamic: 'false',- properties: {- entity_id: { type: 'keyword' },- // Parent-only metadata- channel_id: { type: 'keyword', index: true, doc_values: true },- channel_name: {- type: 'text',- fields: { keyword: { type: 'keyword', ignore_above: 128 } },- },- participant_ids: { type: 'keyword', index: true, doc_values: true },- started_at_seconds: {- type: 'date',- format: 'epoch_second',- index: false,- doc_values: true,- },- ended_at_seconds: {- type: 'date',- format: 'epoch_second',- index: false,- doc_values: true,- },- // Child-only fields- transcript_id: { type: 'keyword', index: false, doc_values: true },- speaker_id: { type: 'keyword', index: true, doc_values: true },- sequence_num: { type: 'integer', index: false, doc_values: true },- content: { type: 'text', analyzer: 'standard' },- // Aliases preserved for any reader that still expects them; both map- // to the parent's call-start timestamp.- created_at_seconds: { type: 'alias', path: 'started_at_seconds' },- updated_at_seconds: { type: 'alias', path: 'started_at_seconds' },- // Join relationship- call_relation: {- type: 'join',- relations: { [RELATION_PARENT]: RELATION_CHILD },- },- },- },-};--async function run() {- const c = client();- const exists = (await c.indices.exists({ index: INDEX })).body;- if (exists) {- console.log(`${INDEX} already exists; nothing to do.`);- return;- }- console.log(`Creating ${INDEX} (no alias)`);- await c.indices.create({ index: INDEX, body: BODY });- console.log(`${INDEX} created`);-}--run().catch((err) => {- console.error(err);- process.exit(1);-});diff --git a/infra/stacks/opensearch/helpers/scripts/create_chats_v2.ts b/infra/stacks/opensearch/helpers/scripts/create_chats_v2.tsdeleted file mode 100644index 53c957f3a8..0000000000--- a/infra/stacks/opensearch/helpers/scripts/create_chats_v2.ts+++ /dev/null@@ -1,82 +0,0 @@-/**- * Creates `chats_v2` with the parent/child join mapping used for the- * multi-term AND search migration. Does NOT touch the `chats` alias —- * `chats_v2` is created idle so the search_processing_service backfill- * can populate it (with `index_override: "chats_v2"`) while production- * reads/writes continue to flow through `chats` -> `chats_v1`.- *- * Swap the alias separately, once backfill is caught up and the new search- * query path is deployed.- *- * Usage:- * bun scripts/create_chats_v2.ts- *- * Idempotent — safe to re-run; if `chats_v2` already exists nothing- * happens.- */-require('dotenv').config();--import { client } from '../client';-import { SHARD_SETTINGS, SLOWLOG_SETTINGS } from '../constants';--const INDEX = 'chats_v2';-const RELATION_PARENT = 'chat';-const RELATION_CHILD = 'message';--const BODY = {- settings: {- ...SHARD_SETTINGS,- ...SLOWLOG_SETTINGS,- refresh_interval: '1s',- },- mappings: {- dynamic: 'false',- properties: {- entity_id: { type: 'keyword' },- // Parent-only metadata- title: {- type: 'text',- fields: { keyword: { type: 'keyword', ignore_above: 50 } },- },- user_id: { type: 'keyword', index: true, doc_values: true },- updated_at_seconds: {- type: 'date',- format: 'epoch_second',- index: false,- doc_values: true,- },- // Child-only fields- chat_message_id: { type: 'keyword', index: false, doc_values: true },- content: { type: 'text', analyzer: 'standard' },- role: { type: 'keyword', index: false, doc_values: true },- created_at_seconds: {- type: 'date',- format: 'epoch_second',- index: false,- doc_values: true,- },- // Join relationship- chat_relation: {- type: 'join',- relations: { [RELATION_PARENT]: RELATION_CHILD },- },- },- },-};--async function run() {- const c = client();- const exists = (await c.indices.exists({ index: INDEX })).body;- if (exists) {- console.log(`${INDEX} already exists; nothing to do.`);- return;- }- console.log(`Creating ${INDEX} (no alias)`);- await c.indices.create({ index: INDEX, body: BODY });- console.log(`${INDEX} created`);-}--run().catch((err) => {- console.error(err);- process.exit(1);-});diff --git a/infra/stacks/opensearch/helpers/scripts/create_indices.ts b/infra/stacks/opensearch/helpers/scripts/create_indices.tsindex 19682c1a54..7969cd412e 100644--- a/infra/stacks/opensearch/helpers/scripts/create_indices.ts+++ b/infra/stacks/opensearch/helpers/scripts/create_indices.ts@@ -1,13 +1,18 @@import type { Client } from '@opensearch-project/opensearch';import { client } from '../client';import {+ CALL_RECORDS_ALIAS,+ CALL_RECORDS_INDEX,CHANNELS_ALIAS,CHANNELS_INDEX,+ CHATS_ALIAS,+ CHATS_INDEX,DOCUMENTS_ALIAS,DOCUMENTS_INDEX,EMAILS_ALIAS,EMAILS_INDEX,SHARD_SETTINGS,+ SLOWLOG_SETTINGS,} from '../constants';type CreateIndexArgs = {@@ -411,6 +416,105 @@ const EMAIL_BODY = {},};+// chats and call_records use parent/child join mappings for multi-term AND+// search. Their bodies live here (single entrypoint); the join relation names+// are the only structural difference from the flat indices above.+const CHATS_RELATION_PARENT = 'chat';+const CHATS_RELATION_CHILD = 'message';++const CHATS_V2_BODY = {+ settings: {+ ...SHARD_SETTINGS,+ ...SLOWLOG_SETTINGS,+ refresh_interval: '1s',+ },+ mappings: {+ dynamic: 'false',+ properties: {+ entity_id: { type: 'keyword' },+ // Parent-only metadata+ title: {+ type: 'text',+ fields: { keyword: { type: 'keyword', ignore_above: 50 } },+ },+ user_id: { type: 'keyword', index: true, doc_values: true },+ updated_at_seconds: {+ type: 'date',+ format: 'epoch_second',+ index: false,+ doc_values: true,+ },+ // Child-only fields+ chat_message_id: { type: 'keyword', index: false, doc_values: true },+ content: { type: 'text', analyzer: 'standard' },+ role: { type: 'keyword', index: false, doc_values: true },+ created_at_seconds: {+ type: 'date',+ format: 'epoch_second',+ index: false,+ doc_values: true,+ },+ // Join relationship+ chat_relation: {+ type: 'join',+ relations: { [CHATS_RELATION_PARENT]: CHATS_RELATION_CHILD },+ },+ },+ },+};++const CALL_RECORDS_RELATION_PARENT = 'call';+const CALL_RECORDS_RELATION_CHILD = 'segment';++const CALL_RECORDS_V2_BODY = {+ settings: {+ ...SHARD_SETTINGS,+ ...SLOWLOG_SETTINGS,+ refresh_interval: '2s',+ },+ mappings: {+ dynamic: 'false',+ properties: {+ entity_id: { type: 'keyword' },+ // Parent-only metadata+ channel_id: { type: 'keyword', index: true, doc_values: true },+ channel_name: {+ type: 'text',+ fields: { keyword: { type: 'keyword', ignore_above: 128 } },+ },+ participant_ids: { type: 'keyword', index: true, doc_values: true },+ started_at_seconds: {+ type: 'date',+ format: 'epoch_second',+ index: false,+ doc_values: true,+ },+ ended_at_seconds: {+ type: 'date',+ format: 'epoch_second',+ index: false,+ doc_values: true,+ },+ // Child-only fields+ transcript_id: { type: 'keyword', index: false, doc_values: true },+ speaker_id: { type: 'keyword', index: true, doc_values: true },+ sequence_num: { type: 'integer', index: false, doc_values: true },+ content: { type: 'text', analyzer: 'standard' },+ // Aliases preserved for any reader that still expects them; both map+ // to the parent's call-start timestamp.+ created_at_seconds: { type: 'alias', path: 'started_at_seconds' },+ updated_at_seconds: { type: 'alias', path: 'started_at_seconds' },+ // Join relationship+ call_relation: {+ type: 'join',+ relations: {+ [CALL_RECORDS_RELATION_PARENT]: CALL_RECORDS_RELATION_CHILD,+ },+ },+ },+ },+};+async function createIndices() {const opensearchClient = client();console.log('Creating indices...');@@ -421,8 +525,6 @@ async function createIndices() {aliasName: DOCUMENTS_ALIAS,body: DOCUMENT_BODY,});- // chats and call_records use parent/child join mappings and are- // created by create_chats_v2.ts / create_call_records_v2.ts.await createIndexWithAlias(opensearchClient, {indexName: EMAILS_INDEX,aliasName: EMAILS_ALIAS,@@ -433,6 +535,19 @@ async function createIndices() {aliasName: CHANNELS_ALIAS,body: CHANNEL_BODY,});+ // chats and call_records use parent/child join mappings (bodies inlined+ // above as CHATS_V2_BODY / CALL_RECORDS_V2_BODY). Idempotent — no-ops where+ // they already exist.+ await createIndexWithAlias(opensearchClient, {+ indexName: CHATS_INDEX,+ aliasName: CHATS_ALIAS,+ body: CHATS_V2_BODY,+ });+ await createIndexWithAlias(opensearchClient, {+ indexName: CALL_RECORDS_INDEX,+ aliasName: CALL_RECORDS_ALIAS,+ body: CALL_RECORDS_V2_BODY,+ });console.log('done');} catch (error) {console.error('Error', error);diff --git a/rust/cloud-storage/search_processing_service/Cargo.toml b/rust/cloud-storage/search_processing_service/Cargo.tomlindex 874ee6f925..034f81198f 100644--- a/rust/cloud-storage/search_processing_service/Cargo.toml+++ b/rust/cloud-storage/search_processing_service/Cargo.toml@@ -6,9 +6,10 @@ publish = falseversion = "0.1.0"[features]-default = ["processing", "service"]+default = ["processing", "service", "pdf"]processing = []service = ["macro_db_client", "models_properties", "properties_db_client"]+pdf = ["dep:pdfium-render"][[bin]]name = "search_processing_service"@@ -58,7 +59,7 @@ models_search = { path = "../models_search" }nom = { workspace = true }opensearch_client = { path = "../opensearch_client" }ensure_exists = { path = "../ensure_exists", features = ["dynamodb"] }-pdfium-render = { workspace = true }+pdfium-render = { workspace = true, optional = true }properties_db_client = { path = "../properties_db_client", optional = true }rust-embed = "8.6.0"s3_client = { path = "../s3_client" }diff --git a/rust/cloud-storage/search_processing_service/src/main.rs b/rust/cloud-storage/search_processing_service/src/main.rsindex 0c77a7b063..04dd1b4056 100644--- a/rust/cloud-storage/search_processing_service/src/main.rs+++ b/rust/cloud-storage/search_processing_service/src/main.rs@@ -13,6 +13,7 @@ use config::{Config, Environment};use lexical_client::LexicalClient;use macro_entrypoint::MacroEntrypoint;use opensearch_client::OpensearchClient;+#[cfg(feature = "pdf")]use rust_embed::RustEmbed;use sqlx::PgPool;use sqlx::postgres::PgPoolOptions;@@ -54,6 +55,7 @@ async fn resolve_readonly_pool(read_only_db_url: DatabaseUrlReadonly) -> Option<}}+#[cfg(feature = "pdf")]#[allow(dead_code)]#[derive(RustEmbed)]#[folder = "pdfium-lib/linux/"]@@ -133,6 +135,7 @@ async fn main() -> anyhow::Result<()> {use std::sync::Arc;// Ensures that pdfium binary exists so we can kill the container early on failure+ #[cfg(feature = "pdf")]if !std::fs::exists("./pdfium-lib/linux/libpdfium.so").expect("able to find file") {anyhow::bail!("libpdfium.so is missing");} else {diff --git a/rust/cloud-storage/search_processing_service/src/parsers/mod.rs b/rust/cloud-storage/search_processing_service/src/parsers/mod.rsindex f959fa64e2..3eff36d7de 100644--- a/rust/cloud-storage/search_processing_service/src/parsers/mod.rs+++ b/rust/cloud-storage/search_processing_service/src/parsers/mod.rs@@ -1,4 +1,5 @@pub mod canvas;pub mod docx;pub mod markdown;+#[cfg(feature = "pdf")]pub mod pdf;diff --git a/rust/cloud-storage/search_processing_service/src/process/document/raw_document.rs b/rust/cloud-storage/search_processing_service/src/process/document/raw_document.rsindex 80314f7f3b..bcc8879611 100644--- a/rust/cloud-storage/search_processing_service/src/process/document/raw_document.rs+++ b/rust/cloud-storage/search_processing_service/src/process/document/raw_document.rs@@ -18,8 +18,10 @@ use s3_key::{build_docx_to_pdf_converted_document_key,};+#[cfg(feature = "pdf")]+use crate::parsers::pdf::parse_pdf_pages;use crate::{- parsers::{canvas::parse_canvas, markdown::parse_markdown_legacy, pdf::parse_pdf_pages},+ parsers::{canvas::parse_canvas, markdown::parse_markdown_legacy},process::document::document_info::{DocumentInfo, get_document_info},};@@ -222,23 +224,32 @@ pub async fn update_search_with_raw_document(let mut upserts: Vec<UpsertDocumentArgs> = match file_type {FileType::Pdf | FileType::Docx => {- let pages_content = parse_pdf_pages(content).context("unable to parse pdf")?;- pages_content- .iter()- .enumerate()- .map(|(i, page_content)| UpsertDocumentArgs {- document_id: search_extractor_message.document_id.clone(),- node_id: i.to_string(), // page number- raw_content: None,- document_name: document_name.clone(),- content: page_content.clone(),- owner_id: search_extractor_message.user_id.clone(),- file_type: file_type.to_string(),- updated_at_seconds: updated_at,- sub_type: sub_type.clone(),- properties: vec![],- })- .collect()+ #[cfg(feature = "pdf")]+ {+ let pages_content = parse_pdf_pages(content).context("unable to parse pdf")?;+ pages_content+ .iter()+ .enumerate()+ .map(|(i, page_content)| UpsertDocumentArgs {+ document_id: search_extractor_message.document_id.clone(),+ node_id: i.to_string(), // page number+ raw_content: None,+ document_name: document_name.clone(),+ content: page_content.clone(),+ owner_id: search_extractor_message.user_id.clone(),+ file_type: file_type.to_string(),+ updated_at_seconds: updated_at,+ sub_type: sub_type.clone(),+ properties: vec![],+ })+ .collect()+ }+ #[cfg(not(feature = "pdf"))]+ {+ let _ = content;+ tracing::debug!("pdf/docx indexing skipped: pdf feature disabled");+ vec![]+ }}FileType::Canvas => {let content =diff --git a/rust/cloud-storage/tools/xtask/src/local/build/zigbuild.rs b/rust/cloud-storage/tools/xtask/src/local/build/zigbuild.rsindex 0687005977..27e8cdf1a1 100644--- a/rust/cloud-storage/tools/xtask/src/local/build/zigbuild.rs+++ b/rust/cloud-storage/tools/xtask/src/local/build/zigbuild.rs@@ -62,6 +62,10 @@ pub fn run(stage: &Stage, target: Target) -> Result<()> {.arg("--no-default-features").arg("--bin").arg(svc.cargo_bin);+ let features = svc.build_features();+ if !features.is_empty() {+ cmd.arg("--features").arg(features.join(","));+ }stage.run(&format!("Building {} (no default features)", svc.cargo_bin),&mut cmd,diff --git a/rust/cloud-storage/tools/xtask/src/local/inventory.rs b/rust/cloud-storage/tools/xtask/src/local/inventory.rsindex 2d8642daa3..63d6e9f7da 100644--- a/rust/cloud-storage/tools/xtask/src/local/inventory.rs+++ b/rust/cloud-storage/tools/xtask/src/local/inventory.rs@@ -41,10 +41,10 @@ pub struct RustService {/// Opt-in services (e.g. `search_processing_service`) only start with an/// explicit profile and are never converted to the runtime image.pub opt_in: bool,- /// Built locally with `--no-default-features` (true only for- /// `authentication_service`): its `rate_limit` feature is in the default- /// set, and dropping it removes the 1-request/minute login-code throttle- /// that otherwise makes local dev painful.+ /// Built locally with `--no-default-features` (plus any [`Self::build_features`]).+ /// `authentication_service` drops the default `rate_limit` feature (removing+ /// the 1-request/minute login-code throttle that makes local dev painful);+ /// `search_processing_service` drops the default amd64-only `pdf` feature.pub no_default_features: bool,}@@ -59,10 +59,20 @@ impl RustService {pub fn is_opt_in(&self) -> bool {self.opt_in}++ /// Cargo features to re-enable alongside `--no-default-features` for this+ /// service's local build (empty = none). Only `search_processing_service`+ /// needs this: dropping default features removes the amd64-only `pdf`+ /// feature, but its bin still requires `processing`/`service`.+ pub fn build_features(&self) -> &'static [&'static str] {+ match self.cargo_bin {+ "search_processing_service" => &["processing", "service"],+ _ => &[],+ }+ }}-/// The full service inventory: the 12 binaries bundled today plus the opt-in-/// `search_processing_service`.+/// The full service inventory of the local service binaries.pub const RUST_SERVICES: &[RustService] = &[RustService {compose_name: "authentication-service",@@ -209,12 +219,13 @@ pub const RUST_SERVICES: &[RustService] = &[host_port: Some(Port::SearchProcessing),path_prefix: None,is_websocket: false,- // Opt-in: profiles:[processors], amd64-only libpdfium, separate- // Dockerfile. Auto-starts in no mode; never converted to the runtime- // image by gen-compose.- modes: &[],- opt_in: true,- no_default_features: false,+ // Local-only: the default `pdf` feature bundles an amd64-only libpdfium,+ // so local builds drop it (see `build_features`) and cross-compile native+ // like every other service. Dev/prod still deploy from the dedicated+ // Dockerfile with the default (pdf-enabled) build.+ modes: &[Mode::Local],+ opt_in: false,+ no_default_features: true,},];diff --git a/rust/cloud-storage/tools/xtask/src/local/inventory/test.rs b/rust/cloud-storage/tools/xtask/src/local/inventory/test.rsindex 4cfbea20d0..29d11cb014 100644--- a/rust/cloud-storage/tools/xtask/src/local/inventory/test.rs+++ b/rust/cloud-storage/tools/xtask/src/local/inventory/test.rs@@ -3,14 +3,12 @@ use super::*;#[test]fn local_binaries_are_unique_and_complete() {let bins = local_binaries();- // 12 distinct binaries (the bundled set), excluding the opt-in processor.- assert_eq!(bins.len(), 12, "{bins:?}");+ // 13 distinct binaries (the bundled set, including the local-only+ // search_processing_service).+ assert_eq!(bins.len(), 13, "{bins:?}");assert!(bins.contains(&"pubsub_workers"));assert!(bins.contains(&"document_upload_finalizer_local_worker"));- assert!(- !bins.contains(&"search_processing_service"),- "opt-in must be excluded"- );+ assert!(bins.contains(&"search_processing_service"));let mut sorted = bins.clone();sorted.dedup();assert_eq!(sorted.len(), bins.len(), "binaries must be deduplicated");diff --git a/rust/cloud-storage/tools/xtask/src/local/local_env.rs b/rust/cloud-storage/tools/xtask/src/local/local_env.rsindex debcd58aac..36b2eb8140 100644--- a/rust/cloud-storage/tools/xtask/src/local/local_env.rs+++ b/rust/cloud-storage/tools/xtask/src/local/local_env.rs@@ -117,6 +117,13 @@ impl StorageEnv {for table in resources::TABLES {env.insert(table.env_key.into(), table.name.into());}+ // search_processing_service self-creates this DynamoDB table on startup+ // in Local (BackfillJobs::ensure_table), so it is not in the provisioned+ // catalog above — it only needs the name wired here.+ env.insert(+ "BACKFILL_JOBS_TABLE".into(),+ "search-processing-backfill-jobs".into(),+ );}}diff --git a/rust/cloud-storage/tools/xtask/src/local/mod.rs b/rust/cloud-storage/tools/xtask/src/local/mod.rsindex 711b4871da..d0a094d6bc 100644--- a/rust/cloud-storage/tools/xtask/src/local/mod.rs+++ b/rust/cloud-storage/tools/xtask/src/local/mod.rs@@ -30,6 +30,7 @@ pub mod kickstart;pub mod local_env;pub mod localstack;pub mod mailpit;+pub mod opensearch;pub mod proxy;pub mod resources;pub mod stage;@@ -559,6 +560,11 @@ fn bring_up_infra(fa.arg("up").arg("-d").arg("fusionauth");stage.run("Starting FusionAuth", &mut fa)?;fusionauth::wait_ready(stage, instance)?;++ // OpenSearch is up (`--wait`) but empty. Create the search indices ++ // aliases (idempotent) so the unified search path works out of the box+ // instead of 404ing on the missing `documents`/`chats`/… indices.+ opensearch::provision_indices(stage, instance, &env.merged)?;}Ok(())}diff --git a/rust/cloud-storage/tools/xtask/src/local/opensearch.rs b/rust/cloud-storage/tools/xtask/src/local/opensearch.rsnew file mode 100644index 0000000000..32828f69dd--- /dev/null+++ b/rust/cloud-storage/tools/xtask/src/local/opensearch.rs@@ -0,0 +1,55 @@+//! Local OpenSearch index bootstrap. The index mappings live in the TS infra+//! helpers (the canonical, test-guarded source), so we run `create_indices.ts`+//! against the local cluster rather than duplicate the mapping bodies in Rust.+//! It creates every index + alias the search reads (documents, emails, channels,+//! chats, call_records) and is idempotent — safe to run on every startup.++use std::collections::BTreeMap;+use std::process::Command;++use anyhow::Result;++use super::instance::{Instance, Port};+use super::{repo_root, stage::Stage};++/// The TS OpenSearch helpers package (its own package.json + bun.lock).+fn helpers_dir() -> std::path::PathBuf {+ repo_root().join("infra/stacks/opensearch/helpers")+}++/// Create the search indices + aliases in the local OpenSearch by running the+/// canonical TS bootstrap script. The local cluster runs with the security+/// plugin disabled, so the credentials are accepted but ignored — they only need+/// to be present for the client to construct.+pub fn provision_indices(+ stage: &Stage,+ instance: &Instance,+ env: &BTreeMap<String, String>,+) -> Result<()> {+ let dir = helpers_dir();+ let url = format!("http://localhost:{}", instance.port(Port::OpenSearch));+ let username = env+ .get("OPENSEARCH_USERNAME")+ .cloned()+ .unwrap_or_else(|| "macrouser".to_string());+ let password = env+ .get("OPENSEARCH_PASSWORD")+ .cloned()+ .unwrap_or_else(|| "local".to_string());++ if !dir.join("node_modules").exists() {+ let mut install = Command::new("bun");+ install.current_dir(&dir).arg("install");+ stage.run("Installing OpenSearch helper deps", &mut install)?;+ }++ let mut cmd = Command::new("bun");+ cmd.current_dir(&dir)+ .args(["run", "scripts/create_indices.ts"])+ .env("ENVIRONMENT", "local")+ .env("OPENSEARCH_URL", &url)+ .env("OPENSEARCH_USERNAME", &username)+ .env("OPENSEARCH_PASSWORD", &password);+ stage.run("Creating search indices", &mut cmd)?;+ Ok(())+}diff --git a/rust/sync-service/Dockerfile b/rust/sync-service/Dockerfileindex a6b079c67b..b4b6bfb6a9 100644--- a/rust/sync-service/Dockerfile+++ b/rust/sync-service/Dockerfile@@ -51,4 +51,4 @@ EXPOSE 8787# Generate .dev.vars from environment variables so wrangler can resolve secret bindings,# apply local D1 migrations, then start.-CMD sh -c 'printf "LOCAL_API_KEY=%s\nDOCUMENT_PERMISSIONS_SECRET=%s\n" "$INTERNAL_API_SECRET_KEY" "$DOCUMENT_PERMISSIONS_SECRET" > .dev.vars && CI=true npx wrangler d1 migrations apply USER_PEER_MAPPING --local --config wrangler.docker.toml && exec npx wrangler dev --local --ip 0.0.0.0 --port 8787 --config wrangler.docker.toml'+CMD sh -c 'printf "LOCAL_API_KEY=%s\nDOCUMENT_PERMISSIONS_SECRET=%s\nSPS_API_SECRET_KEY=%s\n" "$INTERNAL_API_SECRET_KEY" "$DOCUMENT_PERMISSIONS_SECRET" "$INTERNAL_API_SECRET_KEY" > .dev.vars && CI=true npx wrangler d1 migrations apply USER_PEER_MAPPING --local --config wrangler.docker.toml && exec npx wrangler dev --local --ip 0.0.0.0 --port 8787 --config wrangler.docker.toml'diff --git a/rust/sync-service/wrangler.docker.toml b/rust/sync-service/wrangler.docker.tomlindex 607455f598..2a653af9f9 100644--- a/rust/sync-service/wrangler.docker.toml+++ b/rust/sync-service/wrangler.docker.toml@@ -30,7 +30,7 @@ bucket_name = "document-snapshot-bucket-dev"[vars]INTERNAL_API_SECRET_KEY = "LOCAL_API_KEY"-SPS_URL = "https://search-processing-dev.macro.com"+SPS_URL = "http://search-processing-service:8080"DSS_URL = "http://document_storage_service:8080"DSS_INTERNAL_AUTH_KEY = "local"