Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

feat(local): run search indexing in run_local (#4431)

gbirman331c1e341f72
diff --git a/docker-compose-databases.yml b/docker-compose-databases.yml
index 341565b3f8..18933fe09b 100644
--- a/docker-compose-databases.yml
+++ b/docker-compose-databases.yml
@@ -35,7 +35,9 @@ services:
networks:
- databases
search:
- image: opensearchproject/opensearch:latest
+ build:
+ context: ./infra/local/opensearch
+ image: macro-local-opensearch:dev
environment:
cluster.name: search
discovery.type: single-node
diff --git a/docker-compose.yml b/docker-compose.yml
index 2c401ffe2c..d5870cc129 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -143,11 +143,13 @@ services:
# VITE_SYNC_SERVICE_HOST=sync-service-playground.macroverse.workers.dev VITE_LOCAL_SERVERS=ALL bun run dev
# if you do so
- OVERRIDE_SYNC_SERVICE_URL=http://sync-service:8787
+ - OVERRIDE_LEXICAL_SERVICE_URL=http://lexical-service:8096
depends_on:
- redis
- connection_gateway
- authentication-service
- sync_service
+ - lexical_service
networks:
databases:
services:
@@ -261,19 +263,15 @@ services:
# Search indexing worker
# ============================================================================
search_processing_service:
- profiles:
- - processors
- <<: *common-env
- # Pinned because search_processing_service/pdfium-lib/linux/libpdfium.so is amd64-only;
- # on Apple Silicon this forces QEMU emulation for build and runtime.
- platform: linux/amd64
- build:
- context: ./rust/cloud-storage
- dockerfile: Dockerfile.search_processing_service.dev
+ <<: [*common-env, *rust-services-image]
+ command: ["/app/out/search_processing_service"]
ports:
- "8092:8080"
+ environment:
+ - OVERRIDE_LEXICAL_SERVICE_URL=http://lexical-service:8096
depends_on:
- email_service
+ - lexical_service
networks:
databases:
services:
diff --git a/infra/local/opensearch/Dockerfile b/infra/local/opensearch/Dockerfile
new file mode 100644
index 0000000000..013327267e
--- /dev/null
+++ b/infra/local/opensearch/Dockerfile
@@ -0,0 +1,9 @@
+# Local OpenSearch with the analysis-icu plugin baked in. The channels index
+# mapping uses a custom `content_text` analyzer whose `icu_folding` filter comes
+# from analysis-icu, which the stock image does not ship. Baking it at build time
+# (vs installing on every container start) keeps startup fast and offline-safe.
+#
+# Pinned to 3.5 to match the AWS dev/prod domain engine version
+# (infra/stacks/opensearch: `OpenSearch_3.5`).
+FROM opensearchproject/opensearch:3.5.0
+RUN /usr/share/opensearch/bin/opensearch-plugin install --batch analysis-icu
diff --git a/infra/stacks/opensearch/helpers/scripts/create_call_records_v2.ts b/infra/stacks/opensearch/helpers/scripts/create_call_records_v2.ts
deleted file mode 100644
index 2a485d8ef0..0000000000
--- a/infra/stacks/opensearch/helpers/scripts/create_call_records_v2.ts
+++ /dev/null
@@ -1,89 +0,0 @@
-/**
- * Creates `call_records_v2` with the parent/child join mapping used for
- * the multi-term AND search migration. Does NOT touch the `call_records`
- * alias — `call_records_v2` is created idle so the
- * search_processing_service backfill can populate it (with
- * `index_override: "call_records_v2"`) while production reads/writes
- * continue to flow through `call_records` -> `call_records_v1`.
- *
- * Swap the alias separately, once backfill is caught up and the new
- * search query path is deployed.
- *
- * Usage:
- * bun scripts/create_call_records_v2.ts
- *
- * Idempotent — safe to re-run; if `call_records_v2` already exists
- * nothing happens.
- */
-require('dotenv').config();
-
-import { client } from '../client';
-import { SHARD_SETTINGS, SLOWLOG_SETTINGS } from '../constants';
-
-const INDEX = 'call_records_v2';
-const RELATION_PARENT = 'call';
-const RELATION_CHILD = 'segment';
-
-const BODY = {
- settings: {
- ...SHARD_SETTINGS,
- ...SLOWLOG_SETTINGS,
- refresh_interval: '2s',
- },
- mappings: {
- dynamic: 'false',
- properties: {
- entity_id: { type: 'keyword' },
- // Parent-only metadata
- channel_id: { type: 'keyword', index: true, doc_values: true },
- channel_name: {
- type: 'text',
- fields: { keyword: { type: 'keyword', ignore_above: 128 } },
- },
- participant_ids: { type: 'keyword', index: true, doc_values: true },
- started_at_seconds: {
- type: 'date',
- format: 'epoch_second',
- index: false,
- doc_values: true,
- },
- ended_at_seconds: {
- type: 'date',
- format: 'epoch_second',
- index: false,
- doc_values: true,
- },
- // Child-only fields
- transcript_id: { type: 'keyword', index: false, doc_values: true },
- speaker_id: { type: 'keyword', index: true, doc_values: true },
- sequence_num: { type: 'integer', index: false, doc_values: true },
- content: { type: 'text', analyzer: 'standard' },
- // Aliases preserved for any reader that still expects them; both map
- // to the parent's call-start timestamp.
- created_at_seconds: { type: 'alias', path: 'started_at_seconds' },
- updated_at_seconds: { type: 'alias', path: 'started_at_seconds' },
- // Join relationship
- call_relation: {
- type: 'join',
- relations: { [RELATION_PARENT]: RELATION_CHILD },
- },
- },
- },
-};
-
-async function run() {
- const c = client();
- const exists = (await c.indices.exists({ index: INDEX })).body;
- if (exists) {
- console.log(`${INDEX} already exists; nothing to do.`);
- return;
- }
- console.log(`Creating ${INDEX} (no alias)`);
- await c.indices.create({ index: INDEX, body: BODY });
- console.log(`${INDEX} created`);
-}
-
-run().catch((err) => {
- console.error(err);
- process.exit(1);
-});
diff --git a/infra/stacks/opensearch/helpers/scripts/create_chats_v2.ts b/infra/stacks/opensearch/helpers/scripts/create_chats_v2.ts
deleted file mode 100644
index 53c957f3a8..0000000000
--- a/infra/stacks/opensearch/helpers/scripts/create_chats_v2.ts
+++ /dev/null
@@ -1,82 +0,0 @@
-/**
- * Creates `chats_v2` with the parent/child join mapping used for the
- * multi-term AND search migration. Does NOT touch the `chats` alias —
- * `chats_v2` is created idle so the search_processing_service backfill
- * can populate it (with `index_override: "chats_v2"`) while production
- * reads/writes continue to flow through `chats` -> `chats_v1`.
- *
- * Swap the alias separately, once backfill is caught up and the new search
- * query path is deployed.
- *
- * Usage:
- * bun scripts/create_chats_v2.ts
- *
- * Idempotent — safe to re-run; if `chats_v2` already exists nothing
- * happens.
- */
-require('dotenv').config();
-
-import { client } from '../client';
-import { SHARD_SETTINGS, SLOWLOG_SETTINGS } from '../constants';
-
-const INDEX = 'chats_v2';
-const RELATION_PARENT = 'chat';
-const RELATION_CHILD = 'message';
-
-const BODY = {
- settings: {
- ...SHARD_SETTINGS,
- ...SLOWLOG_SETTINGS,
- refresh_interval: '1s',
- },
- mappings: {
- dynamic: 'false',
- properties: {
- entity_id: { type: 'keyword' },
- // Parent-only metadata
- title: {
- type: 'text',
- fields: { keyword: { type: 'keyword', ignore_above: 50 } },
- },
- user_id: { type: 'keyword', index: true, doc_values: true },
- updated_at_seconds: {
- type: 'date',
- format: 'epoch_second',
- index: false,
- doc_values: true,
- },
- // Child-only fields
- chat_message_id: { type: 'keyword', index: false, doc_values: true },
- content: { type: 'text', analyzer: 'standard' },
- role: { type: 'keyword', index: false, doc_values: true },
- created_at_seconds: {
- type: 'date',
- format: 'epoch_second',
- index: false,
- doc_values: true,
- },
- // Join relationship
- chat_relation: {
- type: 'join',
- relations: { [RELATION_PARENT]: RELATION_CHILD },
- },
- },
- },
-};
-
-async function run() {
- const c = client();
- const exists = (await c.indices.exists({ index: INDEX })).body;
- if (exists) {
- console.log(`${INDEX} already exists; nothing to do.`);
- return;
- }
- console.log(`Creating ${INDEX} (no alias)`);
- await c.indices.create({ index: INDEX, body: BODY });
- console.log(`${INDEX} created`);
-}
-
-run().catch((err) => {
- console.error(err);
- process.exit(1);
-});
diff --git a/infra/stacks/opensearch/helpers/scripts/create_indices.ts b/infra/stacks/opensearch/helpers/scripts/create_indices.ts
index 19682c1a54..7969cd412e 100644
--- a/infra/stacks/opensearch/helpers/scripts/create_indices.ts
+++ b/infra/stacks/opensearch/helpers/scripts/create_indices.ts
@@ -1,13 +1,18 @@
import type { Client } from '@opensearch-project/opensearch';
import { client } from '../client';
import {
+ CALL_RECORDS_ALIAS,
+ CALL_RECORDS_INDEX,
CHANNELS_ALIAS,
CHANNELS_INDEX,
+ CHATS_ALIAS,
+ CHATS_INDEX,
DOCUMENTS_ALIAS,
DOCUMENTS_INDEX,
EMAILS_ALIAS,
EMAILS_INDEX,
SHARD_SETTINGS,
+ SLOWLOG_SETTINGS,
} from '../constants';
type CreateIndexArgs = {
@@ -411,6 +416,105 @@ const EMAIL_BODY = {
},
};
+// chats and call_records use parent/child join mappings for multi-term AND
+// search. Their bodies live here (single entrypoint); the join relation names
+// are the only structural difference from the flat indices above.
+const CHATS_RELATION_PARENT = 'chat';
+const CHATS_RELATION_CHILD = 'message';
+
+const CHATS_V2_BODY = {
+ settings: {
+ ...SHARD_SETTINGS,
+ ...SLOWLOG_SETTINGS,
+ refresh_interval: '1s',
+ },
+ mappings: {
+ dynamic: 'false',
+ properties: {
+ entity_id: { type: 'keyword' },
+ // Parent-only metadata
+ title: {
+ type: 'text',
+ fields: { keyword: { type: 'keyword', ignore_above: 50 } },
+ },
+ user_id: { type: 'keyword', index: true, doc_values: true },
+ updated_at_seconds: {
+ type: 'date',
+ format: 'epoch_second',
+ index: false,
+ doc_values: true,
+ },
+ // Child-only fields
+ chat_message_id: { type: 'keyword', index: false, doc_values: true },
+ content: { type: 'text', analyzer: 'standard' },
+ role: { type: 'keyword', index: false, doc_values: true },
+ created_at_seconds: {
+ type: 'date',
+ format: 'epoch_second',
+ index: false,
+ doc_values: true,
+ },
+ // Join relationship
+ chat_relation: {
+ type: 'join',
+ relations: { [CHATS_RELATION_PARENT]: CHATS_RELATION_CHILD },
+ },
+ },
+ },
+};
+
+const CALL_RECORDS_RELATION_PARENT = 'call';
+const CALL_RECORDS_RELATION_CHILD = 'segment';
+
+const CALL_RECORDS_V2_BODY = {
+ settings: {
+ ...SHARD_SETTINGS,
+ ...SLOWLOG_SETTINGS,
+ refresh_interval: '2s',
+ },
+ mappings: {
+ dynamic: 'false',
+ properties: {
+ entity_id: { type: 'keyword' },
+ // Parent-only metadata
+ channel_id: { type: 'keyword', index: true, doc_values: true },
+ channel_name: {
+ type: 'text',
+ fields: { keyword: { type: 'keyword', ignore_above: 128 } },
+ },
+ participant_ids: { type: 'keyword', index: true, doc_values: true },
+ started_at_seconds: {
+ type: 'date',
+ format: 'epoch_second',
+ index: false,
+ doc_values: true,
+ },
+ ended_at_seconds: {
+ type: 'date',
+ format: 'epoch_second',
+ index: false,
+ doc_values: true,
+ },
+ // Child-only fields
+ transcript_id: { type: 'keyword', index: false, doc_values: true },
+ speaker_id: { type: 'keyword', index: true, doc_values: true },
+ sequence_num: { type: 'integer', index: false, doc_values: true },
+ content: { type: 'text', analyzer: 'standard' },
+ // Aliases preserved for any reader that still expects them; both map
+ // to the parent's call-start timestamp.
+ created_at_seconds: { type: 'alias', path: 'started_at_seconds' },
+ updated_at_seconds: { type: 'alias', path: 'started_at_seconds' },
+ // Join relationship
+ call_relation: {
+ type: 'join',
+ relations: {
+ [CALL_RECORDS_RELATION_PARENT]: CALL_RECORDS_RELATION_CHILD,
+ },
+ },
+ },
+ },
+};
+
async function createIndices() {
const opensearchClient = client();
console.log('Creating indices...');
@@ -421,8 +525,6 @@ async function createIndices() {
aliasName: DOCUMENTS_ALIAS,
body: DOCUMENT_BODY,
});
- // chats and call_records use parent/child join mappings and are
- // created by create_chats_v2.ts / create_call_records_v2.ts.
await createIndexWithAlias(opensearchClient, {
indexName: EMAILS_INDEX,
aliasName: EMAILS_ALIAS,
@@ -433,6 +535,19 @@ async function createIndices() {
aliasName: CHANNELS_ALIAS,
body: CHANNEL_BODY,
});
+ // chats and call_records use parent/child join mappings (bodies inlined
+ // above as CHATS_V2_BODY / CALL_RECORDS_V2_BODY). Idempotent — no-ops where
+ // they already exist.
+ await createIndexWithAlias(opensearchClient, {
+ indexName: CHATS_INDEX,
+ aliasName: CHATS_ALIAS,
+ body: CHATS_V2_BODY,
+ });
+ await createIndexWithAlias(opensearchClient, {
+ indexName: CALL_RECORDS_INDEX,
+ aliasName: CALL_RECORDS_ALIAS,
+ body: CALL_RECORDS_V2_BODY,
+ });
console.log('done');
} catch (error) {
console.error('Error', error);
diff --git a/rust/cloud-storage/search_processing_service/Cargo.toml b/rust/cloud-storage/search_processing_service/Cargo.toml
index 874ee6f925..034f81198f 100644
--- a/rust/cloud-storage/search_processing_service/Cargo.toml
+++ b/rust/cloud-storage/search_processing_service/Cargo.toml
@@ -6,9 +6,10 @@ publish = false
version = "0.1.0"
[features]
-default = ["processing", "service"]
+default = ["processing", "service", "pdf"]
processing = []
service = ["macro_db_client", "models_properties", "properties_db_client"]
+pdf = ["dep:pdfium-render"]
[[bin]]
name = "search_processing_service"
@@ -58,7 +59,7 @@ models_search = { path = "../models_search" }
nom = { workspace = true }
opensearch_client = { path = "../opensearch_client" }
ensure_exists = { path = "../ensure_exists", features = ["dynamodb"] }
-pdfium-render = { workspace = true }
+pdfium-render = { workspace = true, optional = true }
properties_db_client = { path = "../properties_db_client", optional = true }
rust-embed = "8.6.0"
s3_client = { path = "../s3_client" }
diff --git a/rust/cloud-storage/search_processing_service/src/main.rs b/rust/cloud-storage/search_processing_service/src/main.rs
index 0c77a7b063..04dd1b4056 100644
--- a/rust/cloud-storage/search_processing_service/src/main.rs
+++ b/rust/cloud-storage/search_processing_service/src/main.rs
@@ -13,6 +13,7 @@ use config::{Config, Environment};
use lexical_client::LexicalClient;
use macro_entrypoint::MacroEntrypoint;
use opensearch_client::OpensearchClient;
+#[cfg(feature = "pdf")]
use rust_embed::RustEmbed;
use sqlx::PgPool;
use sqlx::postgres::PgPoolOptions;
@@ -54,6 +55,7 @@ async fn resolve_readonly_pool(read_only_db_url: DatabaseUrlReadonly) -> Option<
}
}
+#[cfg(feature = "pdf")]
#[allow(dead_code)]
#[derive(RustEmbed)]
#[folder = "pdfium-lib/linux/"]
@@ -133,6 +135,7 @@ async fn main() -> anyhow::Result<()> {
use std::sync::Arc;
// Ensures that pdfium binary exists so we can kill the container early on failure
+ #[cfg(feature = "pdf")]
if !std::fs::exists("./pdfium-lib/linux/libpdfium.so").expect("able to find file") {
anyhow::bail!("libpdfium.so is missing");
} else {
diff --git a/rust/cloud-storage/search_processing_service/src/parsers/mod.rs b/rust/cloud-storage/search_processing_service/src/parsers/mod.rs
index f959fa64e2..3eff36d7de 100644
--- a/rust/cloud-storage/search_processing_service/src/parsers/mod.rs
+++ b/rust/cloud-storage/search_processing_service/src/parsers/mod.rs
@@ -1,4 +1,5 @@
pub mod canvas;
pub mod docx;
pub mod markdown;
+#[cfg(feature = "pdf")]
pub mod pdf;
diff --git a/rust/cloud-storage/search_processing_service/src/process/document/raw_document.rs b/rust/cloud-storage/search_processing_service/src/process/document/raw_document.rs
index 80314f7f3b..bcc8879611 100644
--- a/rust/cloud-storage/search_processing_service/src/process/document/raw_document.rs
+++ b/rust/cloud-storage/search_processing_service/src/process/document/raw_document.rs
@@ -18,8 +18,10 @@ use s3_key::{
build_docx_to_pdf_converted_document_key,
};
+#[cfg(feature = "pdf")]
+use crate::parsers::pdf::parse_pdf_pages;
use crate::{
- parsers::{canvas::parse_canvas, markdown::parse_markdown_legacy, pdf::parse_pdf_pages},
+ parsers::{canvas::parse_canvas, markdown::parse_markdown_legacy},
process::document::document_info::{DocumentInfo, get_document_info},
};
@@ -222,23 +224,32 @@ pub async fn update_search_with_raw_document(
let mut upserts: Vec<UpsertDocumentArgs> = match file_type {
FileType::Pdf | FileType::Docx => {
- let pages_content = parse_pdf_pages(content).context("unable to parse pdf")?;
- pages_content
- .iter()
- .enumerate()
- .map(|(i, page_content)| UpsertDocumentArgs {
- document_id: search_extractor_message.document_id.clone(),
- node_id: i.to_string(), // page number
- raw_content: None,
- document_name: document_name.clone(),
- content: page_content.clone(),
- owner_id: search_extractor_message.user_id.clone(),
- file_type: file_type.to_string(),
- updated_at_seconds: updated_at,
- sub_type: sub_type.clone(),
- properties: vec![],
- })
- .collect()
+ #[cfg(feature = "pdf")]
+ {
+ let pages_content = parse_pdf_pages(content).context("unable to parse pdf")?;
+ pages_content
+ .iter()
+ .enumerate()
+ .map(|(i, page_content)| UpsertDocumentArgs {
+ document_id: search_extractor_message.document_id.clone(),
+ node_id: i.to_string(), // page number
+ raw_content: None,
+ document_name: document_name.clone(),
+ content: page_content.clone(),
+ owner_id: search_extractor_message.user_id.clone(),
+ file_type: file_type.to_string(),
+ updated_at_seconds: updated_at,
+ sub_type: sub_type.clone(),
+ properties: vec![],
+ })
+ .collect()
+ }
+ #[cfg(not(feature = "pdf"))]
+ {
+ let _ = content;
+ tracing::debug!("pdf/docx indexing skipped: pdf feature disabled");
+ vec![]
+ }
}
FileType::Canvas => {
let content =
diff --git a/rust/cloud-storage/tools/xtask/src/local/build/zigbuild.rs b/rust/cloud-storage/tools/xtask/src/local/build/zigbuild.rs
index 0687005977..27e8cdf1a1 100644
--- a/rust/cloud-storage/tools/xtask/src/local/build/zigbuild.rs
+++ b/rust/cloud-storage/tools/xtask/src/local/build/zigbuild.rs
@@ -62,6 +62,10 @@ pub fn run(stage: &Stage, target: Target) -> Result<()> {
.arg("--no-default-features")
.arg("--bin")
.arg(svc.cargo_bin);
+ let features = svc.build_features();
+ if !features.is_empty() {
+ cmd.arg("--features").arg(features.join(","));
+ }
stage.run(
&format!("Building {} (no default features)", svc.cargo_bin),
&mut cmd,
diff --git a/rust/cloud-storage/tools/xtask/src/local/inventory.rs b/rust/cloud-storage/tools/xtask/src/local/inventory.rs
index 2d8642daa3..63d6e9f7da 100644
--- a/rust/cloud-storage/tools/xtask/src/local/inventory.rs
+++ b/rust/cloud-storage/tools/xtask/src/local/inventory.rs
@@ -41,10 +41,10 @@ pub struct RustService {
/// Opt-in services (e.g. `search_processing_service`) only start with an
/// explicit profile and are never converted to the runtime image.
pub opt_in: bool,
- /// Built locally with `--no-default-features` (true only for
- /// `authentication_service`): its `rate_limit` feature is in the default
- /// set, and dropping it removes the 1-request/minute login-code throttle
- /// that otherwise makes local dev painful.
+ /// Built locally with `--no-default-features` (plus any [`Self::build_features`]).
+ /// `authentication_service` drops the default `rate_limit` feature (removing
+ /// the 1-request/minute login-code throttle that makes local dev painful);
+ /// `search_processing_service` drops the default amd64-only `pdf` feature.
pub no_default_features: bool,
}
@@ -59,10 +59,20 @@ impl RustService {
pub fn is_opt_in(&self) -> bool {
self.opt_in
}
+
+ /// Cargo features to re-enable alongside `--no-default-features` for this
+ /// service's local build (empty = none). Only `search_processing_service`
+ /// needs this: dropping default features removes the amd64-only `pdf`
+ /// feature, but its bin still requires `processing`/`service`.
+ pub fn build_features(&self) -> &'static [&'static str] {
+ match self.cargo_bin {
+ "search_processing_service" => &["processing", "service"],
+ _ => &[],
+ }
+ }
}
-/// The full service inventory: the 12 binaries bundled today plus the opt-in
-/// `search_processing_service`.
+/// The full service inventory of the local service binaries.
pub const RUST_SERVICES: &[RustService] = &[
RustService {
compose_name: "authentication-service",
@@ -209,12 +219,13 @@ pub const RUST_SERVICES: &[RustService] = &[
host_port: Some(Port::SearchProcessing),
path_prefix: None,
is_websocket: false,
- // Opt-in: profiles:[processors], amd64-only libpdfium, separate
- // Dockerfile. Auto-starts in no mode; never converted to the runtime
- // image by gen-compose.
- modes: &[],
- opt_in: true,
- no_default_features: false,
+ // Local-only: the default `pdf` feature bundles an amd64-only libpdfium,
+ // so local builds drop it (see `build_features`) and cross-compile native
+ // like every other service. Dev/prod still deploy from the dedicated
+ // Dockerfile with the default (pdf-enabled) build.
+ modes: &[Mode::Local],
+ opt_in: false,
+ no_default_features: true,
},
];
diff --git a/rust/cloud-storage/tools/xtask/src/local/inventory/test.rs b/rust/cloud-storage/tools/xtask/src/local/inventory/test.rs
index 4cfbea20d0..29d11cb014 100644
--- a/rust/cloud-storage/tools/xtask/src/local/inventory/test.rs
+++ b/rust/cloud-storage/tools/xtask/src/local/inventory/test.rs
@@ -3,14 +3,12 @@ use super::*;
#[test]
fn local_binaries_are_unique_and_complete() {
let bins = local_binaries();
- // 12 distinct binaries (the bundled set), excluding the opt-in processor.
- assert_eq!(bins.len(), 12, "{bins:?}");
+ // 13 distinct binaries (the bundled set, including the local-only
+ // search_processing_service).
+ assert_eq!(bins.len(), 13, "{bins:?}");
assert!(bins.contains(&"pubsub_workers"));
assert!(bins.contains(&"document_upload_finalizer_local_worker"));
- assert!(
- !bins.contains(&"search_processing_service"),
- "opt-in must be excluded"
- );
+ assert!(bins.contains(&"search_processing_service"));
let mut sorted = bins.clone();
sorted.dedup();
assert_eq!(sorted.len(), bins.len(), "binaries must be deduplicated");
diff --git a/rust/cloud-storage/tools/xtask/src/local/local_env.rs b/rust/cloud-storage/tools/xtask/src/local/local_env.rs
index debcd58aac..36b2eb8140 100644
--- a/rust/cloud-storage/tools/xtask/src/local/local_env.rs
+++ b/rust/cloud-storage/tools/xtask/src/local/local_env.rs
@@ -117,6 +117,13 @@ impl StorageEnv {
for table in resources::TABLES {
env.insert(table.env_key.into(), table.name.into());
}
+ // search_processing_service self-creates this DynamoDB table on startup
+ // in Local (BackfillJobs::ensure_table), so it is not in the provisioned
+ // catalog above — it only needs the name wired here.
+ env.insert(
+ "BACKFILL_JOBS_TABLE".into(),
+ "search-processing-backfill-jobs".into(),
+ );
}
}
diff --git a/rust/cloud-storage/tools/xtask/src/local/mod.rs b/rust/cloud-storage/tools/xtask/src/local/mod.rs
index 711b4871da..d0a094d6bc 100644
--- a/rust/cloud-storage/tools/xtask/src/local/mod.rs
+++ b/rust/cloud-storage/tools/xtask/src/local/mod.rs
@@ -30,6 +30,7 @@ pub mod kickstart;
pub mod local_env;
pub mod localstack;
pub mod mailpit;
+pub mod opensearch;
pub mod proxy;
pub mod resources;
pub mod stage;
@@ -559,6 +560,11 @@ fn bring_up_infra(
fa.arg("up").arg("-d").arg("fusionauth");
stage.run("Starting FusionAuth", &mut fa)?;
fusionauth::wait_ready(stage, instance)?;
+
+ // OpenSearch is up (`--wait`) but empty. Create the search indices +
+ // aliases (idempotent) so the unified search path works out of the box
+ // instead of 404ing on the missing `documents`/`chats`/… indices.
+ opensearch::provision_indices(stage, instance, &env.merged)?;
}
Ok(())
}
diff --git a/rust/cloud-storage/tools/xtask/src/local/opensearch.rs b/rust/cloud-storage/tools/xtask/src/local/opensearch.rs
new file mode 100644
index 0000000000..32828f69dd
--- /dev/null
+++ b/rust/cloud-storage/tools/xtask/src/local/opensearch.rs
@@ -0,0 +1,55 @@
+//! Local OpenSearch index bootstrap. The index mappings live in the TS infra
+//! helpers (the canonical, test-guarded source), so we run `create_indices.ts`
+//! against the local cluster rather than duplicate the mapping bodies in Rust.
+//! It creates every index + alias the search reads (documents, emails, channels,
+//! chats, call_records) and is idempotent — safe to run on every startup.
+
+use std::collections::BTreeMap;
+use std::process::Command;
+
+use anyhow::Result;
+
+use super::instance::{Instance, Port};
+use super::{repo_root, stage::Stage};
+
+/// The TS OpenSearch helpers package (its own package.json + bun.lock).
+fn helpers_dir() -> std::path::PathBuf {
+ repo_root().join("infra/stacks/opensearch/helpers")
+}
+
+/// Create the search indices + aliases in the local OpenSearch by running the
+/// canonical TS bootstrap script. The local cluster runs with the security
+/// plugin disabled, so the credentials are accepted but ignored — they only need
+/// to be present for the client to construct.
+pub fn provision_indices(
+ stage: &Stage,
+ instance: &Instance,
+ env: &BTreeMap<String, String>,
+) -> Result<()> {
+ let dir = helpers_dir();
+ let url = format!("http://localhost:{}", instance.port(Port::OpenSearch));
+ let username = env
+ .get("OPENSEARCH_USERNAME")
+ .cloned()
+ .unwrap_or_else(|| "macrouser".to_string());
+ let password = env
+ .get("OPENSEARCH_PASSWORD")
+ .cloned()
+ .unwrap_or_else(|| "local".to_string());
+
+ if !dir.join("node_modules").exists() {
+ let mut install = Command::new("bun");
+ install.current_dir(&dir).arg("install");
+ stage.run("Installing OpenSearch helper deps", &mut install)?;
+ }
+
+ let mut cmd = Command::new("bun");
+ cmd.current_dir(&dir)
+ .args(["run", "scripts/create_indices.ts"])
+ .env("ENVIRONMENT", "local")
+ .env("OPENSEARCH_URL", &url)
+ .env("OPENSEARCH_USERNAME", &username)
+ .env("OPENSEARCH_PASSWORD", &password);
+ stage.run("Creating search indices", &mut cmd)?;
+ Ok(())
+}
diff --git a/rust/sync-service/Dockerfile b/rust/sync-service/Dockerfile
index a6b079c67b..b4b6bfb6a9 100644
--- a/rust/sync-service/Dockerfile
+++ b/rust/sync-service/Dockerfile
@@ -51,4 +51,4 @@ EXPOSE 8787
# Generate .dev.vars from environment variables so wrangler can resolve secret bindings,
# apply local D1 migrations, then start.
-CMD sh -c 'printf "LOCAL_API_KEY=%s\nDOCUMENT_PERMISSIONS_SECRET=%s\n" "$INTERNAL_API_SECRET_KEY" "$DOCUMENT_PERMISSIONS_SECRET" > .dev.vars && CI=true npx wrangler d1 migrations apply USER_PEER_MAPPING --local --config wrangler.docker.toml && exec npx wrangler dev --local --ip 0.0.0.0 --port 8787 --config wrangler.docker.toml'
+CMD sh -c 'printf "LOCAL_API_KEY=%s\nDOCUMENT_PERMISSIONS_SECRET=%s\nSPS_API_SECRET_KEY=%s\n" "$INTERNAL_API_SECRET_KEY" "$DOCUMENT_PERMISSIONS_SECRET" "$INTERNAL_API_SECRET_KEY" > .dev.vars && CI=true npx wrangler d1 migrations apply USER_PEER_MAPPING --local --config wrangler.docker.toml && exec npx wrangler dev --local --ip 0.0.0.0 --port 8787 --config wrangler.docker.toml'
diff --git a/rust/sync-service/wrangler.docker.toml b/rust/sync-service/wrangler.docker.toml
index 607455f598..2a653af9f9 100644
--- a/rust/sync-service/wrangler.docker.toml
+++ b/rust/sync-service/wrangler.docker.toml
@@ -30,7 +30,7 @@ bucket_name = "document-snapshot-bucket-dev"
[vars]
INTERNAL_API_SECRET_KEY = "LOCAL_API_KEY"
-SPS_URL = "https://search-processing-dev.macro.com"
+SPS_URL = "http://search-processing-service:8080"
DSS_URL = "http://document_storage_service:8080"
DSS_INTERNAL_AUTH_KEY = "local"