Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
feat(search_processing_service): migrate search processing service to gateway (#6142)
Will Hutchinson9a6f3e6e7e3e
diff --git a/crates/macro_service_urls/src/lib.rs b/crates/macro_service_urls/src/lib.rsindex c24066da76..c2ff8ce955 100644--- a/crates/macro_service_urls/src/lib.rs+++ b/crates/macro_service_urls/src/lib.rs@@ -518,6 +518,12 @@ service_url! {dev: "https://dev-gateway.macro.com/convert",prod: "https://gateway.macro.com/convert",},+ /// Search processing service API URL.+ pub SearchProcessingServiceUrl {+ local: "http://localhost:8092",+ dev: "https://dev-gateway.macro.com/search-processing",+ prod: "https://gateway.macro.com/search-processing",+ },/// Connection gateway HTTP API URL.pub ConnectionGatewayUrl {local: "http://localhost:8082",diff --git a/crates/macro_service_urls/src/test.rs b/crates/macro_service_urls/src/test.rsindex ba4a90a15a..39d5126efc 100644--- a/crates/macro_service_urls/src/test.rs+++ b/crates/macro_service_urls/src/test.rs@@ -52,6 +52,23 @@ fn convert_service_url_has_no_trailing_slash() {}}+#[test]+fn search_processing_service_url_parses() {+ assert_parses_for_all_environments(SearchProcessingServiceUrl::default_for_environment);+}++#[test]+fn search_processing_service_url_has_no_trailing_slash() {+ for environment in ENVS {+ let url = SearchProcessingServiceUrl::default_for_environment(environment);+ assert!(+ !url.as_ref().ends_with('/'),+ "clients concatenate paths, so {} must not end with /",+ url.as_ref()+ );+ }+}+#[test]fn connection_gateway_url_parses() {assert_parses_for_all_environments(ConnectionGatewayUrl::default_for_environment);@@ -276,6 +293,10 @@ fn exported_service_urls_match_local_values() {service_urls.convert_service_url.as_ref(),"http://localhost:8080",);+ assert_eq!(+ service_urls.search_processing_service_url.as_ref(),+ "http://localhost:8092",+ );assert_eq!(service_urls.connection_gateway_url.as_ref(),"http://localhost:8082",@@ -338,6 +359,10 @@ fn exported_service_urls_match_dev_values() {service_urls.convert_service_url.as_ref(),"https://dev-gateway.macro.com/convert",);+ assert_eq!(+ service_urls.search_processing_service_url.as_ref(),+ "https://dev-gateway.macro.com/search-processing",+ );assert_eq!(service_urls.connection_gateway_url.as_ref(),"https://connection-gateway-dev.macro.com",@@ -397,6 +422,10 @@ fn exported_service_urls_match_prod_values() {service_urls.convert_service_url.as_ref(),"https://gateway.macro.com/convert",);+ assert_eq!(+ service_urls.search_processing_service_url.as_ref(),+ "https://gateway.macro.com/search-processing",+ );assert_eq!(service_urls.connection_gateway_url.as_ref(),"https://connection-gateway.macro.com",@@ -457,6 +486,10 @@ fn exported_service_url_override_names_are_derived_from_env_var_names() {ConvertServiceUrl::local().override_env_var_name(),"OVERRIDE_CONVERT_SERVICE_URL",);+ assert_eq!(+ SearchProcessingServiceUrl::local().override_env_var_name(),+ "OVERRIDE_SEARCH_PROCESSING_SERVICE_URL",+ );assert_eq!(ConnectionGatewayUrl::local().override_env_var_name(),"OVERRIDE_CONNECTION_GATEWAY_URL",diff --git a/infra/packages/shared/src/gateway_priorities.ts b/infra/packages/shared/src/gateway_priorities.tsindex a7ed43a5eb..a25fe133c0 100644--- a/infra/packages/shared/src/gateway_priorities.ts+++ b/infra/packages/shared/src/gateway_priorities.ts@@ -8,6 +8,7 @@ export enum GatewayService {CONVERT_SERVICE = 'CONVERT_SERVICE',NOTIFICATION_SERVICE = 'NOTIFICATION_SERVICE',CONTACTS_SERVICE = 'CONTACTS_SERVICE',+ SEARCH_PROCESSING_SERVICE = 'SEARCH_PROCESSING_SERVICE',}/**@@ -24,6 +25,7 @@ export const GATEWAY_PRIORITIES: GatewayPriorityMap = {[GatewayService.UNFURL_SERVICE]: 20,[GatewayService.NOTIFICATION_SERVICE]: 30,[GatewayService.CONTACTS_SERVICE]: 40,+ [GatewayService.SEARCH_PROCESSING_SERVICE]: 50,[GatewayService.CONVERT_SERVICE]: 3000,};diff --git a/infra/stacks/search-processing-service/index.ts b/infra/stacks/search-processing-service/index.tsindex 6fd0520d2e..d4eb1d864c 100644--- a/infra/stacks/search-processing-service/index.ts+++ b/infra/stacks/search-processing-service/index.ts@@ -1,6 +1,7 @@import * as pulumi from '@pulumi/pulumi';import { DynamoDBTable } from '../../packages/resources';import {+ BASE_DOMAIN,getKafkaClusterPolicy,getSearchEventQueue,stack,@@ -76,5 +77,7 @@ const searchProcessingService = new SearchProcessingService(});-export const searchProcessingServiceUrl = pulumi.interpolate`${searchProcessingService.domain}`;+export const searchProcessingServiceUrl = `https://${+ stack === 'prod' ? '' : `${stack}-`+}gateway.${BASE_DOMAIN}/search-processing`;export const searchProcessingServiceRoleArn = searchProcessingService.role.arn;diff --git a/infra/stacks/search-processing-service/service.ts b/infra/stacks/search-processing-service/service.tsindex 711620792c..cab04d7fec 100644--- a/infra/stacks/search-processing-service/service.ts+++ b/infra/stacks/search-processing-service/service.ts@@ -8,6 +8,7 @@ import {datadogAgentContainer,fargateLogRouterSidecarContainer,serviceLoadBalancer,+ ServiceTargetGroup,} from '../../packages/resources';import { EcrImage } from '../../packages/service';import {@@ -15,8 +16,12 @@ import {CLOUD_TRAIL_SNS_TOPIC_ARN,stack,DopplerEcsEnvironment,+ getGatewayAlb,+ GatewayService,} from '../../packages/shared';+const gatewayLoadBalancer = getGatewayAlb();+// NOTE: the BASE_NAME for sps does not match the pulumi project name so do not changeconst BASE_NAME = 'search-processing';const REPO_ROOT = '../../..';@@ -179,6 +184,22 @@ export class SearchProcessingService extends pulumi.ComponentResource {this.serviceAlbSg = sg.serviceAlbSg;this.serviceSg = sg.serviceSg;+ const gatewayTargetGroup = new ServiceTargetGroup(+ `${stack}-${BASE_NAME}`,+ {+ tags: this.tags,+ listenerArn: gatewayLoadBalancer.httpsListenerArn,+ vpcId: vpc.vpcId,+ containerPort: serviceContainerPort,+ service: GatewayService.SEARCH_PROCESSING_SERVICE,+ healthCheckPath,+ pathPatterns: ['/search-processing', '/search-processing/*'],+ serviceSecurityGroupId: this.serviceSg.id,+ albSecurityGroupId: gatewayLoadBalancer.albSecurityGroupId,+ },+ { parent: this }+ );+// lbconst { targetGroup, lb, listener } = serviceLoadBalancer(this, {serviceName: BASE_NAME, // service name@@ -214,6 +235,23 @@ export class SearchProcessingService extends pulumi.ComponentResource {enable: true,rollback: true,},+ // Register tasks in both the legacy ALB's target group and the gateway+ // target group while we migrate to the gateway. An explicit+ // `loadBalancers` replaces the list awsx derives from+ // `portMappings.targetGroup`, so the legacy entry must be listed here+ // too.+ loadBalancers: [+ {+ targetGroupArn: targetGroup.arn,+ containerName: 'service',+ containerPort: serviceContainerPort,+ },+ {+ targetGroupArn: gatewayTargetGroup.target_group.arn,+ containerName: 'service',+ containerPort: serviceContainerPort,+ },+ ],taskDefinitionArgs: {taskRole: {roleArn: this.role.arn,@@ -271,12 +309,19 @@ export class SearchProcessingService extends pulumi.ComponentResource {},{parent: this,+ // ECS refuses a service whose target group is not yet associated with+ // a load balancer; it is the listener rule that creates that+ // association+ dependsOn: [gatewayTargetGroup.listener_rule],});this.service = service;- this.setupAutoScaling();+ this.setupAutoScaling({+ gatewayAlbArnSuffix: gatewayLoadBalancer.albArnSuffix,+ gatewayTargetGroup: gatewayTargetGroup.target_group,+ });this.setupServiceAlarms();@@ -402,7 +447,13 @@ export class SearchProcessingService extends pulumi.ComponentResource {return { serviceAlbSg, serviceSg };}- setupAutoScaling() {+ setupAutoScaling({+ gatewayAlbArnSuffix,+ gatewayTargetGroup,+ }: {+ gatewayAlbArnSuffix: pulumi.Output<string>;+ gatewayTargetGroup: aws.lb.TargetGroup;+ }) {if (!this.service) return;const serviceScalableTarget = new aws.appautoscaling.Target(@@ -418,21 +469,8 @@ export class SearchProcessingService extends pulumi.ComponentResource {{ parent: this });- const lbPortion: pulumi.Output<string> = this.lb.arn.apply((arn) => {- const parts = arn.split(':loadbalancer/');- return parts[1];- });-- const tgPortion: pulumi.Output<string> = this.targetGroup.arn.apply(- (arn) => {- const parts = arn.split(':');- return parts[parts.length - 1];- }- );-- const resourceLabel = pulumi.interpolate`${lbPortion}/${tgPortion}`;+ const resourceLabel = pulumi.interpolate`${gatewayAlbArnSuffix}/${gatewayTargetGroup.arnSuffix}`;- // Create an Auto Scaling policy for request count.new aws.appautoscaling.Policy(`${BASE_NAME}-scaling-policy-request-count-${stack}`,{diff --git a/services/search_processing_service/src/api.rs b/services/search_processing_service/src/api.rsindex 7970501924..c76126b13c 100644--- a/services/search_processing_service/src/api.rs+++ b/services/search_processing_service/src/api.rs@@ -15,6 +15,11 @@ mod internal;pub(crate) mod context;mod swagger;+#[cfg(test)]+mod test;++const GATEWAY_PATH_PREFIX: &str = "/search-processing";+pub async fn setup_and_serve(state: ApiContext,shutdown_token: CancellationToken,@@ -30,13 +35,17 @@ async fn serve(state: ApiContext, shutdown_token: CancellationToken) -> anyhow::let port = state.config.port;let env = state.config.environment;let backfill_jobs = state.backfill_jobs.clone();- let app = api_router()+ let traced_api = api_router().with_state(state).layer(cors.clone())- .layer(ServiceBuilder::new().layer(TraceLayer::new_for_http()))- // The health router is attached here so we don't attach the logging middleware to it- .merge(health::router().layer(cors))- .merge(SwaggerUi::new("/docs").url("/api-doc/openapi.json", swagger::ApiDoc::openapi()));+ .layer(ServiceBuilder::new().layer(TraceLayer::new_for_http()));+ let health = health::router().layer(cors);+ let app = mount_at_root_and_prefix(traced_api.merge(health))+ .merge(SwaggerUi::new("/docs").url("/api-doc/openapi.json", swagger::ApiDoc::openapi()))+ .merge(SwaggerUi::new(format!("{GATEWAY_PATH_PREFIX}/docs")).url(+ format!("{GATEWAY_PATH_PREFIX}/api-doc/openapi.json"),+ swagger::ApiDoc::openapi(),+ ));let listener = tokio::net::TcpListener::bind(format!("0.0.0.0:{port}")).await@@ -93,6 +102,12 @@ async fn shutdown_signal(shutdown_token.cancel();}+fn mount_at_root_and_prefix(inner: Router) -> Router {+ Router::new()+ .merge(inner.clone())+ .nest(GATEWAY_PATH_PREFIX, inner)+}+fn api_router() -> Router<ApiContext> {Router::new().nest("/internal",diff --git a/services/search_processing_service/src/api/test.rs b/services/search_processing_service/src/api/test.rsnew file mode 100644index 0000000000..3da7f89632--- /dev/null+++ b/services/search_processing_service/src/api/test.rs@@ -0,0 +1,40 @@+use super::{health, mount_at_root_and_prefix};+use axum::{+ body::Body,+ http::{Request, StatusCode},+};+use tower::ServiceExt;++#[tokio::test]+async fn health_is_reachable_at_root_and_gateway_prefix() {+ for path in ["/health", "/search-processing/health"] {+ let response = mount_at_root_and_prefix(health::router())+ .oneshot(+ Request::builder()+ .uri(path)+ .method("GET")+ .body(Body::empty())+ .unwrap(),+ )+ .await+ .unwrap();++ assert_eq!(response.status(), StatusCode::OK, "{path}");+ }+}++#[tokio::test]+async fn unprefixed_unknown_path_is_not_rewritten_onto_the_prefix() {+ let response = mount_at_root_and_prefix(health::router())+ .oneshot(+ Request::builder()+ .uri("/missing")+ .method("GET")+ .body(Body::empty())+ .unwrap(),+ )+ .await+ .unwrap();++ assert_eq!(response.status(), StatusCode::NOT_FOUND);+}diff --git a/services/sync-service/tests/utils.ts b/services/sync-service/tests/utils.tsindex bf7715554c..56d0ec4c71 100644--- a/services/sync-service/tests/utils.ts+++ b/services/sync-service/tests/utils.ts@@ -56,7 +56,7 @@ export async function setupMiniflare() {INTERNAL_API_SECRET_KEY: "INTERNAL_API_SECRET",INTERNAL_API_SECRET,SPS_API_SECRET_KEY: "local",- SPS_URL:"http://localhost:8090",+ SPS_URL: "http://localhost:8092",local:true,},compatibilityDate: '2025-03-05'diff --git a/services/sync-service/wrangler.docker.toml b/services/sync-service/wrangler.docker.tomlindex 68c1342960..5dfe9393b2 100644--- a/services/sync-service/wrangler.docker.toml+++ b/services/sync-service/wrangler.docker.toml@@ -59,7 +59,7 @@ name = "sync-service-dev3"[env.dev.vars]INTERNAL_API_SECRET_KEY = "SYNC_SERVICE_KEY_DEV"ENVIRONMENT = "development"-SPS_URL = "https://search-processing-dev.macro.com"+SPS_URL = "https://dev-gateway.macro.com/search-processing"[env.dev.durable_objects]bindings = [@@ -96,7 +96,7 @@ name = "sync-service-prod2"[env.prod.vars]INTERNAL_API_SECRET_KEY = "SYNC_SERVICE_KEY_PROD"ENVIRONMENT = "production"-SPS_URL = "https://search-processing.macro.com"+SPS_URL = "https://gateway.macro.com/search-processing"[env.prod.durable_objects]bindings = [diff --git a/services/sync-service/wrangler.toml b/services/sync-service/wrangler.tomlindex 1f917b57b7..434468990e 100644--- a/services/sync-service/wrangler.toml+++ b/services/sync-service/wrangler.toml@@ -31,7 +31,7 @@ bucket_name = "document-snapshot-bucket-dev"[vars]INTERNAL_API_SECRET_KEY = "LOCAL_API_KEY"-SPS_URL = "https://search-processing-dev.macro.com"+SPS_URL = "http://localhost:8092"DSS_URL = "http://localhost:8086"DSS_INTERNAL_AUTH_KEY = "local"# Tags exported spans with deployment.environment=local, matching the@@ -61,7 +61,7 @@ name = "sync-service-playground"INTERNAL_API_SECRET_KEY = "SYNC_SERVICE_KEY_PLAYGROUND"DOCUMENT_PERMISSIONS_SECRET = "local"ENVIRONMENT = "development"-SPS_URL = "https://search-processing-dev.macro.com"+SPS_URL = "https://dev-gateway.macro.com/search-processing"DSS_URL = "https://dev-gateway.macro.com/dss"[env.playground.durable_objects]@@ -101,7 +101,7 @@ INTERNAL_API_SECRET_KEY = "SYNC_SERVICE_KEY_DEV"ENVIRONMENT = "development"# The proxy injects the Datadog key; the exporter appends the signal path.OTEL_EXPORTER_OTLP_ENDPOINT = "https://macro-prox-dev.macroverse.workers.dev/i/otlp"-SPS_URL = "https://search-processing-dev.macro.com"+SPS_URL = "https://dev-gateway.macro.com/search-processing"DSS_URL = "https://dev-gateway.macro.com/dss"[env.dev.durable_objects]@@ -141,7 +141,7 @@ INTERNAL_API_SECRET_KEY = "SYNC_SERVICE_KEY_PROD"ENVIRONMENT = "production"# The proxy injects the Datadog key; the exporter appends the signal path.OTEL_EXPORTER_OTLP_ENDPOINT = "https://macro-prox-prod.macroverse.workers.dev/i/otlp"-SPS_URL = "https://search-processing.macro.com"+SPS_URL = "https://gateway.macro.com/search-processing"DSS_URL = "https://gateway.macro.com/dss"[env.prod.durable_objects]