Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
feat(convert-service): dual-mount convert-service on the shared gateway (#6131)
Will Hutchinson85b955799890
diff --git a/crates/macro_service_urls/src/lib.rs b/crates/macro_service_urls/src/lib.rsindex 48ee282daa..f0534e9923 100644--- a/crates/macro_service_urls/src/lib.rs+++ b/crates/macro_service_urls/src/lib.rs@@ -512,6 +512,12 @@ service_url! {dev: "https://dev-gateway.macro.com/dss",prod: "https://gateway.macro.com/dss",},+ /// Convert service API URL.+ pub ConvertServiceUrl {+ local: "http://localhost:8080",+ dev: "https://dev-gateway.macro.com/convert",+ prod: "https://gateway.macro.com/convert",+ },/// Connection gateway HTTP API URL.pub ConnectionGatewayUrl {local: "http://localhost:8082",diff --git a/crates/macro_service_urls/src/test.rs b/crates/macro_service_urls/src/test.rsindex 3cc7044841..ec5c346236 100644--- a/crates/macro_service_urls/src/test.rs+++ b/crates/macro_service_urls/src/test.rs@@ -35,6 +35,23 @@ fn document_storage_service_url_parses() {assert_parses_for_all_environments(DocumentStorageServiceUrl::default_for_environment);}+#[test]+fn convert_service_url_parses() {+ assert_parses_for_all_environments(ConvertServiceUrl::default_for_environment);+}++#[test]+fn convert_service_url_has_no_trailing_slash() {+ for environment in ENVS {+ let url = ConvertServiceUrl::default_for_environment(environment);+ assert!(+ !url.as_ref().ends_with('/'),+ "clients concatenate paths, so {} must not end with /",+ url.as_ref()+ );+ }+}+#[test]fn connection_gateway_url_parses() {assert_parses_for_all_environments(ConnectionGatewayUrl::default_for_environment);@@ -243,6 +260,10 @@ fn exported_service_urls_match_local_values() {service_urls.document_storage_service_url.as_ref(),"http://localhost:8086",);+ assert_eq!(+ service_urls.convert_service_url.as_ref(),+ "http://localhost:8080",+ );assert_eq!(service_urls.connection_gateway_url.as_ref(),"http://localhost:8082",@@ -301,6 +322,10 @@ fn exported_service_urls_match_dev_values() {service_urls.document_storage_service_url.as_ref(),"https://dev-gateway.macro.com/dss",);+ assert_eq!(+ service_urls.convert_service_url.as_ref(),+ "https://dev-gateway.macro.com/convert",+ );assert_eq!(service_urls.connection_gateway_url.as_ref(),"https://connection-gateway-dev.macro.com",@@ -356,6 +381,10 @@ fn exported_service_urls_match_prod_values() {service_urls.document_storage_service_url.as_ref(),"https://gateway.macro.com/dss",);+ assert_eq!(+ service_urls.convert_service_url.as_ref(),+ "https://gateway.macro.com/convert",+ );assert_eq!(service_urls.connection_gateway_url.as_ref(),"https://connection-gateway.macro.com",@@ -412,6 +441,10 @@ fn exported_service_url_override_names_are_derived_from_env_var_names() {DocumentStorageServiceUrl::local().override_env_var_name(),"OVERRIDE_DOCUMENT_STORAGE_SERVICE_URL",);+ assert_eq!(+ ConvertServiceUrl::local().override_env_var_name(),+ "OVERRIDE_CONVERT_SERVICE_URL",+ );assert_eq!(ConnectionGatewayUrl::local().override_env_var_name(),"OVERRIDE_CONNECTION_GATEWAY_URL",diff --git a/infra/packages/shared/src/gateway_priorities.ts b/infra/packages/shared/src/gateway_priorities.tsindex da442dd587..393b25b59a 100644--- a/infra/packages/shared/src/gateway_priorities.ts+++ b/infra/packages/shared/src/gateway_priorities.ts@@ -5,6 +5,7 @@export enum GatewayService {DOCUMENT_STORAGE_SERVICE = 'DOCUMENT_STORAGE_SERVICE',UNFURL_SERVICE = 'UNFURL_SERVICE',+ CONVERT_SERVICE = 'CONVERT_SERVICE',}/**@@ -19,6 +20,7 @@ type GatewayPriorityMap = { [K in GatewayService]: number };export const GATEWAY_PRIORITIES: GatewayPriorityMap = {[GatewayService.DOCUMENT_STORAGE_SERVICE]: 10,[GatewayService.UNFURL_SERVICE]: 20,+ [GatewayService.CONVERT_SERVICE]: 3000,};const assigned = Object.values(GATEWAY_PRIORITIES);diff --git a/infra/stacks/convert-service/index.ts b/infra/stacks/convert-service/index.tsindex e4eb70d6db..e5780d3992 100644--- a/infra/stacks/convert-service/index.ts+++ b/infra/stacks/convert-service/index.ts@@ -1,6 +1,6 @@import * as pulumi from '@pulumi/pulumi';import { Queue } from '../../packages/resources';-import { stack } from '../../packages/shared';+import { BASE_DOMAIN, stack } from '../../packages/shared';import { get_coparse_api_vpc } from '../../packages/vpc';import { ConvertService } from './service';@@ -83,4 +83,6 @@ const convertService = new ConvertService('convert-service', {});export const convertServiceRoleArn = pulumi.interpolate`${convertService.role.arn}`;-export const convertServiceUrl = pulumi.interpolate`${convertService.domain}`;+export const convertServiceUrl = `https://${+ stack === 'prod' ? '' : `${stack}-`+}gateway.${BASE_DOMAIN}/convert`;diff --git a/infra/stacks/convert-service/service.ts b/infra/stacks/convert-service/service.tsindex fe1aa7421b..13722dfd56 100644--- a/infra/stacks/convert-service/service.ts+++ b/infra/stacks/convert-service/service.ts@@ -8,15 +8,20 @@ import {datadogAgentContainer,fargateLogRouterSidecarContainer,serviceLoadBalancer,+ ServiceTargetGroup,} from '../../packages/resources';import { EcrImage } from '../../packages/service';import {BASE_DOMAIN,CLOUD_TRAIL_SNS_TOPIC_ARN,DopplerEcsEnvironment,+ getGatewayAlb,+ GatewayService,stack,} from '../../packages/shared';+const gatewayLoadBalancer = getGatewayAlb();+const BASE_NAME = pulumi.getProject();const REPO_ROOT = '../../..';export const SERVICE_DOMAIN_NAME = `convert-service${@@ -164,6 +169,22 @@ export class ConvertService extends pulumi.ComponentResource {this.serviceAlbSg = sg.serviceAlbSg;this.serviceSg = sg.serviceSg;+ const gatewayTargetGroup = new ServiceTargetGroup(+ `${stack}-${BASE_NAME}`,+ {+ tags: this.tags,+ listenerArn: gatewayLoadBalancer.httpsListenerArn,+ vpcId: vpc.vpcId,+ containerPort: serviceContainerPort,+ service: GatewayService.CONVERT_SERVICE,+ healthCheckPath,+ pathPatterns: ['/convert', '/convert/*'],+ serviceSecurityGroupId: this.serviceSg.id,+ albSecurityGroupId: gatewayLoadBalancer.albSecurityGroupId,+ },+ { parent: this }+ );+// Load balancerconst { targetGroup, lb, listener } = serviceLoadBalancer(this, {serviceName: BASE_NAME, // service name@@ -199,6 +220,23 @@ export class ConvertService extends pulumi.ComponentResource {enable: true,rollback: true,},+ // Register tasks in both the legacy ALB's target group and the gateway+ // target group while we migrate to the gateway. An explicit+ // `loadBalancers` replaces the list awsx derives from+ // `portMappings.targetGroup`, so the legacy entry must be listed here+ // too.+ loadBalancers: [+ {+ targetGroupArn: targetGroup.arn,+ containerName: 'service',+ containerPort: serviceContainerPort,+ },+ {+ targetGroupArn: gatewayTargetGroup.target_group.arn,+ containerName: 'service',+ containerPort: serviceContainerPort,+ },+ ],taskDefinitionArgs: {taskRole: {roleArn: this.role.arn,@@ -259,12 +297,19 @@ export class ConvertService extends pulumi.ComponentResource {},{parent: this,+ // ECS refuses a service whose target group is not yet associated with+ // a load balancer; it is the listener rule that creates that+ // association+ dependsOn: [gatewayTargetGroup.listener_rule],});this.service = service;// auto-scaling and alarm notifications- this.setupAutoScaling();+ this.setupAutoScaling({+ gatewayAlbArnSuffix: gatewayLoadBalancer.albArnSuffix,+ gatewayTargetGroup: gatewayTargetGroup.target_group,+ });this.setupServiceAlarms();// DNS@@ -389,7 +434,13 @@ export class ConvertService extends pulumi.ComponentResource {return { serviceAlbSg, serviceSg };}- setupAutoScaling() {+ setupAutoScaling({+ gatewayAlbArnSuffix,+ gatewayTargetGroup,+ }: {+ gatewayAlbArnSuffix: pulumi.Output<string>;+ gatewayTargetGroup: aws.lb.TargetGroup;+ }) {if (!this.service) return;const serviceScalableTarget = new aws.appautoscaling.Target(@@ -405,21 +456,8 @@ export class ConvertService extends pulumi.ComponentResource {{ parent: this });- const lbPortion: pulumi.Output<string> = this.lb.arn.apply((arn) => {- const parts = arn.split(':loadbalancer/');- return parts[1];- });-- const tgPortion: pulumi.Output<string> = this.targetGroup.arn.apply(- (arn) => {- const parts = arn.split(':');- return parts[parts.length - 1];- }- );-- const resourceLabel = pulumi.interpolate`${lbPortion}/${tgPortion}`;+ const resourceLabel = pulumi.interpolate`${gatewayAlbArnSuffix}/${gatewayTargetGroup.arnSuffix}`;- // Create an Auto Scaling policy for request count.new aws.appautoscaling.Policy(`${BASE_NAME}-scaling-policy-request-count-${stack}`,{diff --git a/services/convert_service/helpers/README.md b/services/convert_service/helpers/README.mdindex 48132558d2..41e83edcbd 100644--- a/services/convert_service/helpers/README.md+++ b/services/convert_service/helpers/README.md@@ -10,7 +10,7 @@ The backfill script is used to backfill the Convert Service with docx files.To run the backfill script, you need to provide the following environment variables:-- `BASE_URL`: The base URL of the Convert Service.+- `BASE_URL`: Convert service origin with no trailing slash. On the gateway that is `https://dev-gateway.macro.com/convert` or `https://gateway.macro.com/convert`. The legacy host still works during cutover.- `INTERNAL_API_KEY`: The internal API key for the Convert Service.- `LIMIT`: The maximum number of documents to backfill.- `OFFSET`: The offset to start backfilling from.diff --git a/services/convert_service/load-test/artillery.yml b/services/convert_service/load-test/artillery.ymlindex c0acdbfb7d..af38cc74c9 100644--- a/services/convert_service/load-test/artillery.yml+++ b/services/convert_service/load-test/artillery.yml@@ -1,5 +1,5 @@config:- target: "https://convert-service-dev.macro.com"+ target: "https://dev-gateway.macro.com"processor: "./artillery-function.js"phases:- duration: 10@@ -18,7 +18,7 @@ scenarios:flow:- function: "generateRandomData"- post:- url: "/internal/convert"+ url: "/convert/internal/convert"json:from_bucket: "doc-storage-dev"to_bucket: "doc-storage-dev"diff --git a/services/convert_service/src/api.rs b/services/convert_service/src/api.rsindex 952704af3d..a3656faa5b 100644--- a/services/convert_service/src/api.rs+++ b/services/convert_service/src/api.rs@@ -16,18 +16,30 @@ mod health;// Miscmod swagger;+#[cfg(test)]+mod test;++/// Path prefix the shared gateway ALB forwards unmodified. Dual-mounted+/// alongside `/` so the dedicated ALB keeps working during cutover.+const GATEWAY_PATH_PREFIX: &str = "/convert";+pub async fn setup_and_serve(state: ApiContext) -> anyhow::Result<()> {let cors = macro_cors::cors_layer();let port = state.config.port;let env = state.config.environment;- let app = api_router()+ let traced_api = api_router().with_state(state).layer(cors.clone())- .layer(ServiceBuilder::new().layer(TraceLayer::new_for_http()))- // The health router is attached here so we don't attach the logging middleware to it- .merge(health::router().layer(cors))- .merge(SwaggerUi::new("/docs").url("/api-doc/openapi.json", swagger::ApiDoc::openapi()));+ .layer(ServiceBuilder::new().layer(TraceLayer::new_for_http()));+ // Health stays outside TraceLayer so the probe does not generate request logs.+ let health = health::router().layer(cors);+ let app = mount_at_root_and_prefix(traced_api.merge(health))+ .merge(SwaggerUi::new("/docs").url("/api-doc/openapi.json", swagger::ApiDoc::openapi()))+ .merge(+ SwaggerUi::new("/convert/docs")+ .url("/convert/api-doc/openapi.json", swagger::ApiDoc::openapi()),+ );let listener = tokio::net::TcpListener::bind(format!("0.0.0.0:{}", port)).await@@ -42,6 +54,12 @@ pub async fn setup_and_serve(state: ApiContext) -> anyhow::Result<()> {.context("error starting service")}+fn mount_at_root_and_prefix(inner: Router) -> Router {+ Router::new()+ .merge(inner.clone())+ .nest(GATEWAY_PATH_PREFIX, inner)+}+fn api_router() -> Router<ApiContext> {Router::new().nest("/internal",diff --git a/services/convert_service/src/api/test.rs b/services/convert_service/src/api/test.rsnew file mode 100644index 0000000000..338257befe--- /dev/null+++ b/services/convert_service/src/api/test.rs@@ -0,0 +1,40 @@+use super::{health, mount_at_root_and_prefix};+use axum::{+ body::Body,+ http::{Request, StatusCode},+};+use tower::ServiceExt;++#[tokio::test]+async fn health_is_reachable_at_root_and_gateway_prefix() {+ for path in ["/health", "/convert/health"] {+ let response = mount_at_root_and_prefix(health::router())+ .oneshot(+ Request::builder()+ .uri(path)+ .method("GET")+ .body(Body::empty())+ .unwrap(),+ )+ .await+ .unwrap();++ assert_eq!(response.status(), StatusCode::OK, "{path}");+ }+}++#[tokio::test]+async fn unprefixed_unknown_path_is_not_rewritten_onto_the_prefix() {+ let response = mount_at_root_and_prefix(health::router())+ .oneshot(+ Request::builder()+ .uri("/missing")+ .method("GET")+ .body(Body::empty())+ .unwrap(),+ )+ .await+ .unwrap();++ assert_eq!(response.status(), StatusCode::NOT_FOUND);+}