Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

feat(convert-service): dual-mount convert-service on the shared gateway (#6131)

Will Hutchinson85b955799890
diff --git a/crates/macro_service_urls/src/lib.rs b/crates/macro_service_urls/src/lib.rs
index 48ee282daa..f0534e9923 100644
--- a/crates/macro_service_urls/src/lib.rs
+++ b/crates/macro_service_urls/src/lib.rs
@@ -512,6 +512,12 @@ service_url! {
dev: "https://dev-gateway.macro.com/dss",
prod: "https://gateway.macro.com/dss",
},
+ /// Convert service API URL.
+ pub ConvertServiceUrl {
+ local: "http://localhost:8080",
+ dev: "https://dev-gateway.macro.com/convert",
+ prod: "https://gateway.macro.com/convert",
+ },
/// Connection gateway HTTP API URL.
pub ConnectionGatewayUrl {
local: "http://localhost:8082",
diff --git a/crates/macro_service_urls/src/test.rs b/crates/macro_service_urls/src/test.rs
index 3cc7044841..ec5c346236 100644
--- a/crates/macro_service_urls/src/test.rs
+++ b/crates/macro_service_urls/src/test.rs
@@ -35,6 +35,23 @@ fn document_storage_service_url_parses() {
assert_parses_for_all_environments(DocumentStorageServiceUrl::default_for_environment);
}
+#[test]
+fn convert_service_url_parses() {
+ assert_parses_for_all_environments(ConvertServiceUrl::default_for_environment);
+}
+
+#[test]
+fn convert_service_url_has_no_trailing_slash() {
+ for environment in ENVS {
+ let url = ConvertServiceUrl::default_for_environment(environment);
+ assert!(
+ !url.as_ref().ends_with('/'),
+ "clients concatenate paths, so {} must not end with /",
+ url.as_ref()
+ );
+ }
+}
+
#[test]
fn connection_gateway_url_parses() {
assert_parses_for_all_environments(ConnectionGatewayUrl::default_for_environment);
@@ -243,6 +260,10 @@ fn exported_service_urls_match_local_values() {
service_urls.document_storage_service_url.as_ref(),
"http://localhost:8086",
);
+ assert_eq!(
+ service_urls.convert_service_url.as_ref(),
+ "http://localhost:8080",
+ );
assert_eq!(
service_urls.connection_gateway_url.as_ref(),
"http://localhost:8082",
@@ -301,6 +322,10 @@ fn exported_service_urls_match_dev_values() {
service_urls.document_storage_service_url.as_ref(),
"https://dev-gateway.macro.com/dss",
);
+ assert_eq!(
+ service_urls.convert_service_url.as_ref(),
+ "https://dev-gateway.macro.com/convert",
+ );
assert_eq!(
service_urls.connection_gateway_url.as_ref(),
"https://connection-gateway-dev.macro.com",
@@ -356,6 +381,10 @@ fn exported_service_urls_match_prod_values() {
service_urls.document_storage_service_url.as_ref(),
"https://gateway.macro.com/dss",
);
+ assert_eq!(
+ service_urls.convert_service_url.as_ref(),
+ "https://gateway.macro.com/convert",
+ );
assert_eq!(
service_urls.connection_gateway_url.as_ref(),
"https://connection-gateway.macro.com",
@@ -412,6 +441,10 @@ fn exported_service_url_override_names_are_derived_from_env_var_names() {
DocumentStorageServiceUrl::local().override_env_var_name(),
"OVERRIDE_DOCUMENT_STORAGE_SERVICE_URL",
);
+ assert_eq!(
+ ConvertServiceUrl::local().override_env_var_name(),
+ "OVERRIDE_CONVERT_SERVICE_URL",
+ );
assert_eq!(
ConnectionGatewayUrl::local().override_env_var_name(),
"OVERRIDE_CONNECTION_GATEWAY_URL",
diff --git a/infra/packages/shared/src/gateway_priorities.ts b/infra/packages/shared/src/gateway_priorities.ts
index da442dd587..393b25b59a 100644
--- a/infra/packages/shared/src/gateway_priorities.ts
+++ b/infra/packages/shared/src/gateway_priorities.ts
@@ -5,6 +5,7 @@
export enum GatewayService {
DOCUMENT_STORAGE_SERVICE = 'DOCUMENT_STORAGE_SERVICE',
UNFURL_SERVICE = 'UNFURL_SERVICE',
+ CONVERT_SERVICE = 'CONVERT_SERVICE',
}
/**
@@ -19,6 +20,7 @@ type GatewayPriorityMap = { [K in GatewayService]: number };
export const GATEWAY_PRIORITIES: GatewayPriorityMap = {
[GatewayService.DOCUMENT_STORAGE_SERVICE]: 10,
[GatewayService.UNFURL_SERVICE]: 20,
+ [GatewayService.CONVERT_SERVICE]: 3000,
};
const assigned = Object.values(GATEWAY_PRIORITIES);
diff --git a/infra/stacks/convert-service/index.ts b/infra/stacks/convert-service/index.ts
index e4eb70d6db..e5780d3992 100644
--- a/infra/stacks/convert-service/index.ts
+++ b/infra/stacks/convert-service/index.ts
@@ -1,6 +1,6 @@
import * as pulumi from '@pulumi/pulumi';
import { Queue } from '../../packages/resources';
-import { stack } from '../../packages/shared';
+import { BASE_DOMAIN, stack } from '../../packages/shared';
import { get_coparse_api_vpc } from '../../packages/vpc';
import { ConvertService } from './service';
@@ -83,4 +83,6 @@ const convertService = new ConvertService('convert-service', {
});
export const convertServiceRoleArn = pulumi.interpolate`${convertService.role.arn}`;
-export const convertServiceUrl = pulumi.interpolate`${convertService.domain}`;
+export const convertServiceUrl = `https://${
+ stack === 'prod' ? '' : `${stack}-`
+}gateway.${BASE_DOMAIN}/convert`;
diff --git a/infra/stacks/convert-service/service.ts b/infra/stacks/convert-service/service.ts
index fe1aa7421b..13722dfd56 100644
--- a/infra/stacks/convert-service/service.ts
+++ b/infra/stacks/convert-service/service.ts
@@ -8,15 +8,20 @@ import {
datadogAgentContainer,
fargateLogRouterSidecarContainer,
serviceLoadBalancer,
+ ServiceTargetGroup,
} from '../../packages/resources';
import { EcrImage } from '../../packages/service';
import {
BASE_DOMAIN,
CLOUD_TRAIL_SNS_TOPIC_ARN,
DopplerEcsEnvironment,
+ getGatewayAlb,
+ GatewayService,
stack,
} from '../../packages/shared';
+const gatewayLoadBalancer = getGatewayAlb();
+
const BASE_NAME = pulumi.getProject();
const REPO_ROOT = '../../..';
export const SERVICE_DOMAIN_NAME = `convert-service${
@@ -164,6 +169,22 @@ export class ConvertService extends pulumi.ComponentResource {
this.serviceAlbSg = sg.serviceAlbSg;
this.serviceSg = sg.serviceSg;
+ const gatewayTargetGroup = new ServiceTargetGroup(
+ `${stack}-${BASE_NAME}`,
+ {
+ tags: this.tags,
+ listenerArn: gatewayLoadBalancer.httpsListenerArn,
+ vpcId: vpc.vpcId,
+ containerPort: serviceContainerPort,
+ service: GatewayService.CONVERT_SERVICE,
+ healthCheckPath,
+ pathPatterns: ['/convert', '/convert/*'],
+ serviceSecurityGroupId: this.serviceSg.id,
+ albSecurityGroupId: gatewayLoadBalancer.albSecurityGroupId,
+ },
+ { parent: this }
+ );
+
// Load balancer
const { targetGroup, lb, listener } = serviceLoadBalancer(this, {
serviceName: BASE_NAME, // service name
@@ -199,6 +220,23 @@ export class ConvertService extends pulumi.ComponentResource {
enable: true,
rollback: true,
},
+ // Register tasks in both the legacy ALB's target group and the gateway
+ // target group while we migrate to the gateway. An explicit
+ // `loadBalancers` replaces the list awsx derives from
+ // `portMappings.targetGroup`, so the legacy entry must be listed here
+ // too.
+ loadBalancers: [
+ {
+ targetGroupArn: targetGroup.arn,
+ containerName: 'service',
+ containerPort: serviceContainerPort,
+ },
+ {
+ targetGroupArn: gatewayTargetGroup.target_group.arn,
+ containerName: 'service',
+ containerPort: serviceContainerPort,
+ },
+ ],
taskDefinitionArgs: {
taskRole: {
roleArn: this.role.arn,
@@ -259,12 +297,19 @@ export class ConvertService extends pulumi.ComponentResource {
},
{
parent: this,
+ // ECS refuses a service whose target group is not yet associated with
+ // a load balancer; it is the listener rule that creates that
+ // association
+ dependsOn: [gatewayTargetGroup.listener_rule],
}
);
this.service = service;
// auto-scaling and alarm notifications
- this.setupAutoScaling();
+ this.setupAutoScaling({
+ gatewayAlbArnSuffix: gatewayLoadBalancer.albArnSuffix,
+ gatewayTargetGroup: gatewayTargetGroup.target_group,
+ });
this.setupServiceAlarms();
// DNS
@@ -389,7 +434,13 @@ export class ConvertService extends pulumi.ComponentResource {
return { serviceAlbSg, serviceSg };
}
- setupAutoScaling() {
+ setupAutoScaling({
+ gatewayAlbArnSuffix,
+ gatewayTargetGroup,
+ }: {
+ gatewayAlbArnSuffix: pulumi.Output<string>;
+ gatewayTargetGroup: aws.lb.TargetGroup;
+ }) {
if (!this.service) return;
const serviceScalableTarget = new aws.appautoscaling.Target(
@@ -405,21 +456,8 @@ export class ConvertService extends pulumi.ComponentResource {
{ parent: this }
);
- const lbPortion: pulumi.Output<string> = this.lb.arn.apply((arn) => {
- const parts = arn.split(':loadbalancer/');
- return parts[1];
- });
-
- const tgPortion: pulumi.Output<string> = this.targetGroup.arn.apply(
- (arn) => {
- const parts = arn.split(':');
- return parts[parts.length - 1];
- }
- );
-
- const resourceLabel = pulumi.interpolate`${lbPortion}/${tgPortion}`;
+ const resourceLabel = pulumi.interpolate`${gatewayAlbArnSuffix}/${gatewayTargetGroup.arnSuffix}`;
- // Create an Auto Scaling policy for request count.
new aws.appautoscaling.Policy(
`${BASE_NAME}-scaling-policy-request-count-${stack}`,
{
diff --git a/services/convert_service/helpers/README.md b/services/convert_service/helpers/README.md
index 48132558d2..41e83edcbd 100644
--- a/services/convert_service/helpers/README.md
+++ b/services/convert_service/helpers/README.md
@@ -10,7 +10,7 @@ The backfill script is used to backfill the Convert Service with docx files.
To run the backfill script, you need to provide the following environment variables:
-- `BASE_URL`: The base URL of the Convert Service.
+- `BASE_URL`: Convert service origin with no trailing slash. On the gateway that is `https://dev-gateway.macro.com/convert` or `https://gateway.macro.com/convert`. The legacy host still works during cutover.
- `INTERNAL_API_KEY`: The internal API key for the Convert Service.
- `LIMIT`: The maximum number of documents to backfill.
- `OFFSET`: The offset to start backfilling from.
diff --git a/services/convert_service/load-test/artillery.yml b/services/convert_service/load-test/artillery.yml
index c0acdbfb7d..af38cc74c9 100644
--- a/services/convert_service/load-test/artillery.yml
+++ b/services/convert_service/load-test/artillery.yml
@@ -1,5 +1,5 @@
config:
- target: "https://convert-service-dev.macro.com"
+ target: "https://dev-gateway.macro.com"
processor: "./artillery-function.js"
phases:
- duration: 10
@@ -18,7 +18,7 @@ scenarios:
flow:
- function: "generateRandomData"
- post:
- url: "/internal/convert"
+ url: "/convert/internal/convert"
json:
from_bucket: "doc-storage-dev"
to_bucket: "doc-storage-dev"
diff --git a/services/convert_service/src/api.rs b/services/convert_service/src/api.rs
index 952704af3d..a3656faa5b 100644
--- a/services/convert_service/src/api.rs
+++ b/services/convert_service/src/api.rs
@@ -16,18 +16,30 @@ mod health;
// Misc
mod swagger;
+#[cfg(test)]
+mod test;
+
+/// Path prefix the shared gateway ALB forwards unmodified. Dual-mounted
+/// alongside `/` so the dedicated ALB keeps working during cutover.
+const GATEWAY_PATH_PREFIX: &str = "/convert";
+
pub async fn setup_and_serve(state: ApiContext) -> anyhow::Result<()> {
let cors = macro_cors::cors_layer();
let port = state.config.port;
let env = state.config.environment;
- let app = api_router()
+ let traced_api = api_router()
.with_state(state)
.layer(cors.clone())
- .layer(ServiceBuilder::new().layer(TraceLayer::new_for_http()))
- // The health router is attached here so we don't attach the logging middleware to it
- .merge(health::router().layer(cors))
- .merge(SwaggerUi::new("/docs").url("/api-doc/openapi.json", swagger::ApiDoc::openapi()));
+ .layer(ServiceBuilder::new().layer(TraceLayer::new_for_http()));
+ // Health stays outside TraceLayer so the probe does not generate request logs.
+ let health = health::router().layer(cors);
+ let app = mount_at_root_and_prefix(traced_api.merge(health))
+ .merge(SwaggerUi::new("/docs").url("/api-doc/openapi.json", swagger::ApiDoc::openapi()))
+ .merge(
+ SwaggerUi::new("/convert/docs")
+ .url("/convert/api-doc/openapi.json", swagger::ApiDoc::openapi()),
+ );
let listener = tokio::net::TcpListener::bind(format!("0.0.0.0:{}", port))
.await
@@ -42,6 +54,12 @@ pub async fn setup_and_serve(state: ApiContext) -> anyhow::Result<()> {
.context("error starting service")
}
+fn mount_at_root_and_prefix(inner: Router) -> Router {
+ Router::new()
+ .merge(inner.clone())
+ .nest(GATEWAY_PATH_PREFIX, inner)
+}
+
fn api_router() -> Router<ApiContext> {
Router::new().nest(
"/internal",
diff --git a/services/convert_service/src/api/test.rs b/services/convert_service/src/api/test.rs
new file mode 100644
index 0000000000..338257befe
--- /dev/null
+++ b/services/convert_service/src/api/test.rs
@@ -0,0 +1,40 @@
+use super::{health, mount_at_root_and_prefix};
+use axum::{
+ body::Body,
+ http::{Request, StatusCode},
+};
+use tower::ServiceExt;
+
+#[tokio::test]
+async fn health_is_reachable_at_root_and_gateway_prefix() {
+ for path in ["/health", "/convert/health"] {
+ let response = mount_at_root_and_prefix(health::router())
+ .oneshot(
+ Request::builder()
+ .uri(path)
+ .method("GET")
+ .body(Body::empty())
+ .unwrap(),
+ )
+ .await
+ .unwrap();
+
+ assert_eq!(response.status(), StatusCode::OK, "{path}");
+ }
+}
+
+#[tokio::test]
+async fn unprefixed_unknown_path_is_not_rewritten_onto_the_prefix() {
+ let response = mount_at_root_and_prefix(health::router())
+ .oneshot(
+ Request::builder()
+ .uri("/missing")
+ .method("GET")
+ .body(Body::empty())
+ .unwrap(),
+ )
+ .await
+ .unwrap();
+
+ assert_eq!(response.status(), StatusCode::NOT_FOUND);
+}