Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
import * as pulumi from '@pulumi/pulumi';
// Where every monitor in this stack notifies: GoAlert, and nothing else.
//
// GoAlert (infra/stacks/observability) has two services. Notify posts each
// alert to the Macro Alerts channel and re-posts it hourly until someone
// acknowledges it; the paging service also phones the on-call engineer.
// Datadog never posts to the channel itself, so an alert shows up once, from
// GoAlert. The one exception is the GoAlert health check in goalert.ts.
//
// Datadog sends warning and alert notifications again every RENOTIFY_MINUTES
// while a monitor stays in that state. The repeat dedups into the open GoAlert
// alert, and reopens it if someone closed it while the monitor still fires.
// Declared in goalert.ts.
export const GOALERT = '@webhook-goalert';
export const GOALERT_CLOSE = '@webhook-goalert-close';
export const GOALERT_NOTIFY = '@webhook-goalert-notify';
export const GOALERT_NOTIFY_CLOSE = '@webhook-goalert-notify-close';
export const RENOTIFY_MINUTES = 60;
/**
* The notification handles that end every monitor message. A paging monitor's
* critical alert opens a page; every other warning, alert and no-data
* notification opens a Notify alert.
*
* Datadog allows one state block per state, never nested, so each state gets
* its own. A recovery closes on both services because a paging monitor may
* have warned before it alerted, or recovered from no data; closing an alert
* GoAlert does not have open does nothing.
*/
export function notify({
pages,
environment = 'prod',
}: {
pages: boolean;
/** Dev and staging monitors stay visible in Datadog but notify no one: on-call is for prod. */
environment?: 'prod' | 'dev' | 'staging';
}): string {
if (environment !== 'prod') return '';
const critical = pages
? `{{#is_alert}}Paging on-call. ${GOALERT}{{/is_alert}}`
: `{{#is_alert}}${GOALERT_NOTIFY}{{/is_alert}}`;
const recovery = pages
? `${GOALERT_CLOSE} ${GOALERT_NOTIFY_CLOSE}`
: GOALERT_NOTIFY_CLOSE;
return `${critical}{{#is_warning}}${GOALERT_NOTIFY}{{/is_warning}}{{#is_no_data}}${GOALERT_NOTIFY}{{/is_no_data}}{{#is_recovery}}${recovery}{{/is_recovery}}`;
}
// Datadog's site: dashboard `url`s are paths under it.
export const DATADOG_SITE = 'https://us5.datadoghq.com';
/**
* A monitor message: the state and value against its threshold, what it
* means, where to start, and the notification handles.
*/
export function alertMessage({
description,
runbook,
dashboard,
pages,
alertDetail = '{{value}} (threshold {{threshold}})',
warningDetail = '{{value}} (warning threshold {{warn_threshold}})',
recoveryDetail = '{{value}}',
}: {
description: string;
runbook: string;
/** A dashboard's `url`, a path such as `/dashboard/abc-def-ghi/title`. */
dashboard?: pulumi.Input<string>;
pages: boolean;
/** Text after `CRITICAL:` in the state line. */
alertDetail?: string;
/** Text after `WARNING:` in the state line. */
warningDetail?: string;
/** Text after `RECOVERED:` in the state line. */
recoveryDetail?: string;
}): pulumi.Output<string> {
return pulumi.output(dashboard).apply((path) => {
const links = [
`Runbook: https://github.com/macro-inc/macro/blob/main/${runbook}`,
];
if (path !== undefined) links.push(`Dashboard: ${DATADOG_SITE}${path}`);
return `{{#is_alert}}🔴 CRITICAL: ${alertDetail}{{/is_alert}}{{#is_warning}}🟠 WARNING: ${warningDetail}{{/is_warning}}{{#is_recovery}}✅ RECOVERED: ${recoveryDetail}{{/is_recovery}}
${description}
${links.join('\n')}
${notify({ pages })}`;
});
}