🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import { getBillingProvider, parseDbOrg, Role } from "@langfuse/shared";
import { prisma } from "@langfuse/shared/src/db";
import Stripe from "stripe";
import { env } from "../../env";
import { logger } from "@langfuse/shared/src/server";
import {
CHB_USAGE_CURRENCY,
ChbApiError,
chbBillPlanAmountUSD,
chbBillUsageAmountUSD,
} from "@langfuse/shared/src/server/ee/clickhouseBilling";
import { recordIncrement, traceException } from "@langfuse/shared/src/server";
import { Job } from "bullmq";
import { backOff } from "exponential-backoff";
import { sendCloudSpendAlertEmail } from "@langfuse/shared/src/server";
import { getChbApiClient } from "./chbApiClient";
/**
* Why a spend reading was unavailable. Every reason is a configuration or
* lifecycle state the job cannot act on, never an error — the org is counted
* and skipped so a gap in alerting is visible without paging.
*/
type MissingBillingConfigReason =
| "missing_stripe_customer_id"
| "missing_stripe_subscription_id"
| "missing_chb_organization_id"
| "missing_chb_attached_plan"
| "chb_not_configured"
| "chb_open_bill_not_found"
| "missing_chb_usage_amount"
| "missing_chb_plan_amount"
| "missing_chb_period_start";
const recordMissingBillingConfigSkip = (reason: MissingBillingConfigReason) => {
recordIncrement(
"langfuse.queue.cloud_spend_alert_queue.skipped_orgs_missing_billing_config",
1,
{
unit: "organizations",
reason,
},
);
};
/**
* What both providers have to produce for an alert to be evaluated: the spend
* accrued so far and the start of the period it accrued in. The period start is
* not decoration — it is what makes an alert fire once per cycle instead of on
* every run, and what resets it when the next cycle opens.
*/
type SpendReading = {
currentSpendUSD: number;
currentPeriodStart: Date;
};
type OrgWithAlerts = ReturnType<typeof parseDbOrg>;
/** Stripe's reading: the total of the subscription's preview invoice. */
const readStripeSpend = async (
org: OrgWithAlerts,
): Promise<SpendReading | null> => {
const stripeSecretKey = env.STRIPE_SECRET_KEY;
if (!stripeSecretKey) {
// A Stripe-billed org in a deployment without a Stripe key is a
// misconfiguration, not a state to skip past: throw so the job retries and
// the failure stays visible.
logger.warn("[CLOUD SPEND ALERTS] Stripe secret key not found");
throw new Error("Stripe secret key not found");
}
const stripeCustomerId = org.cloudConfig?.stripe?.customerId;
if (!stripeCustomerId) {
logger.warn(
`[CLOUD SPEND ALERTS] Stripe customer id not found for org ${org.id}`,
);
recordMissingBillingConfigSkip("missing_stripe_customer_id");
return null;
}
const stripeSubscriptionId = org.cloudConfig?.stripe?.activeSubscriptionId;
if (!stripeSubscriptionId) {
logger.warn(
`[CLOUD SPEND ALERTS] Stripe subscription id not found for org ${org.id}`,
);
recordMissingBillingConfigSkip("missing_stripe_subscription_id");
return null;
}
const stripe = new Stripe(stripeSecretKey);
// Get subscription to check billing cycle
const subscription =
await stripe.subscriptions.retrieve(stripeSubscriptionId);
// Create preview invoice to calculate current spend
const canCreateInvoicePreview = [
"active",
"past_due",
"trialing",
"unpaid",
].includes(subscription.status);
if (!canCreateInvoicePreview) {
logger.warn(
`[CLOUD SPEND ALERTS] Cannot create invoice preview for org ${org.id} - subscription status: ${subscription.status}`,
);
return null;
}
const previewInvoice = await backOff(
async () =>
await stripe.invoices.createPreview({
customer:
typeof subscription.customer === "string"
? subscription.customer
: subscription.customer?.id,
subscription: stripeSubscriptionId,
}),
{
numOfAttempts: 3,
},
);
return {
// Stripe reports invoice totals in minor units.
currentSpendUSD: (previewInvoice.total ?? 0) / 100,
currentPeriodStart: new Date(subscription.current_period_start * 1000),
};
};
/**
* CHB's reading: the accrued usage plus the plan fee on the org's open bill.
* This is the CHB equivalent of Stripe's preview invoice — CHB issues no
* preview invoice, but its billing warehouse reports what the open bill has
* accrued so far and the plan fee it carries, both in major-unit USD (see
* `chbBillUsageAmountUSD` and `chbBillPlanAmountUSD`). Like Stripe's, that fee
* is billed in advance, so it can cover the next period rather than this one.
* The warehouse, not m3ter, serves this read, which is what makes hourly
* polling affordable on CHB's side; the price is a reading up to one warehouse
* export old.
*/
const readClickhouseSpend = async (
org: OrgWithAlerts,
): Promise<SpendReading | null> => {
const chOrganizationId = org.cloudConfig?.clickhouse?.organizationId;
if (!chOrganizationId) {
logger.warn(
`[CLOUD SPEND ALERTS] ClickHouse organization id not found for org ${org.id}`,
);
recordMissingBillingConfigSkip("missing_chb_organization_id");
return null;
}
if (!org.cloudConfig?.clickhouse?.attachedPlanId) {
// Cancelled or never subscribed: nothing accrues, so nothing to alert on.
logger.info(
`[CLOUD SPEND ALERTS] Org ${org.id} has no CHB attached plan, skipping`,
);
recordMissingBillingConfigSkip("missing_chb_attached_plan");
return null;
}
const client = getChbApiClient();
if (!client) {
logger.warn(
`[CLOUD SPEND ALERTS] CHB REST client is not configured, cannot read spend for org ${org.id}`,
);
recordMissingBillingConfigSkip("chb_not_configured");
return null;
}
const bills = await backOff(
async () =>
await client.queryBills({ chOrganizationIds: [chOrganizationId] }),
{
numOfAttempts: 3,
// A 4xx is CHB's verdict on the request, not a blip: retrying it just
// spends the budget. 5xx and transport failures still get the retries.
retry: (error: unknown) =>
!(
error instanceof ChbApiError &&
error.status >= 400 &&
error.status < 500
),
},
);
const bill = bills.find(
(candidate) => candidate.organizationId === chOrganizationId,
);
if (!bill) {
// CHB leaves an org without an open bill out of the result: its plan ended
// after the fan-out read it, or it is newer than the warehouse's last
// export. Either way nothing has accrued that we could compare.
logger.info(
`[CLOUD SPEND ALERTS] CHB reports no open bill for org ${org.id}, skipping`,
);
recordMissingBillingConfigSkip("chb_open_bill_not_found");
return null;
}
// Logged raw as well as converted: this is the one place the major-unit
// reading of CHB's amount is observable in production, so a contract change
// shows up here rather than as alerts that quietly stop matching reality.
logger.info(
`[CLOUD SPEND ALERTS] Org ${org.id} CHB bill ${bill.id} reports accrued usage ${bill.usage?.amount} ${bill.usage?.currency ?? CHB_USAGE_CURRENCY} as of ${bill.usage?.asOf} and plan fee ${bill.plan ? `${bill.plan.amount} ${bill.plan.currency ?? CHB_USAGE_CURRENCY}` : "none"} for period ${bill.startDate} - ${bill.endDate}`,
);
const usageUSD = chbBillUsageAmountUSD(bill);
if (usageUSD === null) {
logger.warn(
`[CLOUD SPEND ALERTS] CHB reported no usable ${CHB_USAGE_CURRENCY} usage amount for org ${org.id}, skipping`,
);
recordMissingBillingConfigSkip("missing_chb_usage_amount");
return null;
}
const planUSD = chbBillPlanAmountUSD(bill);
if (planUSD === null) {
// Usage alone would understate what the org owes and alert late.
logger.warn(
`[CLOUD SPEND ALERTS] CHB reported no usable ${CHB_USAGE_CURRENCY} plan fee for org ${org.id}, skipping`,
);
recordMissingBillingConfigSkip("missing_chb_plan_amount");
return null;
}
const periodStart = bill.startDate ? new Date(bill.startDate) : null;
if (!periodStart || Number.isNaN(periodStart.getTime())) {
// Without a period start an alert cannot be held to once per cycle, and
// every run past the threshold would email the org's admins again.
logger.warn(
`[CLOUD SPEND ALERTS] CHB reported no usable period start for org ${org.id}, skipping`,
);
recordMissingBillingConfigSkip("missing_chb_period_start");
return null;
}
return {
currentSpendUSD: usageUSD + planUSD,
currentPeriodStart: periodStart,
};
};
export const handleCloudSpendAlertJob = async (job: Job<{ orgId: string }>) => {
const { orgId } = job.data;
logger.info(`[CLOUD SPEND ALERTS] Processing org ${orgId}`);
// Fetch organization with spend alerts
const orgData = await prisma.organization.findFirst({
where: {
id: orgId,
},
include: {
cloudSpendAlerts: true,
},
});
if (!orgData) {
logger.error(`[CLOUD SPEND ALERTS] Organization ${orgId} not found`);
return;
}
const org = {
...parseDbOrg(orgData),
spendAlerts: orgData.cloudSpendAlerts,
};
// Check if org has spend alerts configured
if (org.spendAlerts.length === 0) {
logger.info(`[CLOUD SPEND ALERTS] No spend alerts for org ${orgId}`);
return;
}
if (org.cloudConfig?.plan === "Hobby") {
// handle case where user has downgraded to hobby
logger.info(
`[CLOUD SPEND ALERTS] Org ${orgId} not entitled to spend alerts (plan: Hobby"})`,
);
return;
}
// Which provider bills this org decides where its accrued spend is read
// from. A CHB org has no Stripe subscription to preview, and a Stripe org is
// unknown to CHB, so this is a hard fork rather than a fallback chain.
const billingProvider = getBillingProvider(org, {
cutoff: env.LANGFUSE_CLOUD_BILLING_CHB_CUTOFF_DATE,
optInOrgIds: env.LANGFUSE_CLOUD_BILLING_CHB_OPT_IN_ORG_IDS,
});
try {
const spend =
billingProvider === "clickhouse"
? await readClickhouseSpend(org)
: await readStripeSpend(org);
// The reader already logged why and counted the skip.
if (!spend) return;
const { currentSpendUSD, currentPeriodStart } = spend;
logger.info(
`[CLOUD SPEND ALERTS] Org ${orgId} current spend: $${currentSpendUSD.toFixed(2)} (provider: ${billingProvider})`,
);
// Get org admins and owners for email notifications (fetch once for all alerts)
const adminMemberships = await prisma.organizationMembership.findMany({
where: {
orgId: orgId,
role: { in: [Role.OWNER, Role.ADMIN] },
},
include: {
user: {
select: { email: true },
},
},
});
const adminEmails = adminMemberships
.map((m) => m.user?.email)
.filter((email): email is string => Boolean(email));
// Check each spend alert for this org
for (const alert of org.spendAlerts) {
const thresholdUSD = parseFloat(alert.threshold.toString());
// Check if threshold is breached
if (currentSpendUSD >= thresholdUSD) {
// Check if already triggered this billing cycle
const alreadyTriggered =
alert.triggeredAt && alert.triggeredAt >= currentPeriodStart;
if (!alreadyTriggered) {
logger.info(
`[CLOUD SPEND ALERTS] Triggering alert ${alert.id} for org ${orgId} - spend $${currentSpendUSD.toFixed(2)} >= threshold $${thresholdUSD.toFixed(2)}`,
);
const detectedAt = new Date();
const detectedAtUtc = detectedAt.toISOString().replace(".000Z", "Z");
if (adminEmails.length > 0) {
try {
// Send email notifications
await sendCloudSpendAlertEmail({
env,
orgId: orgId,
orgName: org.name,
alertTitle: alert.title,
currentSpend: currentSpendUSD,
threshold: thresholdUSD,
// casting due to cross-package type lag; property is supported in implementation
detectedAtUtc,
recipients: adminEmails,
} as any);
recordIncrement(
"langfuse.queue.cloud_spend_alert_queue.emails_sent",
1,
{ unit: "emails" },
);
logger.info(
`[CLOUD SPEND ALERTS] Sent alert emails to ${adminEmails.length} recipients for org ${orgId}`,
);
} catch (e) {
recordIncrement(
"langfuse.queue.cloud_spend_alert_queue.email_failures",
1,
{ unit: "emails" },
);
throw e;
}
}
// Update triggeredAt timestamp (reuse detection time)
await prisma.cloudSpendAlert.update({
where: { id: alert.id },
data: { triggeredAt: detectedAt },
});
recordIncrement(
"langfuse.queue.cloud_spend_alert_queue.triggered_alerts",
1,
{
unit: "alerts",
billing_provider: billingProvider,
},
);
} else {
logger.debug(
`[CLOUD SPEND ALERTS] Alert ${alert.id} for org ${orgId} already triggered this billing cycle`,
);
}
}
// Reset triggeredAt if we're in a new billing cycle and threshold is not breached
if (
currentSpendUSD < thresholdUSD &&
alert.triggeredAt &&
alert.triggeredAt < currentPeriodStart
) {
await prisma.cloudSpendAlert.update({
where: { id: alert.id },
data: { triggeredAt: null },
});
logger.debug(
`[CLOUD SPEND ALERTS] Reset alert ${alert.id} for org ${orgId} - new billing cycle`,
);
}
}
recordIncrement(
"langfuse.queue.cloud_spend_alert_queue.processed_orgs",
1,
{
unit: "organizations",
billing_provider: billingProvider,
},
);
logger.info(`[CLOUD SPEND ALERTS] Completed job for org ${orgId}`);
} catch (error) {
logger.error(`[CLOUD SPEND ALERTS] Error processing org ${orgId}`, {
error,
orgId,
});
traceException(
`[CLOUD SPEND ALERTS] Error processing org ${orgId}: ${error}`,
);
recordIncrement(
"langfuse.queue.cloud_spend_alert_queue.skipped_orgs_with_errors",
1,
{
unit: "organizations",
},
);
throw error; // Let BullMQ handle retry
}
};