🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
const mocks = vi.hoisted(() => ({
env: {
CLICKHOUSE_BILLING_BASE_URL: undefined as string | undefined,
CLICKHOUSE_BILLING_AUTH0_DOMAIN: undefined as string | undefined,
CLICKHOUSE_BILLING_AUTH0_CLIENT_ID: undefined as string | undefined,
CLICKHOUSE_BILLING_AUTH0_CLIENT_SECRET: undefined as string | undefined,
CLICKHOUSE_BILLING_AUTH0_AUDIENCE: "billing-api",
},
logger: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() },
}));
vi.mock("../../logger", () => ({ logger: mocks.logger }));
import {
buildChbApiClient,
ChbApiClient,
ChbApiError,
ChbPaymentRequiredError,
chbBillPlanAmountUSD,
chbBillUsageAmountUSD,
} from "./chbApiClient";
/**
* The env-reading wrapper each app keeps for itself; exercised here against the
* shared fail-closed rule it delegates to.
*/
const buildChbApiClientFromEnv = () => buildChbApiClient(mocks.env);
const CH_ORG_ID = "6dd6ab1d-9e8d-4c1a-8b4f-9a3d1e2c4b5a";
const AUTH0_DOMAIN = "chb-tenant.eu.auth0.com";
const TOKEN_URL = `https://${AUTH0_DOMAIN}/oauth/token`;
/** Shape a `fetch` result the client will accept, or one it must reject. */
const jsonResponse = (status: number, body: unknown) => ({
status,
ok: status >= 200 && status < 300,
json: async () => body,
});
/**
* One mock for both legs the client makes: the Auth0 client-credentials grant
* and the CHB request itself. Responses for the CHB leg are queued, so a test
* can drive a 401-then-200 replay.
*/
const chbResponses: ReturnType<typeof jsonResponse>[] = [];
let tokenCounter = 0;
const fetchMock = vi.fn(async (url: URL | string, _init: RequestInit) => {
if (String(url) === TOKEN_URL) {
tokenCounter += 1;
return jsonResponse(200, {
access_token: `chb-access-token-${tokenCounter}`,
expires_in: 86_400,
});
}
return chbResponses.shift() ?? jsonResponse(200, {});
});
/** Queue the CHB leg's response(s), in order. */
const onChb = (...responses: ReturnType<typeof jsonResponse>[]) => {
chbResponses.push(...responses);
};
const tokenCalls = () =>
fetchMock.mock.calls.filter((call) => String(call[0]) === TOKEN_URL);
const chbCalls = () =>
fetchMock.mock.calls.filter((call) => String(call[0]) !== TOKEN_URL);
/** The client's last request to CHB itself, as (url, init). */
const lastChbCall = () => {
const call = chbCalls().at(-1);
if (!call) throw new Error("the client never called CHB");
return { url: call[0] as URL, init: call[1] };
};
const headersOf = (init: RequestInit) => init.headers as Record<string, string>;
const client = () =>
new ChbApiClient({
// Deliberately carries a path prefix: the URL join must keep it.
baseUrl: "https://chb.example.com/api/v1",
auth: {
auth0Domain: AUTH0_DOMAIN,
clientId: "client-id",
clientSecret: "client-secret",
audience: "billing-api",
},
});
describe("chbApiClient", () => {
beforeEach(() => {
vi.clearAllMocks();
chbResponses.length = 0;
tokenCounter = 0;
mocks.env.CLICKHOUSE_BILLING_BASE_URL = undefined;
mocks.env.CLICKHOUSE_BILLING_AUTH0_DOMAIN = undefined;
mocks.env.CLICKHOUSE_BILLING_AUTH0_CLIENT_ID = undefined;
mocks.env.CLICKHOUSE_BILLING_AUTH0_CLIENT_SECRET = undefined;
vi.stubGlobal("fetch", fetchMock);
});
afterEach(() => {
vi.unstubAllGlobals();
});
describe("authentication", () => {
it("presents the Auth0 access token and never follows a redirect", async () => {
onChb(jsonResponse(200, { portalUrl: "https://pay.example.com/s/1" }));
await client().createPortalSession({
chOrganizationId: CH_ORG_ID,
returnUrl: "https://cloud.langfuse.com/back",
});
// CHB verifies the token against Auth0, so a static secret would 401.
const { init } = lastChbCall();
expect(headersOf(init).authorization).toBe("Bearer chb-access-token-1");
// A redirect would replay the token against the redirect target.
expect(init.redirect).toBe("error");
});
it("reuses one token across requests", async () => {
onChb(jsonResponse(200, { id: "plan_1" }), jsonResponse(200, {}));
const chb = client();
await chb.getAttachedPlan({ chOrganizationId: CH_ORG_ID });
await chb.clearScheduledChange({
chOrganizationId: CH_ORG_ID,
});
expect(tokenCalls()).toHaveLength(1);
expect(chbCalls()).toHaveLength(2);
});
it("mints a fresh token and replays once when CHB rejects the token", async () => {
onChb(
jsonResponse(401, { error: "Invalid or expired token" }),
jsonResponse(200, {}),
);
await client().setScheduledChange({
chOrganizationId: CH_ORG_ID,
change: { type: "cancel", when: "billing_cycle_end" },
idempotencyKey:
"chb.attachedplan.scheduled.set:attachedPlanId=b1:op=abc",
});
expect(tokenCalls()).toHaveLength(2);
const [first, second] = chbCalls();
expect(headersOf(first![1]).authorization).toBe(
"Bearer chb-access-token-1",
);
expect(headersOf(second![1]).authorization).toBe(
"Bearer chb-access-token-2",
);
// The replay must stay idempotent on CHB's side.
expect(headersOf(second![1])["Idempotency-Key"]).toBe(
"chb.attachedplan.scheduled.set:attachedPlanId=b1:op=abc",
);
});
it("surfaces a 401 that survives the replay", async () => {
onChb(jsonResponse(401, {}), jsonResponse(401, {}));
const error = await client()
.getAttachedPlan({ chOrganizationId: CH_ORG_ID })
.catch((e) => e);
// One replay, not a loop.
expect(chbCalls()).toHaveLength(2);
expect(error).toBeInstanceOf(ChbApiError);
expect(error.status).toBe(401);
});
});
describe("request wiring", () => {
it("keeps the base url path prefix when joining the request path", async () => {
onChb(jsonResponse(200, { id: "plan_1" }));
await client().getAttachedPlan({ chOrganizationId: CH_ORG_ID });
expect(lastChbCall().url.toString()).toBe(
"https://chb.example.com/api/v1/attachedplan?fields=plan%2Cperiod%2Cpayment%2Cscheduled",
);
});
it("scopes the scheduled-change routes by header, with no id in the path", async () => {
onChb(jsonResponse(200, {}));
await client().clearScheduledChange({ chOrganizationId: CH_ORG_ID });
// CHB resolves the attached plan from the organization header, so the
// path carries nothing caller-supplied.
const { url, init } = lastChbCall();
expect(url.pathname).toBe("/api/v1/attachedplan/scheduled");
expect(headersOf(init)["CH-Organization-Id"]).toBe(CH_ORG_ID);
});
it("scopes the invoice list by an organizationId query parameter", async () => {
onChb(jsonResponse(200, { invoices: [] }));
await client().listInvoices({ chOrganizationId: CH_ORG_ID });
const { url } = lastChbCall();
expect(url.pathname).toBe("/api/v1/invoices");
expect(url.searchParams.get("organizationId")).toBe(CH_ORG_ID);
});
it("queries bills by organization ids in the body, not by header", async () => {
onChb(jsonResponse(200, []));
await client().queryBills({ chOrganizationIds: [CH_ORG_ID] });
const { url, init } = lastChbCall();
expect(init.method).toBe("POST");
expect(url.pathname).toBe("/api/v1/bills/query");
expect(url.searchParams.get("fields")).toBe("usage,plan");
expect(JSON.parse(init.body as string)).toEqual({
organizationIds: [CH_ORG_ID],
});
expect(headersOf(init)["CH-Organization-Id"]).toBeUndefined();
});
it("always sends a checkout idempotency key, generating one when the caller has none", async () => {
const session = {
checkoutUrl: "https://pay.example.com/c/1",
organizationId: CH_ORG_ID,
};
const checkout = (idempotencyKey?: string) =>
client().createCheckoutSession({
email: "[email protected]",
planCode: "LANGFUSE_PRO",
returnUrl: "https://cloud.langfuse.com/back",
idempotencyKey,
});
onChb(jsonResponse(200, session));
await checkout("chb.checkout.create:orgId=org-1:op=abc");
expect(JSON.parse(lastChbCall().init.body as string).idempotencyKey).toBe(
"chb.checkout.create:orgId=org-1:op=abc",
);
// CHB rejects a checkout without a key; a generated one keeps the call
// valid and merely gives up deduplication.
onChb(jsonResponse(200, session));
await checkout(undefined);
const generated = JSON.parse(lastChbCall().init.body as string)
.idempotencyKey as string;
expect(generated).toMatch(/^[0-9a-f-]{36}$/);
});
it("sends the organization name as CHB's checkout name field", async () => {
const session = {
checkoutUrl: "https://pay.example.com/c/1",
organizationId: CH_ORG_ID,
};
const checkout = (name?: string) =>
client().createCheckoutSession({
name,
email: "[email protected]",
planCode: "LANGFUSE_PRO",
returnUrl: "https://cloud.langfuse.com/back",
});
onChb(jsonResponse(200, session));
await checkout("Acme Inc");
expect(JSON.parse(lastChbCall().init.body as string).name).toBe(
"Acme Inc",
);
// CHB rejects a blank name with a 400 but falls back to its own default
// when the field is absent. Langfuse's org-name schema does not trim, so
// a whitespace-only name must be omitted rather than block checkout.
for (const name of [undefined, "", " "]) {
onChb(jsonResponse(200, session));
await checkout(name);
expect(
JSON.parse(lastChbCall().init.body as string),
).not.toHaveProperty("name");
}
});
it("scopes org endpoints with CH-Organization-Id and omits it elsewhere", async () => {
onChb(jsonResponse(200, { id: "plan_1" }));
await client().getAttachedPlan({ chOrganizationId: CH_ORG_ID });
expect(headersOf(lastChbCall().init)["CH-Organization-Id"]).toBe(
CH_ORG_ID,
);
onChb(
jsonResponse(200, {
checkoutUrl: "https://pay.example.com/c/1",
organizationId: CH_ORG_ID,
}),
);
await client().createCheckoutSession({
email: "[email protected]",
planCode: "LANGFUSE_PRO",
returnUrl: "https://cloud.langfuse.com/back",
});
// Checkout creates the CH organization, so there is none to scope to yet.
expect(
headersOf(lastChbCall().init)["CH-Organization-Id"],
).toBeUndefined();
});
it("sets content-type only when there is a body", async () => {
onChb(jsonResponse(202, {}));
await client().setScheduledChange({
chOrganizationId: CH_ORG_ID,
change: {
type: "downgrade",
when: "billing_cycle_end",
planCode: "LANGFUSE_CORE",
},
});
const withBody = lastChbCall();
expect(headersOf(withBody.init)["content-type"]).toBe("application/json");
expect(JSON.parse(withBody.init.body as string)).toEqual({
type: "downgrade",
when: "billing_cycle_end",
planCode: "LANGFUSE_CORE",
});
onChb(jsonResponse(202, {}));
await client().clearScheduledChange({
chOrganizationId: CH_ORG_ID,
});
const withoutBody = lastChbCall();
expect(withoutBody.init.method).toBe("DELETE");
expect(withoutBody.init.body).toBeUndefined();
expect(headersOf(withoutBody.init)["content-type"]).toBeUndefined();
});
it("forwards an idempotency key as a header, and omits it when absent", async () => {
onChb(jsonResponse(202, {}));
await client().setScheduledChange({
chOrganizationId: CH_ORG_ID,
change: { type: "cancel", when: "billing_cycle_end" },
idempotencyKey:
"chb.attachedplan.scheduled.set:attachedPlanId=plan_1:op=abc",
});
expect(headersOf(lastChbCall().init)["Idempotency-Key"]).toBe(
"chb.attachedplan.scheduled.set:attachedPlanId=plan_1:op=abc",
);
onChb(jsonResponse(202, {}));
await client().setScheduledChange({
chOrganizationId: CH_ORG_ID,
change: { type: "cancel", when: "billing_cycle_end" },
});
expect(headersOf(lastChbCall().init)["Idempotency-Key"]).toBeUndefined();
});
});
describe("error mapping", () => {
it("maps 409 to ChbPaymentRequiredError", async () => {
onChb(jsonResponse(409, { error: "no_payment_method" }));
const error = await client()
.setScheduledChange({
chOrganizationId: CH_ORG_ID,
change: {
type: "upgrade",
when: "immediate",
planCode: "LANGFUSE_PRO_TEAMS",
},
})
.catch((e) => e);
// Callers branch on the subclass, so both the class and the 409 status
// are part of the contract.
expect(error).toBeInstanceOf(ChbPaymentRequiredError);
expect(error).toBeInstanceOf(ChbApiError);
expect(error.status).toBe(409);
expect(error.body).toEqual({ error: "no_payment_method" });
// A 409 is an expected UX branch, not an incident.
expect(mocks.logger.error).not.toHaveBeenCalled();
});
it("maps any other non-2xx to ChbApiError and logs it", async () => {
onChb(jsonResponse(503, { error: "unavailable" }));
const error = await client()
.getAttachedPlan({ chOrganizationId: CH_ORG_ID })
.catch((e) => e);
expect(error).toBeInstanceOf(ChbApiError);
expect(error).not.toBeInstanceOf(ChbPaymentRequiredError);
expect(error.status).toBe(503);
expect(mocks.logger.error).toHaveBeenCalledTimes(1);
});
it("does not fail on a non-2xx with an unparsable body", async () => {
onChb({
status: 502,
ok: false,
json: async () => {
throw new Error("not json");
},
});
const error = await client()
.getAttachedPlan({ chOrganizationId: CH_ORG_ID })
.catch((e) => e);
expect(error).toBeInstanceOf(ChbApiError);
expect(error.status).toBe(502);
expect(error.body).toBeUndefined();
});
});
describe("response parsing", () => {
it("ignores unknown fields so additive CHB changes cannot break us", async () => {
onChb(
jsonResponse(200, {
id: "plan_1",
plan: {
code: "LANGFUSE_PRO_TEAMS",
amount: 199,
currency: "USD",
recurrence: "monthly",
extra: true,
},
period: { startDate: "2026-08-01T00:00:00Z" },
payment: {
status: "active",
provider: { name: "stripe", customerId: "cus_1" },
},
unknownTopLevel: { nested: 1 },
}),
);
const attachedPlan = await client().getAttachedPlan({
chOrganizationId: CH_ORG_ID,
});
expect(attachedPlan.id).toBe("plan_1");
expect(attachedPlan.plan?.code).toBe("LANGFUSE_PRO_TEAMS");
expect(attachedPlan.payment?.provider?.customerId).toBe("cus_1");
// Nothing required beyond `id`, so a sparse attached plan still parses.
expect(attachedPlan.scheduled).toBeUndefined();
});
it("parses a bill without usage instead of failing the whole batch", async () => {
onChb(
jsonResponse(200, [
{
id: "bill_1",
organizationId: CH_ORG_ID,
startDate: "2026-09-01T00:00:00.000Z",
endDate: "2026-10-01T00:00:00.000Z",
usage: {
amount: 1250.68,
currency: "USD",
asOf: "2026-09-29T12:00:00.000Z",
},
plan: {
amount: 199,
currency: "USD",
startDate: "2026-10-01T00:00:00.000Z",
endDate: "2026-11-01T00:00:00.000Z",
},
},
{ id: "bill_2", organizationId: "other-org", extra: true },
]),
);
const bills = await client().queryBills({
chOrganizationIds: [CH_ORG_ID, "other-org"],
});
expect(bills.map((bill) => bill.id)).toEqual(["bill_1", "bill_2"]);
expect(bills[0]!.usage?.amount).toBe(1250.68);
expect(bills[0]!.plan?.amount).toBe(199);
expect(bills[1]!.usage).toBeUndefined();
expect(bills[1]!.plan).toBeUndefined();
});
it("reads the portal URL from CHB's portalUrl field", async () => {
onChb(jsonResponse(200, { portalUrl: "https://pay.example.com/s/1" }));
await expect(
client().createPortalSession({
chOrganizationId: CH_ORG_ID,
returnUrl: "https://cloud.langfuse.com/back",
}),
).resolves.toBe("https://pay.example.com/s/1");
});
it("reads the checkout URL from CHB's checkoutUrl field", async () => {
onChb(
jsonResponse(200, {
organizationId: CH_ORG_ID,
checkoutUrl: "https://pay.example.com/c/1",
}),
);
await expect(
client().createCheckoutSession({
email: "[email protected]",
planCode: "LANGFUSE_PRO",
returnUrl: "https://cloud.langfuse.com/back",
}),
).resolves.toEqual({
organizationId: CH_ORG_ID,
checkoutUrl: "https://pay.example.com/c/1",
});
});
it("defaults the invoice list to empty when CHB omits the key", async () => {
onChb(jsonResponse(200, {}));
await expect(
client().listInvoices({
chOrganizationId: CH_ORG_ID,
}),
).resolves.toEqual([]);
});
it("rejects a checkout session without a uuid organization id", async () => {
// The id is persisted into cloudConfig, where the stored schema demands a
// uuid — rejecting here keeps a bad id from ever reaching the database.
onChb(
jsonResponse(200, {
checkoutUrl: "https://pay.example.com/c/1",
organizationId: "not-a-uuid",
}),
);
await expect(
client().createCheckoutSession({
email: "[email protected]",
planCode: "LANGFUSE_PRO",
returnUrl: "https://cloud.langfuse.com/back",
}),
).rejects.toThrow();
});
});
describe("buildChbApiClientFromEnv", () => {
const setAll = () => {
mocks.env.CLICKHOUSE_BILLING_BASE_URL = "https://chb.example.com";
mocks.env.CLICKHOUSE_BILLING_AUTH0_DOMAIN = AUTH0_DOMAIN;
mocks.env.CLICKHOUSE_BILLING_AUTH0_CLIENT_ID = "client-id";
mocks.env.CLICKHOUSE_BILLING_AUTH0_CLIENT_SECRET = "client-secret";
};
it.each([
"CLICKHOUSE_BILLING_BASE_URL",
"CLICKHOUSE_BILLING_AUTH0_DOMAIN",
"CLICKHOUSE_BILLING_AUTH0_CLIENT_ID",
"CLICKHOUSE_BILLING_AUTH0_CLIENT_SECRET",
] as const)("returns null when %s is missing", (missing) => {
setAll();
mocks.env[missing] = undefined;
// Fail closed: a partly configured deployment must not call CHB at all.
expect(buildChbApiClientFromEnv()).toBeNull();
});
it("builds a client once every credential is present", () => {
setAll();
expect(buildChbApiClientFromEnv()).toBeInstanceOf(ChbApiClient);
});
});
/**
* CHB reports this amount in USD and refuses any other currency, as a
* major-unit figure — unlike `ChbInvoice.amount`, which is minor units — so
* a caller comparing it against a USD spend threshold uses it as-is.
*/
describe("chbBillUsageAmountUSD", () => {
const withUsage = (usage: unknown) =>
({
id: "bill_1",
organizationId: CH_ORG_ID,
startDate: "2026-09-01T00:00:00.000Z",
usage,
}) as Parameters<typeof chbBillUsageAmountUSD>[0];
it("returns the amount unchanged as USD", () => {
expect(
chbBillUsageAmountUSD(withUsage({ amount: 1234.56, currency: "USD" })),
).toBe(1234.56);
});
it("accepts a lowercase currency", () => {
expect(
chbBillUsageAmountUSD(withUsage({ amount: 10, currency: "usd" })),
).toBe(10);
});
it("treats an absent currency as USD, which is all CHB emits", () => {
expect(chbBillUsageAmountUSD(withUsage({ amount: 10 }))).toBe(10);
});
/** Nothing spent yet is a reading; no reading is not. */
it("distinguishes zero spend from a missing amount", () => {
expect(
chbBillUsageAmountUSD(withUsage({ amount: 0, currency: "USD" })),
).toBe(0);
expect(
chbBillUsageAmountUSD(withUsage({ amount: null, currency: "USD" })),
).toBeNull();
expect(chbBillUsageAmountUSD(withUsage(null))).toBeNull();
expect(
chbBillUsageAmountUSD({ id: "bill_1", organizationId: CH_ORG_ID }),
).toBeNull();
});
/**
* A non-USD currency means the contract moved; the number can no longer be
* held against a USD threshold, so it must not be guessed at.
*/
it("refuses a currency it cannot compare against a USD threshold", () => {
expect(
chbBillUsageAmountUSD(withUsage({ amount: 1000, currency: "EUR" })),
).toBeNull();
});
it("refuses a non-finite amount", () => {
expect(
chbBillUsageAmountUSD(
withUsage({ amount: Number.POSITIVE_INFINITY, currency: "USD" }),
),
).toBeNull();
expect(
chbBillUsageAmountUSD(withUsage({ amount: NaN, currency: "USD" })),
).toBeNull();
});
});
/**
* The plan fee shares the usage amount's major-unit/USD contract; what it
* adds is that CHB leaves `plan` out for a bill without a standing charge.
*/
describe("chbBillPlanAmountUSD", () => {
const withPlan = (plan: unknown) =>
({
id: "bill_1",
organizationId: CH_ORG_ID,
startDate: "2026-09-01T00:00:00.000Z",
plan,
}) as Parameters<typeof chbBillPlanAmountUSD>[0];
it("returns the amount unchanged as USD", () => {
expect(
chbBillPlanAmountUSD(withPlan({ amount: 499.9, currency: "USD" })),
).toBe(499.9);
});
/** No standing charge is a fee of 0; a fee without a figure is no reading. */
it("distinguishes an absent plan from a plan without a usable amount", () => {
expect(
chbBillPlanAmountUSD({ id: "bill_1", organizationId: CH_ORG_ID }),
).toBe(0);
expect(chbBillPlanAmountUSD(withPlan(null))).toBe(0);
expect(
chbBillPlanAmountUSD(withPlan({ amount: null, currency: "USD" })),
).toBeNull();
expect(
chbBillPlanAmountUSD(withPlan({ amount: 199, currency: "EUR" })),
).toBeNull();
});
});
});