Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): break historical credit usage down by endpoint and by day (#5291)
Rakshith Ramprakash12c69db4faf6
diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.ymlindex 6fa3890ea..964b1b6ec 100644--- a/.github/workflows/test-server.yml+++ b/.github/workflows/test-server.yml@@ -245,6 +245,15 @@ jobs:run: pnpm exec vitest run src/services/autumn/__tests__/autumn.service.test.ts --reporter=defaultworking-directory: apps/api+ - name: Run historical credit usage regression tests+ if: matrix.engine == 'fetch' && matrix.proxy == 'no-proxy' && matrix.nuq == 'postgres'+ run: >-+ pnpm exec vitest run+ src/services/autumn/__tests__/usage.test.ts+ src/controllers/v2/__tests__/credit-usage-historical.test.ts+ --reporter=default+ working-directory: apps/api+- name: Run billing batch regression testsif: matrix.engine == 'fetch' && matrix.proxy == 'no-proxy' && matrix.nuq == 'postgres'run: pnpm exec vitest run src/services/billing/__tests__/batch_billing.test.ts --reporter=defaultdiff --git a/apps/api/src/controllers/v2/__tests__/credit-usage-historical.test.ts b/apps/api/src/controllers/v2/__tests__/credit-usage-historical.test.tsnew file mode 100644index 000000000..eb13176b7--- /dev/null+++ b/apps/api/src/controllers/v2/__tests__/credit-usage-historical.test.ts@@ -0,0 +1,155 @@+import type { Response } from "express";+import type { RequestWithAuth } from "../types";++const mocks = vi.hoisted(() => ({+ getTeamHistoricalUsage: vi.fn(),+ getTeamHistoricalUsageByApiKey: vi.fn(),+ getTeamHistoricalUsageByEndpoint: vi.fn(),+}));++vi.mock("../../../services/autumn/usage", () => mocks);++import { creditUsageHistoricalController } from "../credit-usage-historical";++function request(query: Record<string, unknown>) {+ return { query, auth: { team_id: "team-1" } } as unknown as RequestWithAuth;+}++function response() {+ const res: any = {};+ res.status = vi.fn(() => res);+ res.json = vi.fn(() => res);+ return res as Response & { status: any; json: any };+}++const MONTH = {+ startDate: "2026-04-01T00:00:00.000Z",+ endDate: null,+ creditsUsed: 5,+};++beforeEach(() => {+ vi.clearAllMocks();+ mocks.getTeamHistoricalUsage.mockResolvedValue([MONTH]);+ mocks.getTeamHistoricalUsageByApiKey.mockResolvedValue([+ { ...MONTH, apiKey: "Default" },+ ]);+ mocks.getTeamHistoricalUsageByEndpoint.mockResolvedValue([+ { ...MONTH, endpoint: "scrape" },+ ]);+});++describe("creditUsageHistoricalController", () => {+ it("answers with monthly totals and no bin field when no option is given", async () => {+ const res = response();+ await creditUsageHistoricalController(request({}), res);++ expect(mocks.getTeamHistoricalUsage).toHaveBeenCalledWith(+ "team-1",+ "month",+ );+ expect(res.json).toHaveBeenCalledWith({ success: true, periods: [MONTH] });+ expect(JSON.stringify(res.json.mock.calls[0][0])).toBe(+ JSON.stringify({ success: true, periods: [MONTH] }),+ );+ });++ it("keeps the byApiKey response as it was", async () => {+ const res = response();+ await creditUsageHistoricalController(request({ byApiKey: "true" }), res);++ expect(mocks.getTeamHistoricalUsageByApiKey).toHaveBeenCalledWith(+ "team-1",+ "month",+ );+ expect(res.json).toHaveBeenCalledWith({+ success: true,+ periods: [{ ...MONTH, apiKey: "Default" }],+ });+ });++ it("breaks usage down by endpoint", async () => {+ const res = response();+ await creditUsageHistoricalController(request({ byEndpoint: "true" }), res);++ expect(mocks.getTeamHistoricalUsageByEndpoint).toHaveBeenCalledWith(+ "team-1",+ "month",+ );+ expect(res.json).toHaveBeenCalledWith({+ success: true,+ periods: [{ ...MONTH, endpoint: "scrape" }],+ });+ });++ it("passes the bin through and echoes it", async () => {+ const res = response();+ await creditUsageHistoricalController(+ request({ byEndpoint: "true", bin: "day" }),+ res,+ );++ expect(mocks.getTeamHistoricalUsageByEndpoint).toHaveBeenCalledWith(+ "team-1",+ "day",+ );+ expect(res.json).toHaveBeenCalledWith({+ success: true,+ bin: "day",+ periods: [{ ...MONTH, endpoint: "scrape" }],+ });+ });++ it("echoes an explicit month bin", async () => {+ const res = response();+ await creditUsageHistoricalController(request({ bin: "month" }), res);++ expect(res.json).toHaveBeenCalledWith({+ success: true,+ bin: "month",+ periods: [MONTH],+ });+ });++ it("sorts periods by start date", async () => {+ mocks.getTeamHistoricalUsageByEndpoint.mockResolvedValue([+ { ...MONTH, endpoint: "scrape" },+ {+ startDate: "2026-03-01T00:00:00.000Z",+ endDate: "2026-04-01T00:00:00.000Z",+ endpoint: "crawl",+ creditsUsed: 2,+ },+ ]);+ const res = response();+ await creditUsageHistoricalController(request({ byEndpoint: "true" }), res);++ expect(+ res.json.mock.calls[0][0].periods.map((p: any) => p.endpoint),+ ).toEqual(["crawl", "scrape"]);+ });++ it.each([["week"], [["day", "month"]]])("rejects bin %j", async bin => {+ const res = response();+ await creditUsageHistoricalController(request({ bin }), res);++ expect(res.status).toHaveBeenCalledWith(400);+ expect(res.json).toHaveBeenCalledWith({+ success: false,+ error: 'bin must be "month" or "day".',+ });+ expect(mocks.getTeamHistoricalUsage).not.toHaveBeenCalled();+ });++ it("rejects byApiKey together with byEndpoint", async () => {+ const res = response();+ await creditUsageHistoricalController(+ request({ byApiKey: "true", byEndpoint: "true" }),+ res,+ );++ expect(res.status).toHaveBeenCalledWith(400);+ expect(mocks.getTeamHistoricalUsageByApiKey).not.toHaveBeenCalled();+ expect(mocks.getTeamHistoricalUsageByEndpoint).not.toHaveBeenCalled();+ });+});diff --git a/apps/api/src/controllers/v2/credit-usage-historical.ts b/apps/api/src/controllers/v2/credit-usage-historical.tsindex 6fb3571e0..bba8598f2 100644--- a/apps/api/src/controllers/v2/credit-usage-historical.ts+++ b/apps/api/src/controllers/v2/credit-usage-historical.ts@@ -3,14 +3,18 @@ import { ErrorResponse, RequestWithAuth } from "./types";import {getTeamHistoricalUsage,getTeamHistoricalUsageByApiKey,+ getTeamHistoricalUsageByEndpoint,+ HistoricalBin,} from "../../services/autumn/usage";interface CreditUsageHistoricalResponse {success: true;+ bin?: HistoricalBin;periods: {startDate: string | null;endDate: string | null;apiKey?: string;+ endpoint?: string;creditsUsed: number;}[];}@@ -20,10 +24,35 @@ export async function creditUsageHistoricalController(res: Response<CreditUsageHistoricalResponse | ErrorResponse>,): Promise<void> {const byApiKey = req.query.byApiKey === "true";+ const byEndpoint = req.query.byEndpoint === "true";+ const requestedBin = req.query.bin;- const periods: CreditUsageHistoricalResponse["periods"] = byApiKey- ? await getTeamHistoricalUsageByApiKey(req.auth.team_id)- : await getTeamHistoricalUsage(req.auth.team_id);+ if (+ requestedBin !== undefined &&+ requestedBin !== "month" &&+ requestedBin !== "day"+ ) {+ res.status(400).json({+ success: false,+ error: 'bin must be "month" or "day".',+ });+ return;+ }++ if (byApiKey && byEndpoint) {+ res.status(400).json({+ success: false,+ error: "byApiKey and byEndpoint cannot be combined.",+ });+ return;+ }++ const bin: HistoricalBin = requestedBin ?? "month";+ const periods: CreditUsageHistoricalResponse["periods"] = byEndpoint+ ? await getTeamHistoricalUsageByEndpoint(req.auth.team_id, bin)+ : byApiKey+ ? await getTeamHistoricalUsageByApiKey(req.auth.team_id, bin)+ : await getTeamHistoricalUsage(req.auth.team_id, bin);periods.sort((a, b) => {const aTime = a.startDate ? Date.parse(a.startDate) : NaN;@@ -36,8 +65,11 @@ export async function creditUsageHistoricalController(return aTime - bTime;});- res.json({- success: true,- periods,- });+ // `bin` is echoed only when it was asked for, so a caller can tell that this+ // API honoured it and the response without it stays as it was.+ res.json(+ requestedBin === undefined+ ? { success: true, periods }+ : { success: true, bin, periods },+ );}diff --git a/apps/api/src/services/autumn/__tests__/usage.test.ts b/apps/api/src/services/autumn/__tests__/usage.test.tsindex a08882ce4..045c05029 100644--- a/apps/api/src/services/autumn/__tests__/usage.test.ts+++ b/apps/api/src/services/autumn/__tests__/usage.test.ts@@ -64,6 +64,7 @@ import {getTeamBalance,getTeamHistoricalUsage,getTeamHistoricalUsageByApiKey,+ getTeamHistoricalUsageByEndpoint,} from "../usage";beforeEach(() => {@@ -860,6 +861,43 @@ describe("getTeamHistoricalUsage", () => {},]);});++ it("buckets daily usage by UTC day when asked for day bins", async () => {+ mockAggregate.mockResolvedValue({+ list: [+ {+ period: Date.parse("2026-03-30T00:00:00.000Z"),+ values: { CREDITS: 20 },+ },+ {+ period: Date.parse("2026-04-01T00:00:00.000Z"),+ values: { CREDITS: 1 },+ },+ ],+ });++ await expect(getTeamHistoricalUsage("team-1", "day")).resolves.toEqual([+ {+ startDate: "2026-03-30T00:00:00.000Z",+ endDate: "2026-03-31T00:00:00.000Z",+ creditsUsed: 20,+ },+ {+ startDate: "2026-04-01T00:00:00.000Z",+ endDate: null,+ creditsUsed: 1,+ },+ ]);++ expect(mockAggregate).toHaveBeenCalledWith(+ expect.objectContaining({+ featureId: "CREDITS",+ range: "90d",+ binSize: "day",+ }),+ expect.anything(),+ );+ });});// The byApiKey breakdown asks Autumn for fixed 7-day slices. At this instant@@ -936,6 +974,41 @@ describe("getTeamHistoricalUsageByApiKey", () => {]);});+ it("buckets grouped usage by UTC day when asked for day bins", async () => {+ apiKeysData = [+ { id: 101, name: "Default" },+ { id: 202, name: "postman" },+ ];++ autumnServesDays([+ { day: "2026-03-31T00:00:00.000Z", credits: { "101": 26, "202": 3 } },+ { day: "2026-04-02T00:00:00.000Z", credits: { "202": 5 } },+ ]);++ await expect(+ getTeamHistoricalUsageByApiKey("team-1", "day"),+ ).resolves.toEqual([+ {+ startDate: "2026-03-31T00:00:00.000Z",+ endDate: "2026-04-01T00:00:00.000Z",+ apiKey: "Default",+ creditsUsed: 26,+ },+ {+ startDate: "2026-03-31T00:00:00.000Z",+ endDate: "2026-04-01T00:00:00.000Z",+ apiKey: "postman",+ creditsUsed: 3,+ },+ {+ startDate: "2026-04-02T00:00:00.000Z",+ endDate: null,+ apiKey: "postman",+ creditsUsed: 5,+ },+ ]);+ });+it("asks Autumn for 7-day slices across the window, today's slice first", async () => {autumnServesDays([]);@@ -1198,6 +1271,206 @@ describe("getTeamHistoricalUsageByApiKey", () => {});});+/**+ * Like `autumnServesDays`, but each day carries grouped values per feature,+ * the shape Autumn answers with when several features are asked for at once.+ */+function autumnServesGroupedDays(+ days: Array<{ day: string; grouped: Record<string, Record<string, number>> }>,+) {+ mockAggregate.mockImplementation(async (args: any) => ({+ list: days+ .map(d => ({ period: Date.parse(d.day), groupedValues: d.grouped }))+ .filter(+ entry =>+ entry.period >= args.customRange.start &&+ entry.period <= args.customRange.end,+ ),+ }));+}++describe("getTeamHistoricalUsageByEndpoint", () => {+ beforeEach(() => {+ vi.useFakeTimers({ toFake: ["Date"] });+ vi.setSystemTime(new Date(BY_API_KEY_NOW));+ });++ afterEach(() => {+ vi.useRealTimers();+ });++ it("asks Autumn for credit and search usage grouped by endpoint, in 7-day slices", async () => {+ autumnServesGroupedDays([]);++ await getTeamHistoricalUsageByEndpoint("team-1");++ expect(mockAggregate).toHaveBeenCalledTimes(BY_API_KEY_SLICE_COUNT);+ expect(mockAggregate.mock.calls[0][0].customRange).toEqual({+ start: CURRENT_SLICE_START,+ end: Date.parse(BY_API_KEY_NOW),+ });+ for (const [args, options] of mockAggregate.mock.calls) {+ expect(args).toEqual(+ expect.objectContaining({+ customerId: "org-1",+ entityId: "team-1",+ featureId: ["CREDITS", "SEARCH_CREDITS"],+ binSize: "day",+ groupBy: "properties.endpoint",+ maxGroups: 250,+ }),+ );+ expect(options).toEqual(+ expect.objectContaining({+ timeoutMs: EXPECTED_HISTORICAL_TIMEOUT_MS,+ retryCodes: ["429"],+ }),+ );+ }+ });++ it("sums each endpoint across features into calendar-month buckets", async () => {+ autumnServesGroupedDays([+ {+ day: "2026-03-30T00:00:00.000Z",+ grouped: {+ CREDITS: { scrape: 10, crawl: 40 },+ SEARCH_CREDITS: { search: 6 },+ },+ },+ {+ day: "2026-03-31T00:00:00.000Z",+ grouped: { CREDITS: { scrape: 5 }, SEARCH_CREDITS: { search: 2 } },+ },+ {+ day: "2026-04-02T00:00:00.000Z",+ grouped: { CREDITS: { scrape: 1 } },+ },+ ]);++ await expect(getTeamHistoricalUsageByEndpoint("team-1")).resolves.toEqual([+ {+ startDate: "2026-03-01T00:00:00.000Z",+ endDate: "2026-04-01T00:00:00.000Z",+ endpoint: "crawl",+ creditsUsed: 40,+ },+ {+ startDate: "2026-03-01T00:00:00.000Z",+ endDate: "2026-04-01T00:00:00.000Z",+ endpoint: "scrape",+ creditsUsed: 15,+ },+ {+ startDate: "2026-03-01T00:00:00.000Z",+ endDate: "2026-04-01T00:00:00.000Z",+ endpoint: "search",+ creditsUsed: 8,+ },+ {+ startDate: "2026-04-01T00:00:00.000Z",+ endDate: null,+ endpoint: "scrape",+ creditsUsed: 1,+ },+ ]);+ });++ it("buckets endpoint usage by UTC day when asked for day bins", async () => {+ autumnServesGroupedDays([+ {+ day: "2026-04-17T00:00:00.000Z",+ grouped: { CREDITS: { scrape: 3 }, SEARCH_CREDITS: { search: 2 } },+ },+ {+ day: "2026-04-19T00:00:00.000Z",+ grouped: { CREDITS: { map: 1 } },+ },+ ]);++ await expect(+ getTeamHistoricalUsageByEndpoint("team-1", "day"),+ ).resolves.toEqual([+ {+ startDate: "2026-04-17T00:00:00.000Z",+ endDate: "2026-04-18T00:00:00.000Z",+ endpoint: "scrape",+ creditsUsed: 3,+ },+ {+ startDate: "2026-04-17T00:00:00.000Z",+ endDate: "2026-04-18T00:00:00.000Z",+ endpoint: "search",+ creditsUsed: 2,+ },+ {+ startDate: "2026-04-19T00:00:00.000Z",+ endDate: null,+ endpoint: "map",+ creditsUsed: 1,+ },+ ]);+ });++ it("labels usage without an endpoint as 'unknown'", async () => {+ autumnServesGroupedDays([+ {+ day: "2026-04-17T00:00:00.000Z",+ grouped: { CREDITS: { "": 4, scrape: 1 } },+ },+ ]);++ await expect(getTeamHistoricalUsageByEndpoint("team-1")).resolves.toEqual([+ {+ startDate: "2026-04-01T00:00:00.000Z",+ endDate: null,+ endpoint: "scrape",+ creditsUsed: 1,+ },+ {+ startDate: "2026-04-01T00:00:00.000Z",+ endDate: null,+ endpoint: "unknown",+ creditsUsed: 4,+ },+ ]);+ });++ it("caches settled slices under its own key, apart from the byApiKey breakdown", async () => {+ autumnServesGroupedDays([+ {+ day: "2026-02-10T00:00:00.000Z",+ grouped: { CREDITS: { scrape: 7 } },+ },+ ]);++ await getTeamHistoricalUsageByEndpoint("team-1");++ expect(mockSetValue).toHaveBeenCalledTimes(BY_API_KEY_SLICE_COUNT - 1);+ for (const [key] of mockSetValue.mock.calls) {+ expect(key).toMatch(/^historical-usage-by-endpoint:v1:team-1:\d+$/);+ }++ mockAggregate.mockClear();+ autumnServesDays([]);+ await getTeamHistoricalUsageByApiKey("team-1");++ // Nothing cached for the endpoint breakdown answers the byApiKey one.+ expect(mockAggregate).toHaveBeenCalledTimes(BY_API_KEY_SLICE_COUNT);+ });++ it("returns an empty history (no org fallback) when the entity is missing", async () => {+ mockAggregate.mockRejectedValueOnce(+ Object.assign(new Error("not found"), { statusCode: 404 }),+ );++ await expect(getTeamHistoricalUsageByEndpoint("team-1")).resolves.toEqual(+ [],+ );+ expect(mockAggregate).toHaveBeenCalledTimes(1);+ });+});+// ---------------------------------------------------------------------------// Per-call Autumn timeout for the historical aggregations// ---------------------------------------------------------------------------diff --git a/apps/api/src/services/autumn/usage.ts b/apps/api/src/services/autumn/usage.tsindex eb0783ba2..c9971cdaf 100644--- a/apps/api/src/services/autumn/usage.ts+++ b/apps/api/src/services/autumn/usage.ts@@ -4,7 +4,10 @@ import { dbRr } from "../../db/connection";import * as schema from "../../db/schema";import { getValue, setValue } from "../redis";import { autumnClient } from "./client";-import { CREDITS_FEATURE_ID } from "./autumn.service";+import {+ CREDITS_FEATURE_ID,+ SEARCH_CREDITS_FEATURE_ID,+} from "./autumn.service";export const TOKENS_PER_CREDIT = 15;const HISTORICAL_RANGE = "90d";@@ -13,32 +16,33 @@ const DAY_MS = 24 * 60 * 60 * 1000;const HISTORICAL_WINDOW_MS = 90 * DAY_MS;// The historical/analytics aggregations below are bucketed by day (and-// optionally grouped by API key), so Autumn has to walk raw events and their-// cost scales with the team's event volume. That routinely takes several-// seconds. The Autumn client's global 2s timeout is sized for the+// optionally grouped by API key or endpoint), so Autumn has to walk raw events+// and their cost scales with the team's event volume. That routinely takes+// several seconds. The Autumn client's global 2s timeout is sized for the// latency-sensitive balance checks on the request hot path, and it is far too// tight here, so these calls override it per call.const HISTORICAL_AGGREGATE_TIMEOUT_MS = 15000;-// Grouping by API key over the whole 90-day window does not finish inside that-// timeout for teams with tens of millions of events in it, so the byApiKey-// breakdown asks Autumn for fixed 7-day slices instead. A slice that ended-// over a day ago no longer changes and is cached for as long as it can still-// fall inside the window, so a warm request only asks Autumn for the slice-// holding today (and, for a day after a slice boundary, the one before it).-const BY_API_KEY_SLICE_MS = 7 * DAY_MS;-const BY_API_KEY_SLICE_SETTLE_MS = DAY_MS;-const BY_API_KEY_SLICE_CACHE_TTL_SECONDS = HISTORICAL_WINDOW_MS / 1000;-const BY_API_KEY_SLICE_CONCURRENCY = 2;--// Autumn's maximum. Its default of 9 folds every further key into one "Other"-// group per day, which the response can only label "Unknown".-const BY_API_KEY_MAX_GROUPS = 250;--// Autumn limits event aggregates per customer per second, and a cold byApiKey+// Grouping over the whole 90-day window does not finish inside that timeout+// for teams with tens of millions of events in it, so the grouped breakdowns+// (byApiKey, byEndpoint) ask Autumn for fixed 7-day slices instead. A slice+// that ended over a day ago no longer changes and is cached for as long as it+// can still fall inside the window, so a warm request only asks Autumn for the+// slice holding today (and, for a day after a slice boundary, the one before+// it).+const GROUPED_SLICE_MS = 7 * DAY_MS;+const GROUPED_SLICE_SETTLE_MS = DAY_MS;+const GROUPED_SLICE_CACHE_TTL_SECONDS = HISTORICAL_WINDOW_MS / 1000;+const GROUPED_SLICE_CONCURRENCY = 2;++// Autumn's maximum. Its default of 9 folds every further group into one+// "Other" group per day, which the response could not label.+const GROUPED_MAX_GROUPS = 250;++// Autumn limits event aggregates per customer per second, and a cold grouped// request makes one call per slice. A rate-limited slice waits and retries;// every other status fails the request as before.-const BY_API_KEY_RATE_LIMIT_RETRY = {+const GROUPED_RATE_LIMIT_RETRY = {retries: {strategy: "backoff" as const,backoff: {@@ -52,6 +56,27 @@ const BY_API_KEY_RATE_LIMIT_RETRY = {retryCodes: ["429"],};+/** How a grouped breakdown asks Autumn for its days and caches them. */+interface UsageGrouping {+ groupBy: string;+ featureId: string | string[];+ cachePrefix: string;+}++const API_KEY_GROUPING: UsageGrouping = {+ groupBy: "properties.apiKeyId",+ featureId: CREDITS_FEATURE_ID,+ cachePrefix: "historical-usage-by-api-key:v1",+};++// Search usage is recorded against SEARCH_CREDITS, so the endpoint breakdown+// reads both features; without it search would be missing as an endpoint.+const ENDPOINT_GROUPING: UsageGrouping = {+ groupBy: "properties.endpoint",+ featureId: [CREDITS_FEATURE_ID, SEARCH_CREDITS_FEATURE_ID],+ cachePrefix: "historical-usage-by-endpoint:v1",+};+// ---------------------------------------------------------------------------// Types// ---------------------------------------------------------------------------@@ -118,116 +143,181 @@ async function lookupApiKeyNames(return nameMap;}-function toMonthStartIso(period: unknown): string | null {+/** Width of a historical usage period. */+export type HistoricalBin = "month" | "day";++function toBinStartIso(period: unknown, bin: HistoricalBin): string | null {if (period == null) return null;const date = new Date(period as string | number);if (isNaN(date.getTime())) return null;return new Date(- Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), 1),+ Date.UTC(+ date.getUTCFullYear(),+ date.getUTCMonth(),+ bin === "day" ? date.getUTCDate() : 1,+ ),).toISOString();}-function nextMonthIso(monthStartIso: string): string {- const date = new Date(monthStartIso);+function nextBinIso(binStartIso: string, bin: HistoricalBin): string {+ const date = new Date(binStartIso);return new Date(- Date.UTC(date.getUTCFullYear(), date.getUTCMonth() + 1, 1),+ Date.UTC(+ date.getUTCFullYear(),+ date.getUTCMonth() + (bin === "month" ? 1 : 0),+ date.getUTCDate() + (bin === "day" ? 1 : 0),+ ),).toISOString();}-function aggregateHistoricalPeriodsByMonth(list: any[]): HistoricalPeriod[] {- const monthTotals = new Map<string, number>();+function aggregateHistoricalPeriods(+ list: any[],+ bin: HistoricalBin,+): HistoricalPeriod[] {+ const binTotals = new Map<string, number>();for (const entry of list) {- const monthStart = toMonthStartIso(entry.period);- if (!monthStart) continue;+ const binStart = toBinStartIso(entry.period, bin);+ if (!binStart) continue;- monthTotals.set(- monthStart,- (monthTotals.get(monthStart) ?? 0) ++ binTotals.set(+ binStart,+ (binTotals.get(binStart) ?? 0) +(entry.values?.[CREDITS_FEATURE_ID] ?? 0),);}- const monthStarts = [...monthTotals.keys()].sort();+ const binStarts = [...binTotals.keys()].sort();- return monthStarts.map((startDate, i) => ({+ return binStarts.map((startDate, i) => ({startDate,- endDate: i < monthStarts.length - 1 ? nextMonthIso(startDate) : null,- creditsUsed: monthTotals.get(startDate) ?? 0,+ endDate: i < binStarts.length - 1 ? nextBinIso(startDate, bin) : null,+ creditsUsed: binTotals.get(startDate) ?? 0,}));}-function getGroupedCredits(entry: any): Record<string, number> | undefined {- return (- entry.groupedValues?.[CREDITS_FEATURE_ID] ??- entry.grouped_values?.[CREDITS_FEATURE_ID]- );+/** Sums each group's credits across the grouping's features. */+function getGroupedCredits(+ entry: any,+ grouping: UsageGrouping,+): Record<string, number> | undefined {+ let credits: Record<string, number> | undefined;+ for (const featureId of [grouping.featureId].flat()) {+ const values: Record<string, number> | undefined =+ entry.groupedValues?.[featureId] ?? entry.grouped_values?.[featureId];+ if (!values) continue;++ credits ??= {};+ for (const [group, value] of Object.entries(values)) {+ credits[group] = (credits[group] ?? 0) + value;+ }+ }+ return credits;}-/** One day of credits used, keyed by the `apiKeyId` Autumn grouped on. */-interface ApiKeyUsageDay {+/** One day of credits used, keyed by the property Autumn grouped on. */+interface GroupedUsageDay {period: number;credits: Record<string, number>;}-async function aggregateHistoricalPeriodsByApiKeyMonth(- days: ApiKeyUsageDay[],-): Promise<HistoricalPeriodByApiKey[]> {- const monthApiKeyTotals = new Map<string, Map<string, number>>();- const allApiKeyIds = new Set<string>();+/**+ * Rolls daily grouped usage into bins. Groups whose labels match (e.g.+ * several unresolved API key ids all labelled "Unknown") are merged, so the+ * result has one row per label per bin, sorted by label.+ */+function aggregateGroupedPeriods(+ days: GroupedUsageDay[],+ bin: HistoricalBin,+ label: (group: string) => string,+): {+ startDate: string;+ endDate: string | null;+ label: string;+ creditsUsed: number;+}[] {+ const binGroupTotals = new Map<string, Map<string, number>>();for (const day of days) {- const monthStart = toMonthStartIso(day.period);- if (!monthStart) continue;+ const binStart = toBinStartIso(day.period, bin);+ if (!binStart) continue;- const monthTotals =- monthApiKeyTotals.get(monthStart) ?? new Map<string, number>();+ const binTotals = binGroupTotals.get(binStart) ?? new Map<string, number>();- for (const [apiKeyId, creditsUsed] of Object.entries(day.credits)) {- allApiKeyIds.add(apiKeyId);- monthTotals.set(apiKeyId, (monthTotals.get(apiKeyId) ?? 0) + creditsUsed);+ for (const [group, creditsUsed] of Object.entries(day.credits)) {+ binTotals.set(group, (binTotals.get(group) ?? 0) + creditsUsed);}- monthApiKeyTotals.set(monthStart, monthTotals);+ binGroupTotals.set(binStart, binTotals);}- const nameMap = await lookupApiKeyNames([...allApiKeyIds]);- const monthStarts = [...monthApiKeyTotals.keys()].sort();- const results: HistoricalPeriodByApiKey[] = [];-- for (let i = 0; i < monthStarts.length; i++) {- const startDate = monthStarts[i];- const endDate = i < monthStarts.length - 1 ? nextMonthIso(startDate) : null;- const monthTotals = monthApiKeyTotals.get(startDate);-- if (!monthTotals) continue;-- // Collapse rows whose IDs resolve to the same display name (e.g. multiple- // unresolved IDs all show as "Unknown") so the response has one row per- // name per month.- const byName = new Map<string, number>();- for (const [apiKeyId, creditsUsed] of monthTotals) {- const name = nameMap[apiKeyId];- byName.set(name, (byName.get(name) ?? 0) + creditsUsed);+ const binStarts = [...binGroupTotals.keys()].sort();+ const results: {+ startDate: string;+ endDate: string | null;+ label: string;+ creditsUsed: number;+ }[] = [];++ for (let i = 0; i < binStarts.length; i++) {+ const startDate = binStarts[i];+ const endDate =+ i < binStarts.length - 1 ? nextBinIso(startDate, bin) : null;+ const binTotals = binGroupTotals.get(startDate);++ if (!binTotals) continue;++ const byLabel = new Map<string, number>();+ for (const [group, creditsUsed] of binTotals) {+ const name = label(group);+ byLabel.set(name, (byLabel.get(name) ?? 0) + creditsUsed);}- for (const [apiKey, creditsUsed] of [...byName.entries()].sort(([a], [b]) =>+ for (const [name, creditsUsed] of [...byLabel.entries()].sort(([a], [b]) =>a.localeCompare(b),)) {- results.push({- startDate,- endDate,- apiKey,- creditsUsed,- });+ results.push({ startDate, endDate, label: name, creditsUsed });}}return results;}+async function aggregateHistoricalPeriodsByApiKey(+ days: GroupedUsageDay[],+ bin: HistoricalBin,+): Promise<HistoricalPeriodByApiKey[]> {+ const apiKeyIds = new Set(days.flatMap(day => Object.keys(day.credits)));+ const nameMap = await lookupApiKeyNames([...apiKeyIds]);++ return aggregateGroupedPeriods(days, bin, apiKeyId => nameMap[apiKeyId]).map(+ ({ startDate, endDate, label, creditsUsed }) => ({+ startDate,+ endDate,+ apiKey: label,+ creditsUsed,+ }),+ );+}++function aggregateHistoricalPeriodsByEndpoint(+ days: GroupedUsageDay[],+ bin: HistoricalBin,+): HistoricalPeriodByEndpoint[] {+ return aggregateGroupedPeriods(+ days,+ bin,+ endpoint => endpoint || "unknown",+ ).map(({ startDate, endDate, label, creditsUsed }) => ({+ startDate,+ endDate,+ endpoint: label,+ creditsUsed,+ }));+}+// ---------------------------------------------------------------------------// Balance (current billing period)// ---------------------------------------------------------------------------@@ -408,17 +498,26 @@ interface HistoricalPeriodByApiKey {creditsUsed: number;}+interface HistoricalPeriodByEndpoint {+ startDate: string | null;+ endDate: string | null;+ endpoint: string;+ creditsUsed: number;+}+/*** Fetches a team's historical credit usage across billing periods from Autumn.** Scopes the aggregate to the team's Autumn entity so the response reflects* only that team's usage, not the whole org. Uses `events.aggregate` with the* last 90 days of daily usage and rolls those daily totals into calendar-month- * buckets in API code. A team with no entity (never provisioned) has no- * team-scoped usage, so we return an empty history rather than the org total.+ * (or, with `bin` "day", daily) buckets in API code. A team with no entity+ * (never provisioned) has no team-scoped usage, so we return an empty history+ * rather than the org total.*/export async function getTeamHistoricalUsage(teamId: string,+ bin: HistoricalBin = "month",): Promise<HistoricalPeriod[]> {if (!autumnClient) {throw new Error(@@ -447,7 +546,7 @@ export async function getTeamHistoricalUsage(return [];}- return aggregateHistoricalPeriodsByMonth(response.list ?? []);+ return aggregateHistoricalPeriods(response.list ?? [], bin);}interface UsageSlice {@@ -460,21 +559,24 @@ interface UsageSlice {* are aligned to the Unix epoch rather than to the window, so a slice keeps* the same bounds (and cache key) while the window slides over it.*/-function byApiKeySlices(windowStart: number, now: number): UsageSlice[] {+function groupedSlices(windowStart: number, now: number): UsageSlice[] {const slices: UsageSlice[] = [];- let start =- Math.floor(windowStart / BY_API_KEY_SLICE_MS) * BY_API_KEY_SLICE_MS;- for (; start <= now; start += BY_API_KEY_SLICE_MS) {- slices.push({ start, end: start + BY_API_KEY_SLICE_MS });+ let start = Math.floor(windowStart / GROUPED_SLICE_MS) * GROUPED_SLICE_MS;+ for (; start <= now; start += GROUPED_SLICE_MS) {+ slices.push({ start, end: start + GROUPED_SLICE_MS });}return slices;}-function sliceCacheKey(teamId: string, slice: UsageSlice): string {- return `historical-usage-by-api-key:v1:${teamId}:${slice.start}`;+function sliceCacheKey(+ grouping: UsageGrouping,+ teamId: string,+ slice: UsageSlice,+): string {+ return `${grouping.cachePrefix}:${teamId}:${slice.start}`;}-function isApiKeyUsageDays(value: unknown): value is ApiKeyUsageDay[] {+function isGroupedUsageDays(value: unknown): value is GroupedUsageDay[] {return (Array.isArray(value) &&value.every(@@ -489,14 +591,15 @@ function isApiKeyUsageDays(value: unknown): value is ApiKeyUsageDay[] {}async function readCachedSlice(+ grouping: UsageGrouping,teamId: string,slice: UsageSlice,-): Promise<ApiKeyUsageDay[] | null> {+): Promise<GroupedUsageDay[] | null> {try {- const cached = await getValue(sliceCacheKey(teamId, slice));+ const cached = await getValue(sliceCacheKey(grouping, teamId, slice));if (cached === null) return null;const parsed = JSON.parse(cached);- if (isApiKeyUsageDays(parsed)) return parsed;+ if (isGroupedUsageDays(parsed)) return parsed;logger.warn("Ignoring malformed historical usage cache entry", {teamId,sliceStart: slice.start,@@ -512,15 +615,16 @@ async function readCachedSlice(}async function writeCachedSlice(+ grouping: UsageGrouping,teamId: string,slice: UsageSlice,- days: ApiKeyUsageDay[],+ days: GroupedUsageDay[],): Promise<void> {try {await setValue(- sliceCacheKey(teamId, slice),+ sliceCacheKey(grouping, teamId, slice),JSON.stringify(days),- BY_API_KEY_SLICE_CACHE_TTL_SECONDS,+ GROUPED_SLICE_CACHE_TTL_SECONDS,);} catch (error) {logger.warn("Failed to cache historical usage slice", {@@ -531,36 +635,37 @@ async function writeCachedSlice(}}-async function fetchApiKeyUsageSlice(+async function fetchGroupedUsageSlice(client: NonNullable<typeof autumnClient>,+ grouping: UsageGrouping,orgId: string,teamId: string,slice: UsageSlice,now: number,-): Promise<ApiKeyUsageDay[]> {+): Promise<GroupedUsageDay[]> {const response: any = await client.events.aggregate({customerId: orgId,entityId: teamId,- featureId: CREDITS_FEATURE_ID,+ featureId: grouping.featureId,customRange: { start: slice.start, end: Math.min(slice.end, now) },binSize: HISTORICAL_BIN_SIZE,- groupBy: "properties.apiKeyId",- maxGroups: BY_API_KEY_MAX_GROUPS,+ groupBy: grouping.groupBy,+ maxGroups: GROUPED_MAX_GROUPS,},{timeoutMs: HISTORICAL_AGGREGATE_TIMEOUT_MS,- ...BY_API_KEY_RATE_LIMIT_RETRY,+ ...GROUPED_RATE_LIMIT_RETRY,},);- const days: ApiKeyUsageDay[] = [];+ const days: GroupedUsageDay[] = [];for (const entry of response?.list ?? []) {const period = new Date(entry.period).getTime();// A bin outside the slice belongs to its neighbour, which counts it.if (!(period >= slice.start && period < slice.end)) continue;- const credits = getGroupedCredits(entry);+ const credits = getGroupedCredits(entry, grouping);if (credits) days.push({ period, credits });}return days;@@ -599,18 +704,16 @@ async function mapWithConcurrency<T, R>(}/**- * Fetches a team's historical credit usage grouped by API key from Autumn.- *- * Covers the 90 days up to now, from the start of that UTC day, as daily- * usage grouped by `properties.apiKeyId`, and rolls those daily totals into- * calendar-month buckets in API code. The days come from 7-day slices (see- * `BY_API_KEY_SLICE_MS`): the slice holding today is always asked for live and- * first, so a team with no entity answers 404 there and gets an empty- * history; settled slices come from the cache when they can.+ * Fetches a team's daily usage over the 90 days up to now, from the start of+ * that UTC day, grouped by the grouping's event property. The days come from+ * 7-day slices (see `GROUPED_SLICE_MS`): the slice holding today is always+ * asked for live and first, so a team with no entity answers 404 there and+ * gets no days; settled slices come from the cache when they can.*/-export async function getTeamHistoricalUsageByApiKey(+async function getTeamGroupedUsageDays(teamId: string,-): Promise<HistoricalPeriodByApiKey[]> {+ grouping: UsageGrouping,+): Promise<GroupedUsageDay[]> {const client = autumnClient;if (!client) {throw new Error(@@ -623,14 +726,15 @@ export async function getTeamHistoricalUsageByApiKey(const now = Date.now();const windowStart =Math.floor((now - HISTORICAL_WINDOW_MS) / DAY_MS) * DAY_MS;- const slices = byApiKeySlices(windowStart, now);+ const slices = groupedSlices(windowStart, now);const currentSlice = slices[slices.length - 1];const earlierSlices = slices.slice(0, -1);- let currentDays: ApiKeyUsageDay[];+ let currentDays: GroupedUsageDay[];try {- currentDays = await fetchApiKeyUsageSlice(+ currentDays = await fetchGroupedUsageSlice(client,+ grouping,orgId,teamId,currentSlice,@@ -645,30 +749,60 @@ export async function getTeamHistoricalUsageByApiKey(const earlierDays = await mapWithConcurrency(earlierSlices,- BY_API_KEY_SLICE_CONCURRENCY,+ GROUPED_SLICE_CONCURRENCY,async slice => {- const settled = slice.end <= now - BY_API_KEY_SLICE_SETTLE_MS;+ const settled = slice.end <= now - GROUPED_SLICE_SETTLE_MS;if (settled) {- const cached = await readCachedSlice(teamId, slice);+ const cached = await readCachedSlice(grouping, teamId, slice);if (cached !== null) return cached;}- const days = await fetchApiKeyUsageSlice(+ const days = await fetchGroupedUsageSlice(client,+ grouping,orgId,teamId,slice,now,);- if (settled) await writeCachedSlice(teamId, slice, days);+ if (settled) await writeCachedSlice(grouping, teamId, slice, days);return days;},);- const days = [...earlierDays.flat(), ...currentDays].filter(+ return [...earlierDays.flat(), ...currentDays].filter(day => day.period >= windowStart,);- return aggregateHistoricalPeriodsByApiKeyMonth(days);+}++/**+ * Fetches a team's historical credit usage grouped by API key from Autumn,+ * rolled into calendar-month (or, with `bin` "day", daily) buckets.+ */+export async function getTeamHistoricalUsageByApiKey(+ teamId: string,+ bin: HistoricalBin = "month",+): Promise<HistoricalPeriodByApiKey[]> {+ return aggregateHistoricalPeriodsByApiKey(+ await getTeamGroupedUsageDays(teamId, API_KEY_GROUPING),+ bin,+ );+}++/**+ * Fetches a team's historical credit usage grouped by the endpoint that+ * billed it (scrape, search, crawl, ...) from Autumn, rolled into+ * calendar-month (or, with `bin` "day", daily) buckets. Unlike the totals+ * and the byApiKey breakdown, it includes search usage.+ */+export async function getTeamHistoricalUsageByEndpoint(+ teamId: string,+ bin: HistoricalBin = "month",+): Promise<HistoricalPeriodByEndpoint[]> {+ return aggregateHistoricalPeriodsByEndpoint(+ await getTeamGroupedUsageDays(teamId, ENDPOINT_GROUPING),+ bin,+ );}/**