Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
🏦 fix: Reserve Credits for In-Flight Requests (#15879)
Danny Avila696bcb6c1e29
diff --git a/api/app/clients/BaseClient.js b/api/app/clients/BaseClient.jsindex 950efff002..248f336a22 100644--- a/api/app/clients/BaseClient.js+++ b/api/app/clients/BaseClient.js@@ -20,6 +20,7 @@ const {collectModelBoundHistoricalFileIdState,projectModelBoundSourceFiles,isModelBoundAttachmentFile,+ withBalanceReservations,} = require('@librechat/api');const {Constants,@@ -732,6 +733,18 @@ class BaseClient {}async sendMessage(message, opts = {}) {+ return withBalanceReservations((balanceReservations) =>+ this.sendReservedMessage(message, opts, balanceReservations),+ );+ }++ /**+ * @param {string} message+ * @param {Record<string, unknown>} opts+ * @param {BalanceReservations} balanceReservations - Holds the balance reservation admitting+ * this message; released once its usage is recorded, and by `sendMessage` on any other exit.+ */+ async sendReservedMessage(message, opts, balanceReservations) {const appConfig = this.options.req?.config;/** @type {Promise<TMessage>} */let userMessagePromise;@@ -974,7 +987,7 @@ class BaseClient {balanceConfig?.enabled &&supportsBalanceCheck[this.options.endpointType ?? this.options.endpoint]) {- await checkBalance(+ const balanceAdmission = checkBalance({req: this.options.req,res: this.options.res,@@ -990,12 +1003,13 @@ class BaseClient {{logViolation,getMultiplier: db.getMultiplier,- findBalanceByUser: db.findBalanceByUser,- createAutoRefillTransaction: db.createAutoRefillTransaction,+ reserveBalance: db.reserveBalance,+ renewBalanceReservation: db.renewBalanceReservation,+ releaseBalanceReservation: db.releaseBalanceReservation,balanceConfig,- upsertBalanceFields: db.upsertBalanceFields,},);+ await balanceReservations.track(balanceAdmission);}completionResult = await this.sendCompletion(payload, opts);@@ -1155,6 +1169,7 @@ class BaseClient {completionTokens,});}+ await balanceReservations.release();if (userMessagePromise) {await userMessagePromise;diff --git a/api/app/clients/specs/BaseClient.test.js b/api/app/clients/specs/BaseClient.test.jsindex 8f2bbbafee..3aa013cc8e 100644--- a/api/app/clients/specs/BaseClient.test.js+++ b/api/app/clients/specs/BaseClient.test.js@@ -48,9 +48,22 @@ jest.mock('~/models', () => ({deleteFiles: jest.fn(),getFiles: jest.fn(),updateFileUsage: jest.fn(),+ getMultiplier: jest.fn(),+ reserveBalance: jest.fn(),+ renewBalanceReservation: jest.fn(),+ releaseBalanceReservation: jest.fn(),}));-const { getConvo, getFiles, getMessages, saveConvo, saveMessage } = require('~/models');+const {+ releaseBalanceReservation,+ reserveBalance,+ getMultiplier,+ saveMessage,+ getMessages,+ saveConvo,+ getFiles,+ getConvo,+} = require('~/models');jest.mock('@librechat/agents', () => {const actual = jest.requireActual('@librechat/agents');@@ -2225,6 +2238,91 @@ describe('BaseClient', () => {});});+ describe('balance reservation lifecycle', () => {+ let priorEndpoint;+ let priorEndpointType;+ let events;++ beforeEach(() => {+ priorEndpoint = TestClient.options.endpoint;+ priorEndpointType = TestClient.options.endpointType;+ TestClient.options.endpoint = EModelEndpoint.openAI;+ delete TestClient.options.endpointType;+ TestClient.options.req = { config: { balance: { enabled: true } } };++ events = [];+ getMultiplier.mockReturnValue(1);+ reserveBalance.mockImplementation(async () => {+ events.push('reserve');+ return { reserved: true, balance: 1000 };+ });+ releaseBalanceReservation.mockImplementation(async () => {+ events.push('release');+ });+ TestClient.sendCompletion.mockImplementation(async () => {+ events.push('completion');+ return { completion: 'Mock response text', metadata: undefined };+ });+ TestClient.getTokenCountForResponse = jest.fn().mockReturnValue(50);+ TestClient.recordTokenUsage = jest.fn(async () => {+ events.push('usage');+ });+ TestClient.buildMessages.mockReturnValue({+ prompt: [],+ tokenCountMap: { res: 50 },+ });+ });++ afterEach(() => {+ delete TestClient.options.req;+ TestClient.options.endpoint = priorEndpoint;+ TestClient.options.endpointType = priorEndpointType;+ });++ test('releases the reservation once the response usage is recorded, before persistence', async () => {+ const beforeResponsePersistence = jest.fn(async () => {+ events.push('persist');+ return true;+ });++ await TestClient.sendMessage('Hello', { beforeResponsePersistence });++ expect(events).toEqual(['reserve', 'completion', 'usage', 'release', 'persist']);+ const [{ reservationId, amount }] = reserveBalance.mock.calls[0];+ expect(releaseBalanceReservation).toHaveBeenCalledTimes(1);+ expect(releaseBalanceReservation).toHaveBeenCalledWith({+ user: TestClient.user,+ reservationId,+ amount,+ });+ });++ test('releases the reservation when the completion fails', async () => {+ TestClient.sendCompletion.mockRejectedValue(new Error('provider unavailable'));++ await expect(TestClient.sendMessage('Hello', {})).rejects.toThrow('provider unavailable');++ expect(events).toEqual(['reserve', 'release']);+ });++ test('releases the reservation when work after the completion fails', async () => {+ TestClient.recordTokenUsage.mockRejectedValue(new Error('usage write failed'));++ await expect(TestClient.sendMessage('Hello', {})).rejects.toThrow('usage write failed');++ expect(events).toEqual(['reserve', 'completion', 'release']);+ });++ test('takes no reservation when the balance check refuses the request', async () => {+ reserveBalance.mockResolvedValue({ reserved: false, balance: 0 });++ await expect(TestClient.sendMessage('Hello', {})).rejects.toThrow();++ expect(TestClient.sendCompletion).not.toHaveBeenCalled();+ expect(releaseBalanceReservation).not.toHaveBeenCalled();+ });+ });+describe('getMessagesWithinTokenLimit with instructions', () => {test('should always include instructions when present', async () => {TestClient.maxContextTokens = 50;diff --git a/api/server/controllers/assistants/chat.contentFilter.spec.js b/api/server/controllers/assistants/chat.contentFilter.spec.jsindex 9b3e630102..f5f9ccc7e8 100644--- a/api/server/controllers/assistants/chat.contentFilter.spec.js+++ b/api/server/controllers/assistants/chat.contentFilter.spec.js@@ -112,9 +112,9 @@ jest.mock('~/server/middleware/error', () => ({}));jest.mock('~/models', () => ({- createAutoRefillTransaction: jest.fn(),- findBalanceByUser: jest.fn(),- upsertBalanceFields: jest.fn(),+ releaseBalanceReservation: jest.fn(),+ renewBalanceReservation: jest.fn(),+ reserveBalance: jest.fn(),getTransactions: jest.fn(),getMultiplier: jest.fn(),getConvo: (...args) => mockGetConvo(...args),@@ -137,6 +137,7 @@ jest.mock('./helpers', () => ({const chatV1 = require('./chatV1');const chatV2 = require('./chatV2');const { logger } = require('@librechat/data-schemas');+const { checkBalance, getBalanceConfig } = require('@librechat/api');const { ImageVisionTool } = require('librechat-data-provider');describe.each([@@ -237,6 +238,22 @@ describe.each([expect(mockSendResponse).not.toHaveBeenCalled();}+ it('releases a balance reservation that settles after thread initialization fails', async () => {+ req.config.filters = {};+ const release = jest.fn().mockResolvedValue(undefined);+ getBalanceConfig.mockReturnValue({ enabled: true });+ checkBalance.mockImplementation(+ () => new Promise((resolve) => setTimeout(() => resolve({ release }), 50)),+ );+ mockInitThread.mockRejectedValueOnce(new Error('stop after initThread'));++ await chatController(req, res);++ expect(mockInitThread).toHaveBeenCalledTimes(1);+ expect(checkBalance).toHaveBeenCalledTimes(1);+ expect(release).toHaveBeenCalledTimes(1);+ });+it('blocks persisted instructions before thread, message, run, or stream side effects', async () => {req.config.filters = {agentInstructions: {@@ -571,6 +588,46 @@ describe.each([}if (_version === 'v2') {+ it('keeps the balance reservation until a run that continued in the background settles', async () => {+ req.config.filters = {};+ const release = jest.fn().mockResolvedValue(undefined);+ getBalanceConfig.mockReturnValue({ enabled: true });+ checkBalance.mockResolvedValue({ release });+ mockInitThread.mockResolvedValueOnce({ thread_id: 'thread-existing' });+ let finishBackgroundRun = () => undefined;+ const usage = { prompt_tokens: 1, completion_tokens: 1 };+ mockStreamRunManager+ .mockImplementationOnce(() => ({+ runAssistant: jest.fn().mockResolvedValue(undefined),+ run: { id: 'run-1', status: 'in_progress', usage },+ intermediateText: '',+ messages: [],+ }))+ .mockImplementationOnce(() => ({+ runAssistant: jest.fn(() => new Promise((resolve) => (finishBackgroundRun = resolve))),+ run: { id: 'run-1', status: 'completed', usage },+ intermediateText: '',+ messages: [],+ }));++ const handled = chatController(req, res);+ for (let i = 0; i < 50 && !res.end.mock.calls.length; i++) {+ await new Promise((resolve) => setImmediate(resolve));+ }+ for (let i = 0; i < 20; i++) {+ await new Promise((resolve) => setImmediate(resolve));+ }++ expect(mockStreamRunManager).toHaveBeenCalledTimes(2);+ expect(mockHandleError.mock.calls.map(([error]) => error?.message)).toEqual([]);+ expect(res.end).toHaveBeenCalled();+ expect(release).not.toHaveBeenCalled();++ finishBackgroundRun();+ await handled;+ expect(release).toHaveBeenCalledTimes(1);+ });+describe('V2 final conversation-file preflight', () => {beforeEach(() => {req.config.filters = {diff --git a/api/server/controllers/assistants/chatV1.js b/api/server/controllers/assistants/chatV1.jsindex 5db12b8700..a185e544c9 100644--- a/api/server/controllers/assistants/chatV1.js+++ b/api/server/controllers/assistants/chatV1.js@@ -5,6 +5,7 @@ const {sendEvent,countTokens,checkBalance,+ createBalanceReservations,getBalanceConfig,getSafeErrorText,getModelMaxTokens,@@ -47,10 +48,10 @@ const { createRunBody } = require('~/server/services/createRunBody');const { sendResponse } = require('~/server/middleware/error');const setHeaders = require('~/server/middleware/setHeaders');const {- createAutoRefillTransaction,- findBalanceByUser,- upsertBalanceFields,+ releaseBalanceReservation,+ renewBalanceReservation,getTransactions,+ reserveBalance,getMultiplier,getConvo,getFiles,@@ -126,6 +127,7 @@ const chatV1 = async (req, res) => {/** @type {Run | undefined} - The completed run, undefined if incomplete */let completedRun;let contentRejected = false;+ const balanceReservations = createBalanceReservations();const handleError = async (error) => {const defaultErrorMessage =@@ -304,7 +306,7 @@ const chatV1 = async (req, res) => {// Count tokens up to the current context windowpromptTokens = Math.min(promptTokens, getModelMaxTokens(model));- await checkBalance(+ return await checkBalance({req,res,@@ -316,12 +318,12 @@ const chatV1 = async (req, res) => {},},{- findBalanceByUser,getMultiplier,- createAutoRefillTransaction,+ reserveBalance,+ renewBalanceReservation,+ releaseBalanceReservation,logViolation,balanceConfig,- upsertBalanceFields,},);};@@ -564,7 +566,7 @@ const chatV1 = async (req, res) => {}}- const promises = [initializeThread(), checkBalanceBeforeRun()];+ const promises = [initializeThread(), balanceReservations.track(checkBalanceBeforeRun())];await Promise.all(promises);const sendInitialResponse = () => {@@ -683,7 +685,7 @@ const chatV1 = async (req, res) => {}if (response.run.status === RunStatus.IN_PROGRESS) {- processRun(true);+ balanceReservations.holdUntil(processRun(true));}completedRun = response.run;@@ -755,6 +757,8 @@ const chatV1 = async (req, res) => {}} catch (error) {await handleError(error);+ } finally {+ await balanceReservations.release();}};diff --git a/api/server/controllers/assistants/chatV2.js b/api/server/controllers/assistants/chatV2.jsindex b79053ba2a..24f3b97691 100644--- a/api/server/controllers/assistants/chatV2.js+++ b/api/server/controllers/assistants/chatV2.js@@ -5,6 +5,7 @@ const {sendEvent,countTokens,checkBalance,+ createBalanceReservations,getBalanceConfig,getTransactionsConfig,getModelMaxTokens,@@ -44,9 +45,9 @@ const {getConvo,getMultiplier,getTransactions,- findBalanceByUser,- upsertBalanceFields,- createAutoRefillTransaction,+ reserveBalance,+ renewBalanceReservation,+ releaseBalanceReservation,getFiles,} = require('~/models');const { logViolation, getLogStores } = require('~/cache');@@ -118,6 +119,7 @@ const chatV2 = async (req, res) => {/** @type {Run | undefined} - The completed run, undefined if incomplete */let completedRun;let contentRejected = false;+ const balanceReservations = createBalanceReservations();const getContext = () => ({openai,@@ -175,7 +177,7 @@ const chatV2 = async (req, res) => {// Count tokens up to the current context windowpromptTokens = Math.min(promptTokens, getModelMaxTokens(model));- await checkBalance(+ return await checkBalance({req,res,@@ -187,12 +189,12 @@ const chatV2 = async (req, res) => {},},{- findBalanceByUser,getMultiplier,- createAutoRefillTransaction,+ reserveBalance,+ renewBalanceReservation,+ releaseBalanceReservation,logViolation,balanceConfig,- upsertBalanceFields,},);};@@ -395,7 +397,7 @@ const chatV2 = async (req, res) => {}}- const promises = [initializeThread(), checkBalanceBeforeRun()];+ const promises = [initializeThread(), balanceReservations.track(checkBalanceBeforeRun())];await Promise.all(promises);const sendInitialResponse = () => {@@ -520,7 +522,7 @@ const chatV2 = async (req, res) => {}if (response.run.status === RunStatus.IN_PROGRESS) {- processRun(true);+ balanceReservations.holdUntil(processRun(true));}completedRun = response.run;@@ -593,6 +595,8 @@ const chatV2 = async (req, res) => {}} catch (error) {await handleError(error);+ } finally {+ await balanceReservations.release();}};diff --git a/api/server/services/Schedules/index.js b/api/server/services/Schedules/index.jsindex 4e9c10101c..50944efd01 100644--- a/api/server/services/Schedules/index.js+++ b/api/server/services/Schedules/index.js@@ -27,18 +27,9 @@ function getService() {getAppConfig,findUserById: (userId) =>mongoose.models.User.findById(userId).select('_id tenantId role').lean(),- findBalance: (userId) => mongoose.models.Balance.findOne({ user: userId }).lean(),+ findBalance: (userId) => methods.findBalanceByUser(userId, { includeReservedCredits: true }),upsertBalance: (userId, { set, setOnInsert }) =>- mongoose.models.Balance.findOneAndUpdate(- { user: userId },- {- ...(set && Object.keys(set).length > 0 ? { $set: set } : {}),- ...(setOnInsert && Object.keys(setOnInsert).length > 0- ? { $setOnInsert: setOnInsert }- : {}),- },- { upsert: true, new: true },- ).lean(),+ methods.upsertBalanceFields(userId, set ?? {}, setOnInsert ?? {}),// Compare-and-set: only initialize an existing record while its credit is still null.// No upsert — a CAS miss must re-read the winner, not insert a fresh balance. `null`// in the filter also matches a legacy record whose `tokenCredits` field is absent.@@ -46,7 +37,7 @@ function getService() {mongoose.models.Balance.findOneAndUpdate({ user: userId, tokenCredits: null },{ $set: { tokenCredits, ...(sync && Object.keys(sync).length > 0 ? sync : {}) } },- { new: true },+ { new: true, sort: { _id: 1 } },).lean(),enqueueAgentTrigger,// Reconciliation reads the durable delivery to tell a still-live admission (adiff --git a/api/typedefs.js b/api/typedefs.jsindex 92839f48d4..81bc2b2ea9 100644--- a/api/typedefs.js+++ b/api/typedefs.js@@ -1114,6 +1114,18 @@* @memberof typedefs*/+/**+ * @exports BalanceReservation+ * @typedef {import('@librechat/api').BalanceReservation} BalanceReservation+ * @memberof typedefs+ */++/**+ * @exports BalanceReservations+ * @typedef {import('@librechat/api').BalanceReservations} BalanceReservations+ * @memberof typedefs+ */+/*** @exports Keyv* @typedef {import('keyv')} Keyvdiff --git a/config/list-balances.js b/config/list-balances.jsindex a07407fd13..fd494308e8 100644--- a/config/list-balances.js+++ b/config/list-balances.js@@ -17,7 +17,7 @@ const connect = require('./connect');let users = await User.find({});for (const user of users) {- let balance = await Balance.findOne({ user: user._id });+ let balance = await Balance.findOne({ user: user._id }).sort({ _id: 1 });if (balance !== null) {console.green(`User ${user.name} (${user.email}) has a balance of ${balance.tokenCredits}`);} else {diff --git a/config/set-balance.js b/config/set-balance.jsindex 7e5a2ee3e0..68bc6f56b0 100644--- a/config/set-balance.js+++ b/config/set-balance.js@@ -60,7 +60,7 @@ const connect = require('./connect');console.purple(`Found user: ${user.email}`);}- let balance = await Balance.findOne({ user: user._id }).lean();+ let balance = await Balance.findOne({ user: user._id }).sort({ _id: 1 }).lean();if (!balance) {console.purple('User has no balance!');} else {@@ -81,11 +81,17 @@ const connect = require('./connect');*/let result;try {- result = await Balance.findOneAndUpdate(- { user: user._id },- { tokenCredits: amount },- { upsert: true, new: true },- ).lean();+ result =+ (await Balance.findOneAndUpdate(+ { user: user._id },+ { tokenCredits: amount },+ { new: true, sort: { _id: 1 } },+ ).lean()) ??+ (await Balance.findOneAndUpdate(+ { _id: user._id },+ { $set: { tokenCredits: amount }, $setOnInsert: { user: user._id } },+ { upsert: true, new: true },+ ).lean());} catch (error) {console.red('Error: ' + error.message);console.error(error);diff --git a/librechat.example.yaml b/librechat.example.yamlindex 928588d234..c743bb6445 100644--- a/librechat.example.yaml+++ b/librechat.example.yaml@@ -355,6 +355,7 @@ registration:# refillIntervalValue: 30# refillIntervalUnit: 'days'# refillAmount: 10000+# reservationTtlMs: 1800000 # How long an in-flight request's reserved credits count if never released# Example Transactions settings# Controls whether to save transaction records to the databasediff --git a/packages/api/src/middleware/balance.spec.ts b/packages/api/src/middleware/balance.spec.tsindex b55ee82271..20fd333c66 100644--- a/packages/api/src/middleware/balance.spec.ts+++ b/packages/api/src/middleware/balance.spec.ts@@ -17,10 +17,14 @@ let Balance: mongoose.Model<IBalance>;const findBalanceByUser = (userId: string) => Balance.findOne({ user: userId }).lean<IBalance>();-const upsertBalanceFields = (userId: string, fields: IBalanceUpdate) =>+const upsertBalanceFields = (+ userId: string,+ fields: IBalanceUpdate,+ insertOnly: IBalanceUpdate = {},+) =>Balance.findOneAndUpdate({ user: userId },- { $set: fields },+ { $set: fields, $setOnInsert: insertOnly },{ upsert: true, new: true },).lean<IBalance>();@@ -59,6 +63,30 @@ describe('createSetBalanceConfig', () => {const mockNext: NextFunction = jest.fn();describe('Basic Functionality', () => {+ test('does not overwrite credits another writer set after the balance was read', async () => {+ const userId = new mongoose.Types.ObjectId();+ const getAppConfig = jest.fn().mockResolvedValue({+ balance: { enabled: true, startBalance: 1000 },+ });+ const findThenCharge = async (id: string) => {+ const record = await findBalanceByUser(id);+ await Balance.create({ user: id, tokenCredits: 50 });+ return record;+ };++ const middleware = createSetBalanceConfig({+ getAppConfig,+ findBalanceByUser: findThenCharge,+ upsertBalanceFields,+ });+ const res = createMockResponse();++ await middleware(createMockRequest(userId) as ServerRequest, res as ServerResponse, mockNext);++ expect((await Balance.findOne({ user: userId }).lean())?.tokenCredits).toBe(50);+ expect((res.locals as { balanceData?: IBalance }).balanceData?.tokenCredits).toBe(50);+ });+test('should create balance record for new user with start balance', async () => {const userId = new mongoose.Types.ObjectId();const getAppConfig = jest.fn().mockResolvedValue({diff --git a/packages/api/src/middleware/balance.ts b/packages/api/src/middleware/balance.tsindex c6e8978b9e..44cb34d7e9 100644--- a/packages/api/src/middleware/balance.ts+++ b/packages/api/src/middleware/balance.ts@@ -19,7 +19,11 @@ export interface BalanceMiddlewareOptions {refresh?: boolean;}) => Promise<AppConfig>;findBalanceByUser: (userId: string) => Promise<IBalance | null>;- upsertBalanceFields: (userId: string, fields: IBalanceUpdate) => Promise<IBalance | null>;+ upsertBalanceFields: (+ userId: string,+ fields: IBalanceUpdate,+ insertOnly?: IBalanceUpdate,+ ) => Promise<IBalance | null>;}type BalanceLocals = {@@ -151,6 +155,14 @@ export function createSetBalanceConfig({return;}+ if (userBalanceRecord == null) {+ const { tokenCredits, ...syncFields } = updateFields;+ balanceLocals.balanceData = await upsertBalanceFields(userId, syncFields, {+ tokenCredits,+ });+ return;+ }+balanceLocals.balanceData = await upsertBalanceFields(userId, updateFields);});diff --git a/packages/api/src/middleware/checkBalance.spec.ts b/packages/api/src/middleware/checkBalance.spec.tsindex 8d272d2e60..d976f51cb0 100644--- a/packages/api/src/middleware/checkBalance.spec.ts+++ b/packages/api/src/middleware/checkBalance.spec.ts@@ -1,22 +1,32 @@-import { ViolationTypes } from 'librechat-data-provider';+import mongoose from 'mongoose';+import { MongoMemoryServer } from 'mongodb-memory-server';+import { logger, createMethods, createModels } from '@librechat/data-schemas';+import {+ ViolationTypes,+ MIN_BALANCE_RESERVATION_TTL_MS,+ DEFAULT_BALANCE_RESERVATION_TTL_MS,+} from 'librechat-data-provider';+import type { BalanceConfig, IBalance } from '@librechat/data-schemas';import type { Response } from 'express';-import type { CheckBalanceDeps } from './checkBalance';+import type { BalanceReservation, CheckBalanceDeps } from './checkBalance';import type { ServerRequest } from '~/types/http';-import { checkBalance } from './checkBalance';+import { checkBalance, createBalanceReservations, withBalanceReservations } from './checkBalance';jest.mock('@librechat/data-schemas', () => ({...jest.requireActual('@librechat/data-schemas'),logger: {debug: jest.fn(),error: jest.fn(),+ warn: jest.fn(),},}));describe('checkBalance', () => {const createMockDeps = (overrides: Partial<CheckBalanceDeps> = {}): CheckBalanceDeps => ({- findBalanceByUser: jest.fn().mockResolvedValue({ tokenCredits: 1000 }),+ reserveBalance: jest.fn().mockResolvedValue({ reserved: true, balance: 1000 }),+ renewBalanceReservation: jest.fn().mockResolvedValue(undefined),+ releaseBalanceReservation: jest.fn().mockResolvedValue(undefined),getMultiplier: jest.fn().mockReturnValue(1),- createAutoRefillTransaction: jest.fn(),logViolation: jest.fn().mockResolvedValue(undefined),...overrides,});@@ -32,17 +42,52 @@ describe('checkBalance', () => {model: 'gpt-4',};- it('should return true when user has sufficient balance', async () => {- const deps = createMockDeps();+ const reserveRequest = (deps: CheckBalanceDeps) =>+ (deps.reserveBalance as jest.Mock).mock.calls[0][0];- const result = await checkBalance({ req, res, txData: baseTxData }, deps);- expect(result).toBe(true);+ it('reserves the token cost and releases that reservation exactly once', async () => {+ const deps = createMockDeps({ getMultiplier: jest.fn().mockReturnValue(2) });++ const reservation = await checkBalance({ req, res, txData: baseTxData }, deps);++ expect(deps.reserveBalance).toHaveBeenCalledWith({+ user: 'user-1',+ amount: 200,+ reservationId: expect.any(String),+ expiresAt: expect.any(Date),+ initialBalance: undefined,+ });+ const { reservationId } = reserveRequest(deps);+ expect(deps.releaseBalanceReservation).not.toHaveBeenCalled();++ await Promise.all([reservation.release(), reservation.release()]);+ await reservation.release();++ expect(deps.releaseBalanceReservation).toHaveBeenCalledTimes(1);+ expect(deps.releaseBalanceReservation).toHaveBeenCalledWith({+ user: 'user-1',+ reservationId,+ amount: 200,+ });});- it('should throw when user has insufficient balance', async () => {+ it('logs instead of throwing when a release fails', async () => {const deps = createMockDeps({- findBalanceByUser: jest.fn().mockResolvedValue({ tokenCredits: 10 }),- getMultiplier: jest.fn().mockReturnValue(1),+ releaseBalanceReservation: jest.fn().mockRejectedValue(new Error('DB unavailable')),+ });++ const reservation = await checkBalance({ req, res, txData: baseTxData }, deps);++ await expect(reservation.release()).resolves.toBeUndefined();+ expect(logger.error).toHaveBeenCalledWith(+ '[Balance.check] Failed to release balance reservation',+ expect.objectContaining({ user: 'user-1' }),+ );+ });++ it('throws a TOKEN_BALANCE violation with the unreserved balance when refused', async () => {+ const deps = createMockDeps({+ reserveBalance: jest.fn().mockResolvedValue({ reserved: false, balance: 10 }),});await expect(@@ -58,28 +103,206 @@ describe('checkBalance', () => {);});- describe('lazy balance initialization', () => {- it('should create balance record when no record exists and startBalance is configured', async () => {- const upsertBalanceFields = jest.fn().mockResolvedValue({ tokenCredits: 5000 });+ it('reports no less than zero balance when reservations exceed the credits', async () => {+ const deps = createMockDeps({+ reserveBalance: jest.fn().mockResolvedValue({ reserved: false, balance: -200 }),+ });++ await expect(checkBalance({ req, res, txData: baseTxData }, deps)).rejects.toThrow();++ expect(deps.logViolation).toHaveBeenCalledWith(+ req,+ res,+ ViolationTypes.TOKEN_BALANCE,+ expect.objectContaining({ balance: 0, tokenCost: 100 }),+ 0,+ );+ });++ it('propagates a failure of the balance store instead of reporting a balance violation', async () => {+ const deps = createMockDeps({+ reserveBalance: jest.fn().mockRejectedValue(new Error('DB unavailable')),+ });++ await expect(checkBalance({ req, res, txData: baseTxData }, deps)).rejects.toThrow(+ 'DB unavailable',+ );+ expect(deps.logViolation).not.toHaveBeenCalled();+ });++ describe('reservation expiry', () => {+ const expiryOf = (deps: CheckBalanceDeps) => reserveRequest(deps).expiresAt.getTime();++ it('expires reservations after the configured TTL', async () => {+ const deps = createMockDeps({ balanceConfig: { reservationTtlMs: 20_000 } });+ const before = Date.now();++ await checkBalance({ req, res, txData: baseTxData }, deps);++ expect(expiryOf(deps)).toBeGreaterThanOrEqual(before + 20_000);+ expect(expiryOf(deps)).toBeLessThanOrEqual(Date.now() + 20_000);+ });++ it('falls back to the default TTL, warning once, when the configured TTL is invalid', async () => {+ const balanceConfig = { reservationTtlMs: -1 } as BalanceConfig;+ const before = Date.now();++ for (let i = 0; i < 2; i++) {+ const deps = createMockDeps({ balanceConfig });+ await checkBalance({ req, res, txData: baseTxData }, deps);+ expect(expiryOf(deps)).toBeGreaterThanOrEqual(before + DEFAULT_BALANCE_RESERVATION_TTL_MS);+ }++ expect(logger.warn).toHaveBeenCalledTimes(1);+ });+ });++ describe('reservation renewal', () => {+ beforeEach(() => {+ jest.useFakeTimers();+ });++ afterEach(() => {+ jest.useRealTimers();+ });++ it('renews a held reservation every half TTL until it is released', async () => {+ const deps = createMockDeps({ balanceConfig: { reservationTtlMs: 10_000 } });++ const reservation = await checkBalance({ req, res, txData: baseTxData }, deps);+ const { reservationId } = reserveRequest(deps);++ await jest.advanceTimersByTimeAsync(5_000);+ expect(deps.renewBalanceReservation).toHaveBeenCalledTimes(1);+ expect(deps.renewBalanceReservation).toHaveBeenCalledWith({+ user: 'user-1',+ reservationId,+ expiresAt: new Date(Date.now() + 10_000),+ });+ await jest.advanceTimersByTimeAsync(5_000);+ expect(deps.renewBalanceReservation).toHaveBeenCalledTimes(2);++ await reservation.release();+ await jest.advanceTimersByTimeAsync(50_000);+ expect(deps.renewBalanceReservation).toHaveBeenCalledTimes(2);+ });++ it('retries a failed renewal well before the reservation would expire', async () => {const deps = createMockDeps({- findBalanceByUser: jest.fn().mockResolvedValue(null),- balanceConfig: { startBalance: 5000 },- upsertBalanceFields,+ balanceConfig: { reservationTtlMs: 10_000 },+ renewBalanceReservation: jest+ .fn()+ .mockRejectedValueOnce(new Error('DB unavailable'))+ .mockResolvedValue(undefined),});- const result = await checkBalance({ req, res, txData: baseTxData }, deps);+ const reservation = await checkBalance({ req, res, txData: baseTxData }, deps);- expect(result).toBe(true);- expect(upsertBalanceFields).toHaveBeenCalledWith('user-1', {- user: 'user-1',- tokenCredits: 5000,+ await jest.advanceTimersByTimeAsync(5_000);+ expect(deps.renewBalanceReservation).toHaveBeenCalledTimes(1);+ await jest.advanceTimersByTimeAsync(500);+ expect(deps.renewBalanceReservation).toHaveBeenCalledTimes(2);+ await jest.advanceTimersByTimeAsync(4_999);+ expect(deps.renewBalanceReservation).toHaveBeenCalledTimes(2);++ await reservation.release();+ });++ it('renews from the stored expiry when the admission write is slow', async () => {+ const deps = createMockDeps({+ balanceConfig: { reservationTtlMs: 10_000 },+ reserveBalance: jest.fn(+ () =>+ new Promise((resolve) =>+ setTimeout(() => resolve({ reserved: true, balance: 1000 }), 3_000),+ ),+ ),});++ const admission = checkBalance({ req, res, txData: baseTxData }, deps);+ await jest.advanceTimersByTimeAsync(3_000);+ const reservation = await admission;++ await jest.advanceTimersByTimeAsync(1_999);+ expect(deps.renewBalanceReservation).not.toHaveBeenCalled();+ await jest.advanceTimersByTimeAsync(1);+ expect(deps.renewBalanceReservation).toHaveBeenCalledTimes(1);++ await reservation.release();});- it('should include auto-refill fields when configured', async () => {- const upsertBalanceFields = jest.fn().mockResolvedValue({ tokenCredits: 5000 });+ it('renews from the stored expiry when a renewal write is slow', async () => {+ const deps = createMockDeps({+ balanceConfig: { reservationTtlMs: 10_000 },+ renewBalanceReservation: jest.fn(+ () => new Promise<void>((resolve) => setTimeout(resolve, 3_000)),+ ),+ });++ const reservation = await checkBalance({ req, res, txData: baseTxData }, deps);++ await jest.advanceTimersByTimeAsync(5_000);+ expect(deps.renewBalanceReservation).toHaveBeenCalledTimes(1);+ await jest.advanceTimersByTimeAsync(4_999);+ expect(deps.renewBalanceReservation).toHaveBeenCalledTimes(1);+ await jest.advanceTimersByTimeAsync(1);+ expect(deps.renewBalanceReservation).toHaveBeenCalledTimes(2);++ await reservation.release();+ });++ it('keeps the renewal delay within the timer range for a very long TTL', async () => {+ const deps = createMockDeps({ balanceConfig: { reservationTtlMs: 6_000_000_000 } });++ const reservation = await checkBalance({ req, res, txData: baseTxData }, deps);+ await jest.advanceTimersByTimeAsync(60_000);++ expect(deps.renewBalanceReservation).not.toHaveBeenCalled();+ await reservation.release();+ });++ it('raises a TTL below the minimum so renewal stays bounded', async () => {+ const deps = createMockDeps({ balanceConfig: { reservationTtlMs: 1 } });+ const before = Date.now();++ const reservation = await checkBalance({ req, res, txData: baseTxData }, deps);+ expect(reserveRequest(deps).expiresAt.getTime()).toBe(+ before + MIN_BALANCE_RESERVATION_TTL_MS,+ );++ await jest.advanceTimersByTimeAsync(MIN_BALANCE_RESERVATION_TTL_MS / 2 - 1);+ expect(deps.renewBalanceReservation).not.toHaveBeenCalled();+ await jest.advanceTimersByTimeAsync(1);+ expect(deps.renewBalanceReservation).toHaveBeenCalledTimes(1);++ await reservation.release();+ });++ it('does not renew a zero-cost admission, which holds nothing', async () => {+ const deps = createMockDeps({ balanceConfig: { reservationTtlMs: 10_000 } });++ const reservation = await checkBalance(+ { req, res, txData: { ...baseTxData, amount: 0 } },+ deps,+ );+ await jest.advanceTimersByTimeAsync(50_000);++ expect(deps.renewBalanceReservation).not.toHaveBeenCalled();+ await reservation.release();+ });+ });++ describe('lazy balance initialization', () => {+ it('creates a missing record from startBalance', async () => {+ const deps = createMockDeps({ balanceConfig: { startBalance: 5000 } });++ await checkBalance({ req, res, txData: baseTxData }, deps);++ expect(reserveRequest(deps).initialBalance).toEqual({ user: 'user-1', tokenCredits: 5000 });+ });++ it('includes auto-refill fields when configured', async () => {const deps = createMockDeps({- findBalanceByUser: jest.fn().mockResolvedValue(null),balanceConfig: {startBalance: 5000,autoRefillEnabled: true,@@ -87,180 +310,240 @@ describe('checkBalance', () => {refillIntervalUnit: 'days',refillAmount: 1000,},- upsertBalanceFields,});await checkBalance({ req, res, txData: baseTxData }, deps);- expect(upsertBalanceFields).toHaveBeenCalledWith(- 'user-1',- expect.objectContaining({- user: 'user-1',- tokenCredits: 5000,- autoRefillEnabled: true,- refillIntervalValue: 1,- refillIntervalUnit: 'days',- refillAmount: 1000,- lastRefill: expect.any(Date),- }),- );+ expect(reserveRequest(deps).initialBalance).toEqual({+ user: 'user-1',+ tokenCredits: 5000,+ autoRefillEnabled: true,+ refillIntervalValue: 1,+ refillIntervalUnit: 'days',+ refillAmount: 1000,+ lastRefill: expect.any(Date),+ });});- it('should not include auto-refill fields when config is partial', async () => {- const upsertBalanceFields = jest.fn().mockResolvedValue({ tokenCredits: 5000 });+ it('omits auto-refill fields when the refill config is partial', async () => {const deps = createMockDeps({- findBalanceByUser: jest.fn().mockResolvedValue(null),balanceConfig: { startBalance: 5000, autoRefillEnabled: true },- upsertBalanceFields,});await checkBalance({ req, res, txData: baseTxData }, deps);- expect(upsertBalanceFields).toHaveBeenCalledWith('user-1', {- user: 'user-1',- tokenCredits: 5000,- });+ expect(reserveRequest(deps).initialBalance).toEqual({ user: 'user-1', tokenCredits: 5000 });});- it('should throw a TOKEN_BALANCE violation when lazy-initialized balance is less than token cost', async () => {- const upsertBalanceFields = jest.fn().mockResolvedValue({ tokenCredits: 50 });- const deps = createMockDeps({- findBalanceByUser: jest.fn().mockResolvedValue(null),- getMultiplier: jest.fn().mockReturnValue(1),- balanceConfig: { startBalance: 50 },- upsertBalanceFields,- });+ it('creates a record with a startBalance of 0', async () => {+ const deps = createMockDeps({ balanceConfig: { startBalance: 0 } });- await expect(- checkBalance({ req, res, txData: { ...baseTxData, amount: 100 } }, deps),- ).rejects.toThrow();+ await checkBalance({ req, res, txData: baseTxData }, deps);- expect(upsertBalanceFields).toHaveBeenCalledWith('user-1', {- user: 'user-1',- tokenCredits: 50,- });- expect(deps.logViolation).toHaveBeenCalledWith(- req,- res,- ViolationTypes.TOKEN_BALANCE,- expect.objectContaining({ balance: 50, tokenCost: 100 }),- 0,- );+ expect(reserveRequest(deps).initialBalance).toEqual({ user: 'user-1', tokenCredits: 0 });});- it('should use DB-returned tokenCredits over raw startBalance config constant', async () => {- const upsertBalanceFields = jest.fn().mockResolvedValue({ tokenCredits: 3000 });- const deps = createMockDeps({- findBalanceByUser: jest.fn().mockResolvedValue(null),- getMultiplier: jest.fn().mockReturnValue(1),- balanceConfig: { startBalance: 5000 },- upsertBalanceFields,- });+ it.each([+ ['no balance config', undefined],+ ['no startBalance', {}],+ ])(+ 'throws a TOKEN_BALANCE violation for a missing record with %s',+ async (_case, balanceConfig) => {+ const deps = createMockDeps({+ reserveBalance: jest.fn().mockResolvedValue(null),+ balanceConfig,+ });++ await expect(checkBalance({ req, res, txData: baseTxData }, deps)).rejects.toThrow();+ expect(reserveRequest(deps).initialBalance).toBeUndefined();+ expect(deps.logViolation).toHaveBeenCalledWith(+ req,+ res,+ ViolationTypes.TOKEN_BALANCE,+ expect.objectContaining({ balance: 0 }),+ 0,+ );+ },+ );+ });- await expect(- checkBalance({ req, res, txData: { ...baseTxData, amount: 4000 } }, deps),- ).rejects.toThrow();-- expect(deps.logViolation).toHaveBeenCalledWith(- req,- res,- ViolationTypes.TOKEN_BALANCE,- expect.objectContaining({ balance: 3000, tokenCost: 4000 }),- 0,- );+ describe('balance reservations of a turn', () => {+ const createReservation = () => {+ const reservation: BalanceReservation = { release: jest.fn().mockResolvedValue(undefined) };+ return reservation;+ };++ it('releases an admission that settles after the release was requested', async () => {+ const reservations = createBalanceReservations();+ const reservation = createReservation();+ let admit: (value: BalanceReservation) => void = () => undefined;+ reservations.track(new Promise<BalanceReservation>((resolve) => (admit = resolve)));++ const released = reservations.release();+ expect(reservation.release).not.toHaveBeenCalled();+ admit(reservation);+ await released;++ expect(reservation.release).toHaveBeenCalledTimes(1);});- it('should throw a TOKEN_BALANCE violation when no record and no balanceConfig', async () => {- const deps = createMockDeps({- findBalanceByUser: jest.fn().mockResolvedValue(null),- });+ it('releases nothing for a refused admission and does not reject', async () => {+ const reservations = createBalanceReservations();+ const refused = Promise.reject(new Error('insufficient'));- await expect(checkBalance({ req, res, txData: baseTxData }, deps)).rejects.toThrow();- expect(deps.logViolation).toHaveBeenCalledWith(- req,- res,- ViolationTypes.TOKEN_BALANCE,- expect.objectContaining({ balance: 0 }),- 0,- );+ await expect(reservations.track(refused)).rejects.toThrow('insufficient');+ await expect(reservations.release()).resolves.toBeUndefined();});- it('should throw a TOKEN_BALANCE violation when no record and startBalance is undefined', async () => {- const deps = createMockDeps({- findBalanceByUser: jest.fn().mockResolvedValue(null),- balanceConfig: {},- upsertBalanceFields: jest.fn(),- });+ it('releases each tracked reservation once across repeated releases', async () => {+ const reservations = createBalanceReservations();+ const first = createReservation();+ const second = createReservation();- await expect(checkBalance({ req, res, txData: baseTxData }, deps)).rejects.toThrow();- expect(deps.upsertBalanceFields).not.toHaveBeenCalled();- expect(deps.logViolation).toHaveBeenCalledWith(- req,- res,- ViolationTypes.TOKEN_BALANCE,- expect.objectContaining({ balance: 0 }),- 0,- );+ await reservations.track(Promise.resolve(first));+ await reservations.release();+ await reservations.track(Promise.resolve(second));+ await reservations.release();+ await reservations.release();++ expect(first.release).toHaveBeenCalledTimes(1);+ expect(second.release).toHaveBeenCalledTimes(1);});- it('should throw a TOKEN_BALANCE violation when upsertBalanceFields is not provided', async () => {- const deps = createMockDeps({- findBalanceByUser: jest.fn().mockResolvedValue(null),- balanceConfig: { startBalance: 5000 },- });+ it('keeps reservations held until background work settles, even when it fails', async () => {+ const reservations = createBalanceReservations();+ const reservation = createReservation();+ let finishWork: (error: Error) => void = () => undefined;+ await reservations.track(Promise.resolve(reservation));+ reservations.holdUntil(new Promise((_resolve, reject) => (finishWork = reject)));- await expect(checkBalance({ req, res, txData: baseTxData }, deps)).rejects.toThrow();- expect(deps.logViolation).toHaveBeenCalledWith(- req,- res,- ViolationTypes.TOKEN_BALANCE,- expect.objectContaining({ balance: 0 }),- 0,- );+ const released = reservations.release();+ await new Promise((resolve) => setImmediate(resolve));+ expect(reservation.release).not.toHaveBeenCalled();+ finishWork(new Error('background run failed'));+ await released;++ expect(reservation.release).toHaveBeenCalledTimes(1);});- it('should handle startBalance of 0', async () => {- const upsertBalanceFields = jest.fn().mockResolvedValue({ tokenCredits: 0 });- const deps = createMockDeps({- findBalanceByUser: jest.fn().mockResolvedValue(null),- getMultiplier: jest.fn().mockReturnValue(1),- balanceConfig: { startBalance: 0 },- upsertBalanceFields,- });+ it('releases the turn reservations whether the turn resolves or throws', async () => {+ const kept = createReservation();+ const failed = createReservation();await expect(- checkBalance({ req, res, txData: { ...baseTxData, amount: 100 } }, deps),- ).rejects.toThrow();+ withBalanceReservations(async (reservations) => {+ await reservations.track(Promise.resolve(kept));+ return 'done';+ }),+ ).resolves.toBe('done');+ await expect(+ withBalanceReservations(async (reservations) => {+ await reservations.track(Promise.resolve(failed));+ throw new Error('turn failed');+ }),+ ).rejects.toThrow('turn failed');- expect(upsertBalanceFields).toHaveBeenCalledWith('user-1', {- user: 'user-1',- tokenCredits: 0,- });- expect(deps.logViolation).toHaveBeenCalledWith(- req,- res,- ViolationTypes.TOKEN_BALANCE,- expect.objectContaining({ balance: 0, tokenCost: 100 }),- 0,+ expect(kept.release).toHaveBeenCalledTimes(1);+ expect(failed.release).toHaveBeenCalledTimes(1);+ });+ });++ describe('against a real balance store', () => {+ let mongoServer: MongoMemoryServer;+ let Balance: mongoose.Model<IBalance>;+ let methods: ReturnType<typeof createMethods>;++ beforeAll(async () => {+ mongoServer = await MongoMemoryServer.create();+ await mongoose.connect(mongoServer.getUri());+ createModels(mongoose);+ Balance = mongoose.models.Balance as mongoose.Model<IBalance>;+ methods = createMethods(mongoose);+ });++ afterAll(async () => {+ await mongoose.disconnect();+ await mongoServer.stop();+ });++ beforeEach(async () => {+ await mongoose.connection.dropDatabase();+ });++ const realDeps = (balanceConfig?: BalanceConfig): CheckBalanceDeps => ({+ getMultiplier: () => 1,+ reserveBalance: methods.reserveBalance,+ renewBalanceReservation: methods.renewBalanceReservation,+ releaseBalanceReservation: methods.releaseBalanceReservation,+ logViolation: jest.fn().mockResolvedValue(undefined),+ balanceConfig,+ });++ const admitConcurrently = (user: string, count: number, deps: CheckBalanceDeps, amount = 400) =>+ Promise.allSettled(+ Array.from({ length: count }, () =>+ checkBalance({ req, res, txData: { ...baseTxData, user, amount } }, deps),+ ),);++ const admittedOf = (outcomes: PromiseSettledResult<BalanceReservation>[]) =>+ outcomes.flatMap((outcome) => (outcome.status === 'fulfilled' ? [outcome.value] : []));++ it('admits concurrent requests only against credits no in-flight request holds', async () => {+ const user = new mongoose.Types.ObjectId().toString();+ await Balance.create({ user, tokenCredits: 1000 });+ const deps = realDeps();++ const admitted = admittedOf(await admitConcurrently(user, 20, deps));++ expect(admitted).toHaveLength(2);+ expect(deps.logViolation).toHaveBeenCalledTimes(18);++ await Promise.all(admitted.map((reservation) => reservation.release()));+ expect(admittedOf(await admitConcurrently(user, 3, deps))).toHaveLength(2);});- it('should fall back to balance: 0 when upsertBalanceFields rejects', async () => {- const upsertBalanceFields = jest.fn().mockRejectedValue(new Error('DB unavailable'));- const deps = createMockDeps({- findBalanceByUser: jest.fn().mockResolvedValue(null),- balanceConfig: { startBalance: 5000 },- upsertBalanceFields,+ it('admits every request of a funded concurrent burst', async () => {+ const user = new mongoose.Types.ObjectId().toString();+ await Balance.create({ user, tokenCredits: 100_000 });++ const outcomes = await admitConcurrently(user, 40, realDeps(), 100);++ expect(admittedOf(outcomes)).toHaveLength(40);+ });++ it('refills once for concurrent requests arriving in one refill window', async () => {+ const user = new mongoose.Types.ObjectId().toString();+ await Balance.create({+ user,+ tokenCredits: 5,+ autoRefillEnabled: true,+ refillAmount: 1000,+ refillIntervalValue: 30,+ refillIntervalUnit: 'days',+ lastRefill: new Date('2020-01-01T00:00:00.000Z'),});- await expect(checkBalance({ req, res, txData: baseTxData }, deps)).rejects.toThrow();- expect(deps.logViolation).toHaveBeenCalledWith(- req,- res,- ViolationTypes.TOKEN_BALANCE,- expect.objectContaining({ balance: 0 }),- 0,- );+ const outcomes = await admitConcurrently(user, 10, realDeps());++ expect(admittedOf(outcomes)).toHaveLength(2);+ const stored = await Balance.findOne({ user }).lean();+ expect(stored?.tokenCredits).toBe(1005);+ const refills = await methods.getTransactions({ user, context: 'autoRefill' });+ expect(refills).toHaveLength(1);+ });++ it('lazily initializes a missing record and reserves against it', async () => {+ const user = new mongoose.Types.ObjectId().toString();+ const deps = realDeps({ startBalance: 1000 });++ await checkBalance({ req, res, txData: { ...baseTxData, user, amount: 400 } }, deps);+ const outcomes = await admitConcurrently(user, 4, deps);++ expect(admittedOf(outcomes)).toHaveLength(1);+ const stored = await Balance.findOne({ user }).select('+reservedCredits').lean();+ expect(stored?.tokenCredits).toBe(1000);+ expect(stored?.reservedCredits).toBe(800);});});});diff --git a/packages/api/src/middleware/checkBalance.ts b/packages/api/src/middleware/checkBalance.tsindex e5764962c2..070abca38f 100644--- a/packages/api/src/middleware/checkBalance.ts+++ b/packages/api/src/middleware/checkBalance.ts@@ -1,19 +1,21 @@+import { randomUUID } from 'crypto';import { logger } from '@librechat/data-schemas';-import { getRefillEligibilityDate, ViolationTypes } from 'librechat-data-provider';-import type { BalanceConfig, IBalanceUpdate } from '@librechat/data-schemas';-import type { RefillIntervalUnit } from 'librechat-data-provider';+import {+ ViolationTypes,+ MIN_BALANCE_RESERVATION_TTL_MS,+ DEFAULT_BALANCE_RESERVATION_TTL_MS,+} from 'librechat-data-provider';+import type {+ BalanceReservationRequest,+ BalanceReservationRenewal,+ BalanceReservationRelease,+ BalanceReservationResult,+ IBalanceUpdate,+ BalanceConfig,+} from '@librechat/data-schemas';import type { Response } from 'express';import type { ServerRequest } from '~/types/http';-interface BalanceRecord {- tokenCredits: number;- autoRefillEnabled?: boolean;- refillAmount?: number;- lastRefill?: Date;- refillIntervalValue?: number;- refillIntervalUnit?: RefillIntervalUnit;-}-interface TxData {user: string;model?: string;@@ -26,11 +28,10 @@ interface TxData {}export interface CheckBalanceDeps {- findBalanceByUser: (user: string) => Promise<BalanceRecord | null>;getMultiplier: (params: Record<string, unknown>) => number;- createAutoRefillTransaction: (- data: Record<string, unknown>,- ) => Promise<{ balance: number } | undefined>;+ reserveBalance: (request: BalanceReservationRequest) => Promise<BalanceReservationResult | null>;+ renewBalanceReservation: (params: BalanceReservationRenewal) => Promise<void>;+ releaseBalanceReservation: (params: BalanceReservationRelease) => Promise<void>;logViolation: (req: unknown,res: unknown,@@ -38,17 +39,179 @@ export interface CheckBalanceDeps {errorMessage: Record<string, unknown>,score: number,) => Promise<void>;- /** Balance config for lazy initialization when no record exists */+ /** Balance config for lazy initialization when no record exists, and the reservation TTL */balanceConfig?: BalanceConfig;- /** Upsert function for lazy initialization when no record exists */- upsertBalanceFields?: (userId: string, fields: IBalanceUpdate) => Promise<BalanceRecord | null>;}-/** Checks a user's balance record and handles auto-refill if needed. */-async function checkBalanceRecord(- txData: TxData,+/**+ * Credits held for an admitted request until its usage has been recorded. The reservation is+ * renewed every half TTL until released, so only a reservation whose process stopped expires.+ */+export interface BalanceReservation {+ /** Idempotent; stops renewal. A failed release is logged and left to expire. */+ release: () => Promise<void>;+}++/** The balance reservations admitted during one turn. */+export interface BalanceReservations {+ /** Tracks an admission that may still be pending; returns the same promise. */+ track: <T extends BalanceReservation | undefined>(admission: Promise<T>) => Promise<T>;+ /** Keeps the turn's reservations held until `work` settles, such as a run continuing in the background. */+ holdUntil: (work: Promise<unknown>) => void;+ /**+ * Releases every tracked reservation, first waiting for admissions still pending so a+ * reservation that settles after its turn failed is released too, and for work passed to+ * `holdUntil`. Failed admissions hold nothing.+ */+ release: () => Promise<void>;+}++const noReservation = (): undefined => undefined;++export function createBalanceReservations(): BalanceReservations {+ let admissions: Promise<BalanceReservation | undefined>[] = [];+ return {+ track: (admission) => {+ admissions.push(admission.catch(() => undefined));+ return admission;+ },+ holdUntil: (work) => {+ admissions.push(work.then(noReservation, noReservation));+ },+ release: async () => {+ const pending = admissions;+ admissions = [];+ const reservations = await Promise.all(pending);+ await Promise.all(reservations.map((reservation) => reservation?.release()));+ },+ };+}++/** Runs one turn and releases whatever balance reservations it admitted once it settles. */+export async function withBalanceReservations<T>(+ run: (reservations: BalanceReservations) => Promise<T>,+): Promise<T> {+ const reservations = createBalanceReservations();+ try {+ return await run(reservations);+ } finally {+ await reservations.release();+ }+}++/** Node clamps a timer delay above a signed 32-bit millisecond count to 1 ms. */+const MAX_TIMER_DELAY_MS = 2_147_483_647;++/**+ * Keeps an admitted reservation alive until it is released: renews it at half of its stored+ * expiry, measured from the expiry rather than from when the write that stored it settled, so a+ * slow admission or renewal write cannot let the hold expire before the next renewal. A failed+ * renewal retries well within the remaining half so a transient failure does not let it expire.+ */+function holdReservation(+ {+ user,+ reservationId,+ amount,+ ttlMs,+ expiresAt,+ }: { user: string; reservationId: string; amount: number; ttlMs: number; expiresAt: Date },+ deps: Pick<CheckBalanceDeps, 'renewBalanceReservation' | 'releaseBalanceReservation'>,+): BalanceReservation {+ const retryEveryMs = Math.min(ttlMs / 20, 5_000);+ let timer: NodeJS.Timeout | undefined;+ let released: Promise<void> | undefined;++ const schedule = (delayMs: number) => {+ timer = setTimeout(renew, Math.min(delayMs, MAX_TIMER_DELAY_MS));+ timer.unref();+ };++ const scheduleFrom = (storedExpiry: Date) =>+ schedule(Math.max(0, storedExpiry.getTime() - ttlMs / 2 - Date.now()));++ function renew() {+ const renewedExpiry = new Date(Date.now() + ttlMs);+ deps.renewBalanceReservation({ user, reservationId, expiresAt: renewedExpiry }).then(+ () => (released ? undefined : scheduleFrom(renewedExpiry)),+ (error) => {+ logger.error('[Balance.check] Failed to renew balance reservation', { user, error });+ if (!released) {+ schedule(retryEveryMs);+ }+ },+ );+ }++ if (amount > 0) {+ scheduleFrom(expiresAt);+ }++ return {+ release: () => {+ clearTimeout(timer);+ released ??= deps+ .releaseBalanceReservation({ user, reservationId, amount })+ .catch((error) => {+ logger.error('[Balance.check] Failed to release balance reservation', { user, error });+ });+ return released;+ },+ };+}++let warnedInvalidReservationTtl = false;++function getReservationTtlMs(config?: BalanceConfig): number {+ const ttl = config?.reservationTtlMs;+ if (ttl == null) {+ return DEFAULT_BALANCE_RESERVATION_TTL_MS;+ }+ const valid = Number.isFinite(ttl) && ttl > 0;+ const effective = valid+ ? Math.max(ttl, MIN_BALANCE_RESERVATION_TTL_MS)+ : DEFAULT_BALANCE_RESERVATION_TTL_MS;+ if (effective !== ttl && !warnedInvalidReservationTtl) {+ warnedInvalidReservationTtl = true;+ logger.warn('[Balance.check] Adjusting balance.reservationTtlMs', {+ reservationTtlMs: ttl,+ effectiveMs: effective,+ minimumMs: MIN_BALANCE_RESERVATION_TTL_MS,+ });+ }+ return effective;+}++function buildInitialBalance(user: string, config?: BalanceConfig): IBalanceUpdate | undefined {+ if (config?.startBalance == null) {+ return undefined;+ }+ const fields: IBalanceUpdate = { user, tokenCredits: config.startBalance };+ if (+ config.autoRefillEnabled &&+ config.refillIntervalValue != null &&+ config.refillIntervalUnit != null &&+ config.refillAmount != null+ ) {+ fields.autoRefillEnabled = config.autoRefillEnabled;+ fields.refillIntervalValue = config.refillIntervalValue;+ fields.refillIntervalUnit = config.refillIntervalUnit;+ fields.refillAmount = config.refillAmount;+ fields.lastRefill = new Date();+ }+ return fields;+}++/**+ * Admits a request against the user's balance and holds its token cost until the returned+ * reservation is released, so concurrent requests are admitted only against credits that no+ * other in-flight request holds. A missing balance record is created from `startBalance`.+ * Throws with the balance info if the credits are insufficient.+ */+export async function checkBalance(+ { req, res, txData }: { req: ServerRequest; res: Response; txData: TxData },deps: CheckBalanceDeps,-): Promise<{ canSpend: boolean; balance: number; tokenCost: number }> {+): Promise<BalanceReservation> {const { user, model, endpoint, valueKey, tokenType, amount, endpointTokenConfig } = txData;const multiplier = deps.getMultiplier({valueKey,@@ -58,111 +221,42 @@ async function checkBalanceRecord(endpointTokenConfig,});const tokenCost = amount * multiplier;+ const reservationId = randomUUID();+ const ttlMs = getReservationTtlMs(deps.balanceConfig);- const record = await deps.findBalanceByUser(user);- if (!record) {- if (deps.balanceConfig?.startBalance != null && deps.upsertBalanceFields) {- logger.debug('[Balance.check] Lazy-initializing balance record for user', {- user,- startBalance: deps.balanceConfig.startBalance,- });- try {- const fields: IBalanceUpdate = {- user,- tokenCredits: deps.balanceConfig.startBalance,- };- const config = deps.balanceConfig;- if (- config.autoRefillEnabled &&- config.refillIntervalValue != null &&- config.refillIntervalUnit != null &&- config.refillAmount != null- ) {- fields.autoRefillEnabled = config.autoRefillEnabled;- fields.refillIntervalValue = config.refillIntervalValue;- fields.refillIntervalUnit = config.refillIntervalUnit;- fields.refillAmount = config.refillAmount;- fields.lastRefill = new Date();- }- const created = await deps.upsertBalanceFields(user, fields);- const balance = created?.tokenCredits ?? deps.balanceConfig.startBalance;- return { canSpend: balance >= tokenCost, balance, tokenCost };- } catch (error) {- logger.error('[Balance.check] Failed to lazy-initialize balance record', { user, error });- return { canSpend: false, balance: 0, tokenCost };- }- }- logger.debug('[Balance.check] No balance record found for user', { user });- return { canSpend: false, balance: 0, tokenCost };- }- let balance = record.tokenCredits;-- logger.debug('[Balance.check] Initial state', {+ logger.debug('[Balance.check] Reserving token cost', {user,model,endpoint,valueKey,tokenType,amount,- balance,multiplier,+ tokenCost,endpointTokenConfig: !!endpointTokenConfig,});- if (- balance - tokenCost <= 0 &&- record.autoRefillEnabled &&- record.refillAmount &&- record.refillAmount > 0- ) {- const lastRefillDate = new Date(record.lastRefill ?? 0);- const now = new Date();- if (- isNaN(lastRefillDate.getTime()) ||- now >=- getRefillEligibilityDate(- lastRefillDate,- record.refillIntervalValue ?? 0,- record.refillIntervalUnit ?? 'days',- )- ) {- try {- const result = await deps.createAutoRefillTransaction({- user,- tokenType: 'credits',- context: 'autoRefill',- rawAmount: record.refillAmount,- });- if (result) {- balance = result.balance;- }- } catch (error) {- logger.error('[Balance.check] Failed to record transaction for auto-refill', error);- }- }- }+ const expiresAt = new Date(Date.now() + ttlMs);+ const result = await deps.reserveBalance({+ user,+ reservationId,+ amount: tokenCost,+ expiresAt,+ initialBalance: buildInitialBalance(user, deps.balanceConfig),+ });- logger.debug('[Balance.check] Token cost', { tokenCost });- return { canSpend: balance >= tokenCost, balance, tokenCost };-}+ if (result?.reserved) {+ return holdReservation({ user, reservationId, amount: tokenCost, ttlMs, expiresAt }, deps);+ }-/**- * Checks balance for a user and logs a violation if they cannot spend.- * Throws an error with the balance info if insufficient funds.- */-export async function checkBalance(- { req, res, txData }: { req: ServerRequest; res: Response; txData: TxData },- deps: CheckBalanceDeps,-): Promise<boolean> {- const { canSpend, balance, tokenCost } = await checkBalanceRecord(txData, deps);- if (canSpend) {- return true;+ if (!result) {+ logger.debug('[Balance.check] No balance record found for user', { user });}const type = ViolationTypes.TOKEN_BALANCE;const errorMessage: Record<string, unknown> = {type,- balance,+ balance: Math.max(0, result?.balance ?? 0),tokenCost,promptTokens: txData.amount,};diff --git a/packages/api/src/schedules/service.spec.ts b/packages/api/src/schedules/service.spec.tsindex 8aa1318d8e..0c616ac7d3 100644--- a/packages/api/src/schedules/service.spec.ts+++ b/packages/api/src/schedules/service.spec.ts@@ -257,6 +257,21 @@ describe('balance initialization', () => {expect(outOfBalance).toBe(false);});+ it.each([+ ['all of its credits are held by in-flight requests', 100, true],+ ['part of its credits are free', 50, false],+ ])('pre-skips a record only when %s', async (_case, reservedCredits, outOfBalance) => {+ const { service } = serviceWithBalance({+ tokenCredits: 100,+ reservedCredits,+ autoRefillEnabled: false,+ });++ await expect(service.engineDeps.isOutOfBalance({ id: 'user-1' } as never)).resolves.toBe(+ outOfBalance,+ );+ });+/*** A stale record whose credit is already set but whose refill config drifted still syncs* that config, and the sync must not widen into a credit write.diff --git a/packages/api/src/schedules/service.ts b/packages/api/src/schedules/service.tsindex 24ce736b1c..4c3fb6d7c3 100644--- a/packages/api/src/schedules/service.ts+++ b/packages/api/src/schedules/service.ts@@ -144,6 +144,7 @@ export interface SchedulesServiceDeps {findUserById: (userId: string | Types.ObjectId,) => Promise<{ _id: Types.ObjectId; tenantId?: string; role?: string } | null>;+ /** Reads the balance record together with the credits unexpired in-flight reservations hold. */findBalance: (userId: string) => Promise<IBalance | null>;/*** Upserts a balance record. `setOnInsert` carries fields that must ONLY apply to a@@ -461,6 +462,10 @@ export function createSchedulesService(return false;}let record = await deps.findBalance(user.id);+ // Credits in-flight requests hold are unavailable to this fire as well: the chat+ // balance check admits against the unreserved amount. Taken from this read because+ // the initialization/sync writes below return the record without the total.+ const reservedCredits = record?.reservedCredits ?? 0;// Initialize/sync the record exactly as the chat's balance middleware would,// so a new user's startBalance is applied before we read it (avoids skipping// a schedule that an interactive chat would have allowed).@@ -501,7 +506,7 @@ export function createSchedulesService(}}}- const credits = record?.tokenCredits ?? 0;+ const credits = (record?.tokenCredits ?? 0) - reservedCredits;if (credits > 0) {return false;}diff --git a/packages/data-provider/src/balance.ts b/packages/data-provider/src/balance.tsindex 5a664fbe1b..ed199a3b11 100644--- a/packages/data-provider/src/balance.ts+++ b/packages/data-provider/src/balance.ts@@ -9,6 +9,11 @@ export const REFILL_INTERVAL_UNITS = [export type RefillIntervalUnit = (typeof REFILL_INTERVAL_UNITS)[number];+/** How long an unreleased in-flight balance reservation keeps counting against the balance. */+export const DEFAULT_BALANCE_RESERVATION_TTL_MS = 30 * 60 * 1000;+/** Shortest reservation TTL; a live reservation is renewed every half TTL. */+export const MIN_BALANCE_RESERVATION_TTL_MS = 10 * 1000;+function ensureExhaustive(value: never): void {void value;}diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.tsindex 61ac028835..3a48abcebc 100644--- a/packages/data-provider/src/config.ts+++ b/packages/data-provider/src/config.ts@@ -17,13 +17,17 @@ import {eReasoningParameterFormatSchema,eReasoningResponseKeySchema,} from './schemas';+import {+ REFILL_INTERVAL_UNITS,+ MIN_BALANCE_RESERVATION_TTL_MS,+ DEFAULT_BALANCE_RESERVATION_TTL_MS,+} from './balance';import { ComponentTypes, SettingTypes, OptionTypes } from './generate';import { CODE_ENVIRONMENT_DECISION_VERSION } from './code/workspace';import { MAX_SUBAGENTS, MAX_SUBAGENTS_CEILING } from './limits';import { STATEFUL_CODE_ENVIRONMENTS } from './stateful-code';import { specsConfigSchema, TSpecsConfig } from './models';import { isActionTool } from './types/assistants';-import { REFILL_INTERVAL_UNITS } from './balance';import { fileConfigSchema } from './file-config';import { apiBaseUrl } from './api-endpoints';import { FileSources } from './types/files';@@ -2494,6 +2498,12 @@ export const balanceSchema = z.object({refillIntervalValue: z.number().optional().default(30),refillIntervalUnit: z.enum(REFILL_INTERVAL_UNITS).optional().default('days'),refillAmount: z.number().optional().default(10000),+ reservationTtlMs: z+ .number()+ .int()+ .min(MIN_BALANCE_RESERVATION_TTL_MS)+ .optional()+ .default(DEFAULT_BALANCE_RESERVATION_TTL_MS),});export const transactionsSchema = z.object({diff --git a/packages/data-schemas/src/methods/spendTokens.spec.ts b/packages/data-schemas/src/methods/spendTokens.spec.tsindex 7c743a2b95..975b907547 100644--- a/packages/data-schemas/src/methods/spendTokens.spec.ts+++ b/packages/data-schemas/src/methods/spendTokens.spec.ts@@ -19,9 +19,7 @@ let mongoServer: InstanceType<typeof MongoMemoryServer>;let spendTokens: ReturnType<typeof createSpendTokensMethods>['spendTokens'];let spendStructuredTokens: ReturnType<typeof createSpendTokensMethods>['spendStructuredTokens'];let createTransaction: ReturnType<typeof createTransactionMethods>['createTransaction'];-let createAutoRefillTransaction: ReturnType<- typeof createTransactionMethods->['createAutoRefillTransaction'];+let updateBalance: ReturnType<typeof createTransactionMethods>['updateBalance'];let getCacheMultiplier: ReturnType<typeof createTxMethods>['getCacheMultiplier'];describe('spendTokens', () => {@@ -47,7 +45,7 @@ describe('spendTokens', () => {getCacheMultiplier: txMethods.getCacheMultiplier,});createTransaction = transactionMethods.createTransaction;- createAutoRefillTransaction = transactionMethods.createAutoRefillTransaction;+ updateBalance = transactionMethods.updateBalance;const spendMethods = createSpendTokensMethods(mongoose, {createTransaction: transactionMethods.createTransaction,@@ -590,73 +588,24 @@ describe('spendTokens', () => {expect(Math.abs(totalTokenValue)).toBeCloseTo(actualSpend, -3); // Allow for larger differences});- // Add this new test caseit('should handle multiple concurrent balance increases correctly', async () => {- // Start with zero balanceconst initialBalance = 0;await Balance.create({user: userId,tokenCredits: initialBalance,});- const numberOfRefills = 25;- const refillAmount = 1000;+ const numberOfIncrements = 25;+ const incrementValue = 1000;- const promises: Promise<unknown>[] = [];- for (let i = 0; i < numberOfRefills; i++) {- promises.push(- createAutoRefillTransaction({- user: userId,- tokenType: 'credits',- context: 'concurrent-refill-test',- rawAmount: refillAmount,- balance: { enabled: true },- }),- );- }-- // Wait for all refill transactions to complete- const results = await Promise.all(promises);+ await Promise.all(+ Array.from({ length: numberOfIncrements }, () =>+ updateBalance({ user: userId.toString(), incrementValue }),+ ),+ );- // Verify final balanceconst finalBalance = await Balance.findOne({ user: userId });- expect(finalBalance).toBeDefined();-- // The final balance should be the initial balance plus the sum of all refills- const expectedFinalBalance = initialBalance + numberOfRefills * refillAmount;-- // Use toBeCloseTo for safety, though toBe should work for integer math- expect(finalBalance!.tokenCredits).toBeCloseTo(expectedFinalBalance, 0);-- // Verify all transactions were created- const transactions = await Transaction.find({- user: userId,- context: 'concurrent-refill-test',- });-- // We should have one transaction for each refill attempt- expect(transactions.length).toBe(numberOfRefills);-- // Optional: Verify the sum of increments from the results matches the balance change- const totalIncrementReported = results.reduce((sum: number, result) => {- // Assuming createAutoRefillTransaction returns an object with the increment amount- // Adjust this based on the actual return structure.- // Let's assume it returns { balance: newBalance, transaction: { rawAmount: ... } }- // Or perhaps we check the transaction.rawAmount directly- const r = result as Record<string, Record<string, unknown>>;- return sum + ((r?.transaction?.rawAmount as number) || 0);- }, 0);- expect(totalIncrementReported).toBe(expectedFinalBalance - initialBalance);-- // Optional: Check the sum of tokenValue from saved transactions- let totalTokenValueFromDb = 0;- transactions.forEach((tx) => {- // For refills, rawAmount is positive, and tokenValue might be calculated based on it- // Let's assume tokenValue directly reflects the increment for simplicity here- // If calculation is involved, adjust accordingly- totalTokenValueFromDb += tx.rawAmount!; // Or tx.tokenValue if that holds the increment- });- expect(totalTokenValueFromDb).toBeCloseTo(expectedFinalBalance - initialBalance, 0);+ expect(finalBalance!.tokenCredits).toBe(initialBalance + numberOfIncrements * incrementValue);});it('should create structured transactions for both prompt and completion tokens', async () => {diff --git a/packages/data-schemas/src/methods/transaction.spec.ts b/packages/data-schemas/src/methods/transaction.spec.tsindex badfdaf014..181ae960ca 100644--- a/packages/data-schemas/src/methods/transaction.spec.ts+++ b/packages/data-schemas/src/methods/transaction.spec.ts@@ -25,6 +25,14 @@ let createTransaction: ReturnType<typeof createTransactionMethods>['createTransalet createStructuredTransaction: ReturnType<typeof createTransactionMethods>['createStructuredTransaction'];+let reserveBalance: ReturnType<typeof createTransactionMethods>['reserveBalance'];+let renewBalanceReservation: ReturnType<typeof createTransactionMethods>['renewBalanceReservation'];+let releaseBalanceReservation: ReturnType<+ typeof createTransactionMethods+>['releaseBalanceReservation'];+let findBalanceByUser: ReturnType<typeof createTransactionMethods>['findBalanceByUser'];+let upsertBalanceFields: ReturnType<typeof createTransactionMethods>['upsertBalanceFields'];+let updateBalance: ReturnType<typeof createTransactionMethods>['updateBalance'];let getMultiplier: ReturnType<typeof createTxMethods>['getMultiplier'];let getCacheMultiplier: ReturnType<typeof createTxMethods>['getCacheMultiplier'];@@ -50,6 +58,12 @@ beforeAll(async () => {});createTransaction = transactionMethods.createTransaction;createStructuredTransaction = transactionMethods.createStructuredTransaction;+ reserveBalance = transactionMethods.reserveBalance;+ renewBalanceReservation = transactionMethods.renewBalanceReservation;+ releaseBalanceReservation = transactionMethods.releaseBalanceReservation;+ findBalanceByUser = transactionMethods.findBalanceByUser;+ upsertBalanceFields = transactionMethods.upsertBalanceFields;+ updateBalance = transactionMethods.updateBalance;const spendMethods = createSpendTokensMethods(mongoose, {createTransaction: transactionMethods.createTransaction,@@ -1081,3 +1095,633 @@ describe('Premium Token Pricing Integration Tests', () => {expect(updatedBalance?.tokenCredits).toBeCloseTo(initialBalance - expectedCost, 0);});});++describe('Balance Reservations', () => {+ const inFuture = () => new Date(Date.now() + 60_000);+ const newId = () => new mongoose.Types.ObjectId().toString();+ const reserve = (user: string, amount: number, reservationId = newId()) =>+ reserveBalance({ user, amount, reservationId, expiresAt: inFuture() });+ const readState = (user: mongoose.Types.ObjectId) =>+ Balance.findOne({ user }).select('+reservations +reservedCredits +pendingRefill').lean();++ /** Runs `interleave` once, immediately before the next `Balance` write executes. */+ const interleaveBeforeNextWrite = (interleave: () => Promise<unknown>) => {+ let fired = false;+ const realUpdateOne = Balance.updateOne.bind(Balance);+ const realBulkWrite = Balance.bulkWrite.bind(Balance);+ const before = (write: () => unknown): unknown => {+ if (fired) {+ return write();+ }+ fired = true;+ return interleave().then(() => write());+ };+ jest+ .spyOn(Balance, 'updateOne')+ .mockImplementation(((...args: Parameters<typeof Balance.updateOne>) =>+ before(() => realUpdateOne(...args))) as unknown as typeof Balance.updateOne);+ jest+ .spyOn(Balance, 'bulkWrite')+ .mockImplementation(((...args: Parameters<typeof Balance.bulkWrite>) =>+ before(() => realBulkWrite(...args))) as unknown as typeof Balance.bulkWrite);+ };++ afterEach(() => {+ jest.restoreAllMocks();+ });++ test('returns null when the user has no balance record and no initial balance', async () => {+ await expect(reserve(newId(), 100)).resolves.toBeNull();+ });++ test('creates a missing record from the initial balance and admits against it', async () => {+ const user = new mongoose.Types.ObjectId();++ await expect(+ reserveBalance({+ user: user.toString(),+ amount: 400,+ reservationId: newId(),+ expiresAt: inFuture(),+ initialBalance: { user: user.toString(), tokenCredits: 1000 },+ }),+ ).resolves.toEqual({ reserved: true, balance: 1000 });++ const stored = await readState(user);+ expect(stored?.tokenCredits).toBe(1000);+ expect(stored?.reservedCredits).toBe(400);+ });++ test('never overwrites a record another writer created while initializing', async () => {+ const user = new mongoose.Types.ObjectId();+ const realFindOneAndUpdate = Balance.findOneAndUpdate.bind(Balance);+ jest.spyOn(Balance, 'findOneAndUpdate').mockImplementationOnce(((+ ...args: Parameters<typeof Balance.findOneAndUpdate>+ ) => ({+ lean: () =>+ upsertBalanceFields(user.toString(), { tokenCredits: 50 }).then(() =>+ realFindOneAndUpdate(...args).lean(),+ ),+ })) as unknown as typeof Balance.findOneAndUpdate);++ await expect(+ reserveBalance({+ user: user.toString(),+ amount: 100,+ reservationId: newId(),+ expiresAt: inFuture(),+ initialBalance: { user: user.toString(), tokenCredits: 1000 },+ }),+ ).resolves.toEqual({ reserved: false, balance: 50 });+ expect((await readState(user))?.tokenCredits).toBe(50);+ expect(await Balance.countDocuments({ user })).toBe(1);+ });++ test('creates a missing record under the user id, so racing creators share one record', async () => {+ const reserved = new mongoose.Types.ObjectId();+ const synced = new mongoose.Types.ObjectId();+ const spent = new mongoose.Types.ObjectId();+ const racing = new mongoose.Types.ObjectId();+ const initialReservation = (user: mongoose.Types.ObjectId) =>+ reserveBalance({+ user: user.toString(),+ amount: 400,+ reservationId: newId(),+ expiresAt: inFuture(),+ initialBalance: { user: user.toString(), tokenCredits: 1000 },+ });++ await initialReservation(reserved);+ await upsertBalanceFields(synced.toString(), { tokenCredits: 50 }, { refillAmount: 5 });+ await updateBalance({ user: spent.toString(), incrementValue: 25 });+ const racingResults = await Promise.all(+ Array.from({ length: 10 }, () => initialReservation(racing)),+ );++ for (const user of [reserved, synced, spent, racing]) {+ const records = await Balance.find({ user }).lean();+ expect(records.map((record) => record._id.toString())).toEqual([user.toString()]);+ }+ expect(await Balance.findById(synced).lean()).toMatchObject({+ tokenCredits: 50,+ refillAmount: 5,+ });+ expect((await Balance.findById(spent).lean())?.tokenCredits).toBe(25);+ expect(racingResults.filter((result) => result?.reserved)).toHaveLength(2);+ });++ test('resolves a user with duplicate records to their oldest record on every path', async () => {+ const user = new mongoose.Types.ObjectId();+ const nowSeconds = Math.floor(Date.now() / 1000);+ const newer = mongoose.Types.ObjectId.createFromTime(nowSeconds);+ const older = mongoose.Types.ObjectId.createFromTime(nowSeconds - 3600);+ await Balance.create({ _id: newer, user, tokenCredits: 5000 });+ await Balance.create({ _id: older, user, tokenCredits: 1000 });++ expect((await findBalanceByUser(user.toString()))?._id?.toString()).toBe(older.toString());+ await expect(reserve(user.toString(), 2000)).resolves.toEqual({+ reserved: false,+ balance: 1000,+ });+ await reserve(user.toString(), 400);+ await updateBalance({ user: user.toString(), incrementValue: -100 });+ await upsertBalanceFields(user.toString(), { refillAmount: 7 });++ expect(await Balance.findById(older).select('+reservedCredits').lean()).toMatchObject({+ tokenCredits: 900,+ reservedCredits: 400,+ refillAmount: 7,+ });+ const untouched = await Balance.findById(newer).select('+reservedCredits').lean();+ expect(untouched?.tokenCredits).toBe(5000);+ expect(untouched?.reservedCredits).toBeUndefined();+ });++ test('admits concurrent requests only up to the credits no other request holds', async () => {+ const user = new mongoose.Types.ObjectId();+ await Balance.create({ user, tokenCredits: 1000 });++ const results = await Promise.all(+ Array.from({ length: 10 }, () => reserve(user.toString(), 300)),+ );++ expect(results.filter((result) => result?.reserved)).toHaveLength(3);+ const stored = await readState(user);+ expect(stored?.tokenCredits).toBe(1000);+ expect(stored?.reservations).toHaveLength(3);+ expect(stored?.reservedCredits).toBe(900);+ });++ test('commits each admission of a funded burst on its first write', async () => {+ const user = new mongoose.Types.ObjectId();+ await Balance.create({ user, tokenCredits: 100_000 });+ const realUpdateOne = Balance.updateOne.bind(Balance);+ const writes = jest+ .spyOn(Balance, 'updateOne')+ .mockImplementation(((...args: Parameters<typeof Balance.updateOne>) =>+ new Promise((resolve) => setTimeout(resolve, 25)).then(() =>+ realUpdateOne(...args),+ )) as unknown as typeof Balance.updateOne);++ const results = await Promise.all(+ Array.from({ length: 50 }, () => reserve(user.toString(), 100)),+ );++ expect(results.every((result) => result?.reserved)).toBe(true);+ expect(writes).toHaveBeenCalledTimes(50);+ const stored = await readState(user);+ expect(stored?.reservations).toHaveLength(50);+ expect(stored?.reservedCredits).toBe(5000);+ });++ test('holds whole credits so the running total stays exact', async () => {+ const user = new mongoose.Types.ObjectId();+ await Balance.create({ user, tokenCredits: 1000 });+ const reservationId = newId();++ await expect(reserve(user.toString(), 0.3, reservationId)).resolves.toEqual({+ reserved: true,+ balance: 1000,+ });+ expect((await readState(user))?.reservedCredits).toBe(1);++ await releaseBalanceReservation({ user: user.toString(), reservationId, amount: 0.3 });+ const stored = await readState(user);+ expect(stored?.reservedCredits).toBe(0);+ expect(stored?.reservations).toEqual([]);+ });++ test('releasing a reservation returns its credits to later admissions, once', async () => {+ const user = new mongoose.Types.ObjectId();+ await Balance.create({ user, tokenCredits: 1000 });+ const firstId = newId();++ await expect(reserve(user.toString(), 600, firstId)).resolves.toEqual({+ reserved: true,+ balance: 1000,+ });+ await expect(reserve(user.toString(), 600)).resolves.toEqual({+ reserved: false,+ balance: 400,+ });++ const release = { user: user.toString(), reservationId: firstId, amount: 600 };+ await Promise.all([releaseBalanceReservation(release), releaseBalanceReservation(release)]);+ expect((await readState(user))?.reservedCredits).toBe(0);++ await expect(reserve(user.toString(), 600)).resolves.toEqual({+ reserved: true,+ balance: 1000,+ });+ });++ test('prunes expired reservations together with their credits', async () => {+ const user = new mongoose.Types.ObjectId();+ await Balance.create({+ user,+ tokenCredits: 1000,+ reservedCredits: 900,+ reservations: [{ id: 'stale', amount: 900, expiresAt: new Date(Date.now() - 1000) }],+ });++ const reservationId = newId();+ await expect(reserve(user.toString(), 600, reservationId)).resolves.toEqual({+ reserved: true,+ balance: 1000,+ });++ const stored = await readState(user);+ expect(stored?.reservations?.map((reservation) => reservation.id)).toEqual([reservationId]);+ expect(stored?.reservedCredits).toBe(600);+ });++ test('prunes a backlog of expired reservations in one bulk write', async () => {+ const user = new mongoose.Types.ObjectId();+ const expiredAt = new Date(Date.now() - 1000);+ await Balance.create({+ user,+ tokenCredits: 1000,+ reservedCredits: 201,+ reservations: [+ { id: 'live', amount: 1, expiresAt: inFuture() },+ ...Array.from({ length: 200 }, (_, i) => ({+ id: `stale-${i}`,+ amount: 1,+ expiresAt: expiredAt,+ })),+ ],+ });+ const writes = jest.spyOn(Balance, 'updateOne');+ const bulkWrites = jest.spyOn(Balance, 'bulkWrite');++ const reservationId = newId();+ await expect(reserve(user.toString(), 100, reservationId)).resolves.toEqual({+ reserved: true,+ balance: 999,+ });++ expect(bulkWrites).toHaveBeenCalledTimes(1);+ expect(bulkWrites.mock.calls[0][0]).toHaveLength(200);+ expect(writes).toHaveBeenCalledTimes(1);+ const stored = await readState(user);+ expect(stored?.reservations?.map((reservation) => reservation.id)).toEqual([+ 'live',+ reservationId,+ ]);+ expect(stored?.reservedCredits).toBe(101);+ });++ test('prunes the rest in the same write when one expired reservation is renewed mid-attempt', async () => {+ const user = new mongoose.Types.ObjectId();+ const expiredAt = new Date(Date.now() - 1000);+ await Balance.create({+ user,+ tokenCredits: 1000,+ reservedCredits: 900,+ reservations: [+ { id: 'renewed', amount: 400, expiresAt: expiredAt },+ { id: 'stale', amount: 500, expiresAt: expiredAt },+ ],+ });++ interleaveBeforeNextWrite(() =>+ renewBalanceReservation({+ user: user.toString(),+ reservationId: 'renewed',+ expiresAt: inFuture(),+ }),+ );++ const reads = jest.spyOn(Balance, 'findOne');+ const reservationId = newId();+ await expect(reserve(user.toString(), 600, reservationId)).resolves.toEqual({+ reserved: true,+ balance: 600,+ });+ expect(reads).toHaveBeenCalledTimes(2);+ const stored = await readState(user);+ expect(stored?.reservations?.map((reservation) => reservation.id)).toEqual([+ 'renewed',+ reservationId,+ ]);+ expect(stored?.reservedCredits).toBe(1000);+ });++ test('renews a live reservation and never resurrects a released one', async () => {+ const user = new mongoose.Types.ObjectId();+ await Balance.create({ user, tokenCredits: 1000 });+ const liveId = newId();+ const releasedId = newId();+ await reserve(user.toString(), 300, liveId);+ await reserve(user.toString(), 200, releasedId);+ await releaseBalanceReservation({+ user: user.toString(),+ reservationId: releasedId,+ amount: 200,+ });++ const renewedUntil = new Date(Date.now() + 3_600_000);+ await renewBalanceReservation({+ user: user.toString(),+ reservationId: liveId,+ expiresAt: renewedUntil,+ });+ await renewBalanceReservation({+ user: user.toString(),+ reservationId: releasedId,+ expiresAt: renewedUntil,+ });++ const stored = await readState(user);+ expect(stored?.reservations).toEqual([{ id: liveId, amount: 300, expiresAt: renewedUntil }]);+ expect(stored?.reservedCredits).toBe(300);+ });++ test('keeps a reservation that was renewed after a pruning read saw it expired', async () => {+ const user = new mongoose.Types.ObjectId();+ await Balance.create({+ user,+ tokenCredits: 1000,+ reservedCredits: 900,+ reservations: [{ id: 'renewed', amount: 900, expiresAt: new Date(Date.now() - 1000) }],+ });++ interleaveBeforeNextWrite(() =>+ renewBalanceReservation({+ user: user.toString(),+ reservationId: 'renewed',+ expiresAt: inFuture(),+ }),+ );++ await expect(reserve(user.toString(), 600)).resolves.toEqual({+ reserved: false,+ balance: 100,+ });+ const stored = await readState(user);+ expect(stored?.reservations?.map((reservation) => reservation.id)).toEqual(['renewed']);+ expect(stored?.reservedCredits).toBe(900);+ });++ test('admits a zero-cost request without storing a reservation', async () => {+ const user = new mongoose.Types.ObjectId();+ await Balance.create({ user, tokenCredits: 0 });++ await expect(reserve(user.toString(), 0)).resolves.toEqual({ reserved: true, balance: 0 });++ const stored = await readState(user);+ expect(stored?.reservations).toBeUndefined();+ expect(stored?.reservedCredits).toBeUndefined();+ });++ test('reports only unexpired reservations when a read asks for the reserved credits', async () => {+ const user = new mongoose.Types.ObjectId();+ await Balance.create({+ user,+ tokenCredits: 1000,+ reservedCredits: 1200,+ reservations: [+ { id: 'live', amount: 300, expiresAt: inFuture() },+ { id: 'stale', amount: 900, expiresAt: new Date(Date.now() - 1000) },+ ],+ });++ const record = await findBalanceByUser(user.toString(), { includeReservedCredits: true });++ expect(record?.tokenCredits).toBe(1000);+ expect(record?.reservedCredits).toBe(300);+ expect(record).not.toHaveProperty('reservations');+ });++ test('keeps reservation state out of ordinary balance reads and writes', async () => {+ const user = new mongoose.Types.ObjectId();+ await Balance.create({ user, tokenCredits: 1000 });+ await reserve(user.toString(), 100);++ const hidden = ['reservations', 'reservedCredits', 'pendingRefill'];+ const record = await findBalanceByUser(user.toString());+ const synced = await upsertBalanceFields(user.toString(), { refillAmount: 5 });+ const spent = await updateBalance({ user: user.toString(), incrementValue: -10 });++ expect(record?.tokenCredits).toBe(1000);+ expect(spent.tokenCredits).toBe(990);+ for (const read of [record, synced, spent]) {+ for (const field of hidden) {+ expect(read).not.toHaveProperty(field);+ }+ }+ });++ test('removes a reservation whose write committed but failed to acknowledge', async () => {+ const user = new mongoose.Types.ObjectId();+ await Balance.create({ user, tokenCredits: 1000 });+ const realUpdateOne = Balance.updateOne.bind(Balance);+ jest.spyOn(Balance, 'updateOne').mockImplementationOnce(((+ ...args: Parameters<typeof Balance.updateOne>+ ) =>+ realUpdateOne(...args).then(() => {+ throw new Error('connection reset');+ })) as unknown as typeof Balance.updateOne);++ await expect(reserve(user.toString(), 400)).rejects.toThrow('connection reset');++ const stored = await readState(user);+ expect(stored?.reservations).toEqual([]);+ expect(stored?.reservedCredits).toBe(0);+ });++ test('re-reads when a spend lands between the read and the write', async () => {+ const user = new mongoose.Types.ObjectId();+ await Balance.create({ user, tokenCredits: 1000 });++ interleaveBeforeNextWrite(() =>+ Balance.collection.updateOne({ user }, { $inc: { tokenCredits: -800 } }),+ );++ await expect(reserve(user.toString(), 500)).resolves.toEqual({+ reserved: false,+ balance: 200,+ });+ expect((await readState(user))?.reservedCredits).toBeUndefined();+ });++ test('re-reads when another admission takes the credits between the read and the write', async () => {+ const user = new mongoose.Types.ObjectId();+ await Balance.create({ user, tokenCredits: 1000 });++ interleaveBeforeNextWrite(() => reserve(user.toString(), 800));++ await expect(reserve(user.toString(), 500)).resolves.toEqual({+ reserved: false,+ balance: 200,+ });+ expect((await readState(user))?.reservedCredits).toBe(800);+ });++ describe('auto-refill', () => {+ const refillable = {+ tokenCredits: 0,+ autoRefillEnabled: true,+ refillAmount: 1000,+ refillIntervalValue: 30,+ refillIntervalUnit: 'days' as const,+ lastRefill: new Date('2020-01-01T00:00:00.000Z'),+ };++ test('applies one refill per eligibility window across concurrent admissions', async () => {+ const user = new mongoose.Types.ObjectId();+ await Balance.create({ user, ...refillable });++ const results = await Promise.all(+ Array.from({ length: 10 }, () => reserve(user.toString(), 150)),+ );++ expect(results.filter((result) => result?.reserved)).toHaveLength(6);+ const stored = await readState(user);+ expect(stored?.tokenCredits).toBe(1000);+ expect(stored?.lastRefill.getTime()).toBeGreaterThan(refillable.lastRefill.getTime());+ expect(stored?.pendingRefill).toBeUndefined();+ const refills = await Transaction.find({ user, context: 'autoRefill' }).lean();+ expect(refills).toHaveLength(1);+ expect(refills[0].rawAmount).toBe(1000);+ });++ test.each([+ ['lastRefill advances', { lastRefill: new Date() }, 0],+ ['auto-refill is disabled', { autoRefillEnabled: false }, 0],+ ['the refill amount changes', { refillAmount: 10 }, 10],+ ['the refill interval lengthens', { refillIntervalValue: 100_000 }, 0],+ ])(+ 'applies the current refill settings when %s mid-attempt',+ async (_case, change, credits) => {+ const user = new mongoose.Types.ObjectId();+ await Balance.create({ user, ...refillable });++ interleaveBeforeNextWrite(() => Balance.collection.updateOne({ user }, { $set: change }));++ await expect(reserve(user.toString(), 150)).resolves.toEqual({+ reserved: false,+ balance: credits,+ });+ expect((await readState(user))?.tokenCredits).toBe(credits);+ expect(await Transaction.countDocuments({ user, context: 'autoRefill' })).toBe(+ credits > 0 ? 1 : 0,+ );+ },+ );++ test('does not refill when a release frees enough credits mid-attempt', async () => {+ const user = new mongoose.Types.ObjectId();+ await Balance.create({ user, ...refillable, tokenCredits: 1000 });+ const heldId = newId();+ await reserve(user.toString(), 900, heldId);++ interleaveBeforeNextWrite(() =>+ releaseBalanceReservation({ user: user.toString(), reservationId: heldId, amount: 900 }),+ );++ await expect(reserve(user.toString(), 200)).resolves.toEqual({+ reserved: true,+ balance: 1000,+ });+ const stored = await readState(user);+ expect(stored?.tokenCredits).toBe(1000);+ expect(stored?.lastRefill.getTime()).toBe(refillable.lastRefill.getTime());+ expect(await Transaction.countDocuments({ user, context: 'autoRefill' })).toBe(0);+ });++ test('still applies a due refill when the refilled balance cannot cover the request', async () => {+ const user = new mongoose.Types.ObjectId();+ await Balance.create({ user, ...refillable, refillAmount: 100 });++ await expect(reserve(user.toString(), 500)).resolves.toEqual({+ reserved: false,+ balance: 100,+ });+ const stored = await readState(user);+ expect(stored?.tokenCredits).toBe(100);+ expect(stored?.reservations).toBeUndefined();+ });++ test.each([+ ['covers the request exactly', 100, true, 100],+ ['still falls short', 10, false, 10],+ ])(+ 'applies a refill whose interval is due again at once only once per admission when it %s',+ async (_case, refillAmount, reserved, credits) => {+ const user = new mongoose.Types.ObjectId();+ await Balance.create({ user, ...refillable, refillAmount, refillIntervalValue: 0 });++ await expect(reserve(user.toString(), 100)).resolves.toEqual({+ reserved,+ balance: credits,+ });+ expect((await readState(user))?.tokenCredits).toBe(credits);+ expect(await Transaction.countDocuments({ user, context: 'autoRefill' })).toBe(1);+ },+ );++ test('does not refill while the unreserved balance covers the request', async () => {+ const user = new mongoose.Types.ObjectId();+ await Balance.create({ user, ...refillable, tokenCredits: 500 });++ await expect(reserve(user.toString(), 100)).resolves.toEqual({+ reserved: true,+ balance: 500,+ });+ expect((await readState(user))?.tokenCredits).toBe(500);+ });++ test('records the ledger transaction of a refill whose first recording failed', async () => {+ const user = new mongoose.Types.ObjectId();+ await Balance.create({ user, ...refillable });+ const save = jest+ .spyOn(Transaction.prototype, 'save')+ .mockRejectedValue(new Error('ledger unavailable'));++ await expect(reserve(user.toString(), 150)).resolves.toEqual({+ reserved: true,+ balance: 1000,+ });+ const applied = await readState(user);+ expect(applied?.tokenCredits).toBe(1000);+ expect(applied?.pendingRefill?.rawAmount).toBe(1000);+ expect(await Transaction.countDocuments({ user, context: 'autoRefill' })).toBe(0);++ save.mockRestore();+ await reserve(user.toString(), 150);++ const settled = await readState(user);+ expect(settled?.tokenCredits).toBe(1000);+ expect(settled?.pendingRefill).toBeUndefined();+ const refills = await Transaction.find({ user, context: 'autoRefill' }).lean();+ expect(refills.map((refill) => refill._id.toString())).toEqual([+ applied?.pendingRefill?.transactionId.toString(),+ ]);+ });++ test('does not duplicate a ledger transaction that was recorded before its marker cleared', async () => {+ const user = new mongoose.Types.ObjectId();+ const transactionId = new mongoose.Types.ObjectId();+ await Transaction.create({+ _id: transactionId,+ user,+ tokenType: 'credits',+ context: 'autoRefill',+ rawAmount: 1000,+ });+ await Balance.create({+ user,+ ...refillable,+ tokenCredits: 1000,+ lastRefill: new Date(),+ pendingRefill: { transactionId, rawAmount: 1000 },+ });++ await reserve(user.toString(), 150);++ expect((await readState(user))?.pendingRefill).toBeUndefined();+ expect(await Transaction.countDocuments({ user, context: 'autoRefill' })).toBe(1);+ });+ });+});diff --git a/packages/data-schemas/src/methods/transaction.ts b/packages/data-schemas/src/methods/transaction.tsindex ec39158db4..736d6720d7 100644--- a/packages/data-schemas/src/methods/transaction.ts+++ b/packages/data-schemas/src/methods/transaction.ts@@ -1,9 +1,24 @@-import type { FilterQuery, Model, Types } from 'mongoose';-import type { IBalance, IBalanceUpdate, TransactionData } from '~/types';+import { getRefillEligibilityDate } from 'librechat-data-provider';+import type { AnyBulkWriteOperation, FilterQuery, Model, Types } from 'mongoose';+import type {+ BalanceReservationRequest,+ BalanceReservationRenewal,+ BalanceReservationRelease,+ BalanceReservationResult,+ IBalancePendingRefill,+ IBalanceReservation,+ IBalanceUpdate,+ TransactionData,+ IBalance,+} from '~/types';import type { ITransaction } from '~/schema/transaction';+import { tenantSafeBulkWrite } from '~/utils/tenantBulkWrite';import logger from '~/config/winston';const cancelRate = 1.15;+const maxReservationAttempts = 10;+/** Every balance read resolves a user to their oldest record, so duplicate records stay inert */+const oldestFirst = { _id: 1 } as const;type MultiplierParams = {model?: string;@@ -82,23 +97,24 @@ export function createTransactionMethods(setValues?: IBalanceUpdate;}) => Promise<IBalance>;bulkInsertTransactions: (docs: TransactionData[]) => Promise<void>;- findBalanceByUser: (user: string) => Promise<IBalance | null>;- upsertBalanceFields: (user: string, fields: IBalanceUpdate) => Promise<IBalance | null>;+ findBalanceByUser: (+ user: string,+ options?: { includeReservedCredits?: boolean },+ ) => Promise<IBalance | null>;+ upsertBalanceFields: (+ user: string,+ fields: IBalanceUpdate,+ insertOnly?: IBalanceUpdate,+ ) => Promise<IBalance | null>;getTransactions: (filter: FilterQuery<ITransaction>) => Promise<ITransaction[]>;deleteTransactions: (filter: FilterQuery<ITransaction>,) => Promise<import('mongodb').DeleteResult>;deleteBalances: (filter: FilterQuery<IBalance>) => Promise<import('mongodb').DeleteResult>;createTransaction: (_txData: TxData) => Promise<TransactionResult | undefined>;- createAutoRefillTransaction: (txData: TxData) => Promise<- | {- rate: number;- user: string;- balance: number;- transaction: ITransaction;- }- | undefined- >;+ reserveBalance: (request: BalanceReservationRequest) => Promise<BalanceReservationResult | null>;+ renewBalanceReservation: (params: BalanceReservationRenewal) => Promise<void>;+ releaseBalanceReservation: (params: BalanceReservationRelease) => Promise<void>;createStructuredTransaction: (_txData: TxData) => Promise<TransactionResult | undefined>;} {/** Calculate and set the tokenValue for a transaction */@@ -225,7 +241,7 @@ export function createTransactionMethods(for (let attempt = 1; attempt <= maxRetries; attempt++) {let currentBalanceDoc: IBalance | null;try {- currentBalanceDoc = await Balance.findOne({ user }).lean<IBalance>();+ currentBalanceDoc = await Balance.findOne({ user }).sort(oldestFirst).lean<IBalance>();const currentCredits = currentBalanceDoc ? currentBalanceDoc.tokenCredits : 0;const potentialNewCredits = currentCredits + incrementValue;const newCredits = Math.max(0, potentialNewCredits);@@ -240,7 +256,7 @@ export function createTransactionMethods(let updatedBalance: IBalance | null = null;if (currentBalanceDoc) {updatedBalance = await Balance.findOneAndUpdate(- { user, tokenCredits: currentCredits },+ { _id: currentBalanceDoc._id, tokenCredits: currentCredits },updatePayload,{ new: true },).lean<IBalance>();@@ -250,29 +266,8 @@ export function createTransactionMethods(}lastError = new Error(`Concurrency conflict for user ${user} on attempt ${attempt}.`);} else {- try {- updatedBalance = await Balance.findOneAndUpdate({ user }, updatePayload, {- upsert: true,- new: true,- }).lean<IBalance>();-- if (updatedBalance) {- return updatedBalance;- }- lastError = new Error(- `Upsert race condition suspected for user ${user} on attempt ${attempt}.`,- );- } catch (error: unknown) {- if (- error instanceof Error &&- 'code' in error &&- (error as { code: number }).code === 11000- ) {- lastError = error;- } else {- throw error;- }- }+ await upsertBalanceRecord(user, {}, { tokenCredits: 0 });+ continue;}} catch (error) {logger.error(`[updateBalance] Error during attempt ${attempt} for user ${user}:`, error);@@ -297,41 +292,304 @@ export function createTransactionMethods();}+ function isAutoRefillDue(record: IBalance, now: Date): boolean {+ if (!record.autoRefillEnabled || !(record.refillAmount > 0)) {+ return false;+ }+ const lastRefill = new Date(record.lastRefill ?? 0);+ if (isNaN(lastRefill.getTime())) {+ return true;+ }+ return (+ now >=+ getRefillEligibilityDate(+ lastRefill,+ record.refillIntervalValue ?? 0,+ record.refillIntervalUnit ?? 'days',+ )+ );+ }++ function isDuplicateKeyError(error: unknown): boolean {+ return error instanceof Error && 'code' in error && (error as { code: number }).code === 11000;+ }+/**- * Creates an auto-refill transaction that also updates balance.+ * Records the ledger transaction of an applied auto-refill, then clears its marker. The+ * transaction id is fixed when the refill is applied, so replaying a marker left by a failed or+ * interrupted attempt records the transaction at most once.*/- async function createAutoRefillTransaction(txData: TxData): Promise<- | {- rate: number;- user: string;- balance: number;- transaction: ITransaction;+ async function settleAutoRefill(+ balanceId: unknown,+ user: Types.ObjectId,+ { transactionId, rawAmount }: IBalancePendingRefill,+ ): Promise<boolean> {+ try {+ const Transaction = mongoose.models.Transaction;+ const transaction = new Transaction({+ _id: transactionId,+ user,+ tokenType: 'credits',+ context: 'autoRefill',+ rawAmount,+ });+ calculateTokenValue(transaction);+ await transaction.save();+ } catch (error) {+ if (!isDuplicateKeyError(error)) {+ logger.error('[Balance.reserve] Failed to record auto-refill transaction', error);+ return false;}- | undefined- > {- if (txData.rawAmount != null && isNaN(txData.rawAmount)) {- return;}- const Transaction = mongoose.models.Transaction;- const transaction = new Transaction(txData);- transaction.endpointTokenConfig = txData.endpointTokenConfig;- transaction.inputTokenCount = txData.inputTokenCount;- calculateTokenValue(transaction);- await transaction.save();- const balanceResponse = await updateBalance({- user: transaction.user as string,- incrementValue: txData.rawAmount ?? 0,- setValues: { lastRefill: new Date() },- });- const result = {- rate: transaction.rate as number,- user: transaction.user.toString() as string,- balance: balanceResponse.tokenCredits,- transaction,+ try {+ const Balance = mongoose.models.Balance as Model<IBalance>;+ await Balance.updateOne(+ { _id: balanceId, 'pendingRefill.transactionId': transactionId },+ { $unset: { pendingRefill: 1 } },+ );+ return true;+ } catch (error) {+ logger.error('[Balance.reserve] Failed to clear recorded auto-refill', error);+ return false;+ }+ }++ /**+ * Applies a due auto-refill with a write fenced on every value the refill decision read, and+ * leaves a ledger marker in that same write. Returns false when the fence missed.+ */+ async function applyAutoRefill(record: IBalance, now: Date): Promise<boolean> {+ const Balance = mongoose.models.Balance as Model<IBalance>;+ const pendingRefill: IBalancePendingRefill = {+ transactionId: new mongoose.Types.ObjectId(),+ rawAmount: record.refillAmount,};- logger.debug('[Balance.check] Auto-refill performed', result);- return result;+ const result = await Balance.updateOne(+ {+ _id: record._id,+ tokenCredits: record.tokenCredits ?? null,+ reservedCredits: record.reservedCredits ?? null,+ lastRefill: record.lastRefill ?? null,+ pendingRefill: null,+ autoRefillEnabled: record.autoRefillEnabled,+ refillAmount: record.refillAmount,+ refillIntervalValue: record.refillIntervalValue ?? null,+ refillIntervalUnit: record.refillIntervalUnit ?? null,+ },+ {+ $set: {+ tokenCredits: Math.max(0, (record.tokenCredits ?? 0) + record.refillAmount),+ lastRefill: now,+ pendingRefill,+ },+ },+ );+ if (result.matchedCount !== 1) {+ return false;+ }+ logger.debug('[Balance.reserve] Auto-refill applied', {+ user: record.user,+ rawAmount: record.refillAmount,+ });+ await settleAutoRefill(record._id, record.user, pendingRefill);+ return true;+ }++ /**+ * Removes reservations and their credits from the running total, one atomic update per+ * reservation sent as a single bulk write, so each is removed only while it is still held at the+ * amount read and the rest proceed when one has changed. With `expiredBy`, a reservation is+ * removed only while it is still expired at that instant, so one renewed after it was read+ * survives.+ */+ async function removeReservations(+ filter: FilterQuery<IBalance>,+ reservations: Array<Pick<IBalanceReservation, 'id' | 'amount'>>,+ expiredBy?: Date,+ ): Promise<void> {+ const Balance = mongoose.models.Balance as Model<IBalance>;+ const expiry = expiredBy ? { expiresAt: { $lte: expiredBy } } : {};+ const removals: AnyBulkWriteOperation<IBalance>[] = reservations.map(({ id, amount }) => ({+ updateOne: {+ filter: { ...filter, reservations: { $elemMatch: { id, amount, ...expiry } } },+ update: { $pull: { reservations: { id } }, $inc: { reservedCredits: -amount } },+ },+ }));+ await tenantSafeBulkWrite(Balance, removals as AnyBulkWriteOperation[], { ordered: false });+ }++ /**+ * Applies `fields` to the user's balance record, creating the record when the user has none.+ * A created record is keyed by the user id, so creators racing on a missing record converge on+ * one document instead of each inserting their own; `insertOnly` applies only on creation.+ */+ async function upsertBalanceRecord(+ user: string,+ fields: IBalanceUpdate,+ insertOnly: IBalanceUpdate = {},+ ): Promise<IBalance | null> {+ const Balance = mongoose.models.Balance as Model<IBalance>;+ const { user: _fieldsUser, ...set } = fields;+ const setOnInsert = Object.fromEntries(+ Object.entries(insertOnly).filter(([key]) => key !== 'user' && !(key in set)),+ );+ const updatesExisting = Object.keys(set).length > 0;++ const existing = updatesExisting+ ? await Balance.findOneAndUpdate(+ { user },+ { $set: set },+ { new: true, sort: oldestFirst },+ ).lean<IBalance>()+ : await Balance.findOne({ user }).sort(oldestFirst).lean<IBalance>();+ if (existing) {+ return existing;+ }++ const create = () =>+ Balance.findOneAndUpdate(+ { _id: user },+ { ...(updatesExisting ? { $set: set } : {}), $setOnInsert: { ...setOnInsert, user } },+ { upsert: true, new: true },+ ).lean<IBalance>();+ try {+ return await create();+ } catch (error) {+ if (!isDuplicateKeyError(error)) {+ throw error;+ }+ return create();+ }+ }++ /**+ * Admits one request against the credits that in-flight requests have not reserved, and holds+ * its amount until released or expired.+ *+ * Admission is one conditional `$push`/`$inc`: it matches only while `tokenCredits` is at least+ * the value read and `reservedCredits` still leaves room for the amount, so concurrent+ * admissions of a funded balance all commit, and an admission that lost the credits re-reads.+ * A due auto-refill is applied first by its own fenced write, at most once per admission, so a+ * refill interval that is due again the instant it is applied still refills once. Returns null+ * when the user has no balance record and no `initialBalance` was given.+ */+ async function reserveBalance({+ user,+ reservationId,+ amount,+ expiresAt,+ initialBalance,+ }: BalanceReservationRequest): Promise<BalanceReservationResult | null> {+ const Balance = mongoose.models.Balance as Model<IBalance>;+ let delay = 10;+ let refilled = false;++ for (let attempt = 1; attempt <= maxReservationAttempts; attempt++) {+ const record = await Balance.findOne({ user })+ .sort(oldestFirst)+ .select('+reservations +reservedCredits +pendingRefill')+ .lean<IBalance>();+ if (!record) {+ if (!initialBalance) {+ return null;+ }+ await upsertBalanceRecord(user, {}, initialBalance);+ continue;+ }++ const now = new Date();+ const refillSettled =+ record.pendingRefill == null ||+ (await settleAutoRefill(record._id, record.user, record.pendingRefill));++ const expired = (record.reservations ?? []).filter(+ (reservation) => reservation.expiresAt <= now,+ );+ if (expired.length > 0) {+ await removeReservations({ _id: record._id }, expired, now);+ continue;+ }++ const credits = record.tokenCredits ?? 0;+ const balance = credits - (record.reservedCredits ?? 0);+ if (refillSettled && !refilled && balance - amount <= 0 && isAutoRefillDue(record, now)) {+ refilled = await applyAutoRefill(record, now);+ continue;+ }++ const reserved = balance >= amount;+ if (!reserved || !(amount > 0)) {+ return { reserved, balance };+ }++ const held = Math.ceil(amount);+ let result: Awaited<ReturnType<typeof Balance.updateOne>>;+ try {+ result = await Balance.updateOne(+ {+ _id: record._id,+ tokenCredits: { $gte: credits },+ $or: [+ { reservedCredits: { $lte: credits - amount } },+ { reservedCredits: { $exists: false } },+ ],+ },+ {+ $push: { reservations: { id: reservationId, amount: held, expiresAt } },+ $inc: { reservedCredits: held },+ },+ );+ } catch (error) {+ /** The write may have committed before its acknowledgement was lost; the caller never+ * receives a handle to release it, so remove it here rather than leave it to expire. */+ await removeReservations({ _id: record._id }, [{ id: reservationId, amount: held }]).catch(+ (cleanupError) => {+ logger.error('[Balance.reserve] Failed to remove an unacknowledged reservation', {+ user,+ error: cleanupError,+ });+ },+ );+ throw error;+ }+ if (result.matchedCount === 1) {+ return { reserved, balance };+ }++ if (attempt < maxReservationAttempts) {+ await new Promise((resolve) => setTimeout(resolve, delay + Math.random() * delay));+ delay = Math.min(delay * 2, 500);+ }+ }++ throw new Error(`Balance reservation for user ${user} exceeded its retry bound.`);+ }++ /** Extends an in-flight reservation's expiry; a reservation already released or pruned stays gone. */+ async function renewBalanceReservation({+ user,+ reservationId,+ expiresAt,+ }: BalanceReservationRenewal): Promise<void> {+ const Balance = mongoose.models.Balance as Model<IBalance>;+ await Balance.updateOne(+ { user, 'reservations.id': reservationId },+ { $set: { 'reservations.$[held].expiresAt': expiresAt } },+ { arrayFilters: [{ 'held.id': reservationId }] },+ );+ }++ /** Releases an in-flight reservation; releasing an unknown or already pruned id is a no-op. */+ async function releaseBalanceReservation({+ user,+ reservationId,+ amount,+ }: BalanceReservationRelease): Promise<void> {+ if (!(amount > 0)) {+ return;+ }+ await removeReservations({ user }, [{ id: reservationId, amount: Math.ceil(amount) }]);}/**@@ -424,23 +682,40 @@ export function createTransactionMethods(}}- /** Retrieves a user's balance record. */- async function findBalanceByUser(user: string): Promise<IBalance | null> {+ /**+ * Retrieves a user's balance record. With `includeReservedCredits`, `reservedCredits` is the+ * total of the reservations that have not expired, so a reservation left by a crashed request+ * stops counting at its expiry even before a reservation write prunes it.+ */+ async function findBalanceByUser(+ user: string,+ options?: { includeReservedCredits?: boolean },+ ): Promise<IBalance | null> {const Balance = mongoose.models.Balance as Model<IBalance>;- return Balance.findOne({ user }).lean<IBalance>();+ const query = Balance.findOne({ user }).sort(oldestFirst);+ if (!options?.includeReservedCredits) {+ return query.lean<IBalance>();+ }+ const record = await query.select('+reservations').lean<IBalance>();+ if (!record) {+ return null;+ }+ const now = new Date();+ const { reservations, ...balance } = record;+ const reservedCredits = (reservations ?? []).reduce(+ (sum, reservation) => (reservation.expiresAt > now ? sum + reservation.amount : sum),+ 0,+ );+ return { ...balance, reservedCredits } as IBalance;}- /** Upserts balance fields for a user. */+ /** Upserts balance fields for a user; `insertOnly` fields apply only when the record is created. */async function upsertBalanceFields(user: string,fields: IBalanceUpdate,+ insertOnly?: IBalanceUpdate,): Promise<IBalance | null> {- const Balance = mongoose.models.Balance as Model<IBalance>;- return Balance.findOneAndUpdate(- { user },- { $set: fields },- { upsert: true, new: true },- ).lean<IBalance>();+ return upsertBalanceRecord(user, fields, insertOnly);}/** Deletes transactions matching a filter. */@@ -481,7 +756,9 @@ export function createTransactionMethods(deleteTransactions,deleteBalances,createTransaction,- createAutoRefillTransaction,+ reserveBalance,+ renewBalanceReservation,+ releaseBalanceReservation,createStructuredTransaction,};}diff --git a/packages/data-schemas/src/methods/user.methods.spec.ts b/packages/data-schemas/src/methods/user.methods.spec.tsindex bc122b0239..a85eb5525b 100644--- a/packages/data-schemas/src/methods/user.methods.spec.ts+++ b/packages/data-schemas/src/methods/user.methods.spec.ts@@ -332,6 +332,7 @@ describe('User Methods - Database Tests', () => {const balance = await Balance.findOne({ user: userId });expect(balance).toBeDefined();expect(balance?.tokenCredits).toBe(1000);+ expect(balance?._id.toString()).toBe(userId.toString());});});diff --git a/packages/data-schemas/src/methods/user.ts b/packages/data-schemas/src/methods/user.tsindex d1c4d20b03..e5c472bbd7 100644--- a/packages/data-schemas/src/methods/user.ts+++ b/packages/data-schemas/src/methods/user.ts@@ -272,10 +272,11 @@ export function createUserMethods(};}- await Balance.findOneAndUpdate({ user: user._id }, update, {- upsert: true,- new: true,- }).lean();+ await Balance.findOneAndUpdate(+ { _id: user._id },+ { ...update, $setOnInsert: { user: user._id } },+ { upsert: true, new: true },+ ).lean();}if (returnUser) {diff --git a/packages/data-schemas/src/schema/balance.ts b/packages/data-schemas/src/schema/balance.tsindex 4314d1f2cf..1191098f48 100644--- a/packages/data-schemas/src/schema/balance.ts+++ b/packages/data-schemas/src/schema/balance.ts@@ -41,6 +41,32 @@ const balanceSchema: Schema<t.IBalance> = new Schema<t.IBalance>({type: String,index: true,},+ /** Credits held by in-flight requests; released when each request settles, or pruned once expired */+ reservations: {+ type: [+ {+ _id: false,+ id: { type: String, required: true },+ amount: { type: Number, required: true },+ expiresAt: { type: Date, required: true },+ },+ ],+ default: undefined,+ select: false,+ },+ reservedCredits: {+ type: Number,+ select: false,+ },+ pendingRefill: {+ type: {+ transactionId: { type: Schema.Types.ObjectId, required: true },+ rawAmount: { type: Number, required: true },+ },+ _id: false,+ default: undefined,+ select: false,+ },});export default balanceSchema;diff --git a/packages/data-schemas/src/types/balance.ts b/packages/data-schemas/src/types/balance.tsindex ad14cc1464..e52b4b3f6b 100644--- a/packages/data-schemas/src/types/balance.ts+++ b/packages/data-schemas/src/types/balance.ts@@ -1,6 +1,19 @@import type { RefillIntervalUnit } from 'librechat-data-provider';import type { Document, Types } from 'mongoose';+/** Whole credits held against a balance while the request that reserved them is in flight */+export interface IBalanceReservation {+ id: string;+ amount: number;+ expiresAt: Date;+}++/** An applied auto-refill whose ledger transaction has not been confirmed as recorded */+export interface IBalancePendingRefill {+ transactionId: Types.ObjectId;+ rawAmount: number;+}+export interface IBalance extends Document {user: Types.ObjectId;tokenCredits: number;@@ -11,6 +24,11 @@ export interface IBalance extends Document {lastRefill: Date;refillAmount: number;tenantId?: string;+ /** Reservation state is excluded from reads unless explicitly selected */+ reservations?: IBalanceReservation[];+ /** Sum of `reservations` amounts, maintained by the same writes */+ reservedCredits?: number;+ pendingRefill?: IBalancePendingRefill;}/** Plain data fields for creating or updating a balance record (no Mongoose Document methods) */@@ -23,3 +41,35 @@ export interface IBalanceUpdate {refillAmount?: number;lastRefill?: Date;}++/** Holds credits against a user's balance for the lifetime of one in-flight request */+export interface BalanceReservationRequest {+ user: string;+ /** Unique per request; the release addresses the reservation by this id */+ reservationId: string;+ /** Credits the request is admitted against */+ amount: number;+ /** An unreleased reservation stops counting against the balance at this instant */+ expiresAt: Date;+ /** Creates the balance record with these fields when the user has none */+ initialBalance?: IBalanceUpdate;+}++export interface BalanceReservationRenewal {+ user: string;+ reservationId: string;+ expiresAt: Date;+}++export interface BalanceReservationRelease {+ user: string;+ reservationId: string;+ /** The amount that was reserved */+ amount: number;+}++export interface BalanceReservationResult {+ reserved: boolean;+ /** Credits not held by other in-flight requests, after any auto-refill */+ balance: number;+}diff --git a/packages/data-schemas/src/types/user.ts b/packages/data-schemas/src/types/user.tsindex 251e0ff122..aba23077f6 100644--- a/packages/data-schemas/src/types/user.ts+++ b/packages/data-schemas/src/types/user.ts@@ -93,6 +93,7 @@ export interface BalanceConfig {refillIntervalValue?: number;refillIntervalUnit?: RefillIntervalUnit;refillAmount?: number;+ reservationTtlMs?: number;}export interface CreateUserRequest extends Partial<IUser> {