Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
💎 fix: Gemini Image Gen Tool Vertex AI Auth and File Storage (#11923)
Danny Avilaf3eb197675d8
diff --git a/.env.example b/.env.exampleindex b7ec3a3dad..f6d2ec271f 100644--- a/.env.example+++ b/.env.example@@ -243,10 +243,6 @@ GOOGLE_KEY=user_provided# Option A: Use dedicated Gemini API key for image generation# GEMINI_API_KEY=your-gemini-api-key-# Option B: Use Vertex AI (no API key needed, uses service account)-# Set this to enable Vertex AI and allow tool without requiring API keys-# GEMINI_VERTEX_ENABLED=true-# Vertex AI model for image generation (defaults to gemini-2.5-flash-image)# GEMINI_IMAGE_MODEL=gemini-2.5-flash-imagediff --git a/api/app/clients/tools/manifest.json b/api/app/clients/tools/manifest.jsonindex 7930e67ac9..b0dca8971e 100644--- a/api/app/clients/tools/manifest.json+++ b/api/app/clients/tools/manifest.json@@ -161,9 +161,10 @@"icon": "assets/gemini_image_gen.svg","authConfig": [{- "authField": "GEMINI_API_KEY||GOOGLE_KEY||GEMINI_VERTEX_ENABLED",- "label": "Gemini API Key (Optional if Vertex AI is configured)",- "description": "Your Google Gemini API Key from <a href='https://aistudio.google.com/app/apikey' target='_blank'>Google AI Studio</a>. Leave blank if using Vertex AI with service account."+ "authField": "GEMINI_API_KEY||GOOGLE_KEY||GOOGLE_SERVICE_KEY_FILE",+ "label": "Gemini API Key (optional)",+ "description": "Your Google Gemini API Key from <a href='https://aistudio.google.com/app/apikey' target='_blank'>Google AI Studio</a>. Leave blank to use Vertex AI with a service account (GOOGLE_SERVICE_KEY_FILE or api/data/auth.json).",+ "optional": true}]}diff --git a/api/app/clients/tools/structured/GeminiImageGen.js b/api/app/clients/tools/structured/GeminiImageGen.jsindex c0e5a0ce1d..b201db019d 100644--- a/api/app/clients/tools/structured/GeminiImageGen.js+++ b/api/app/clients/tools/structured/GeminiImageGen.js@@ -1,4 +1,3 @@-const fs = require('fs');const path = require('path');const sharp = require('sharp');const { v4 } = require('uuid');@@ -6,12 +5,7 @@ const { ProxyAgent } = require('undici');const { GoogleGenAI } = require('@google/genai');const { tool } = require('@langchain/core/tools');const { logger } = require('@librechat/data-schemas');-const {- FileContext,- ContentTypes,- FileSources,- EImageOutputType,-} = require('librechat-data-provider');+const { ContentTypes, EImageOutputType } = require('librechat-data-provider');const {geminiToolkit,loadServiceKey,@@ -59,17 +53,12 @@ const displayMessage =* @returns {string} - The processed string*/function replaceUnwantedChars(inputString) {- return inputString?.replace(/[^\w\s\-_.,!?()]/g, '') || '';-}--/**- * Validate and sanitize image format- * @param {string} format - The format to validate- * @returns {string} - Safe format- */-function getSafeFormat(format) {- const allowedFormats = ['png', 'jpg', 'jpeg', 'webp', 'gif'];- return allowedFormats.includes(format?.toLowerCase()) ? format.toLowerCase() : 'png';+ return (+ inputString+ ?.replace(/\r\n|\r|\n/g, ' ')+ .replace(/"/g, '')+ .trim() || ''+ );}/**@@ -117,11 +106,8 @@ async function initializeGeminiClient(options = {}) {return new GoogleGenAI({ apiKey: googleKey });}- // Fall back to Vertex AI with service accountlogger.debug('[GeminiImageGen] Using Vertex AI with service account');const credentialsPath = getDefaultServiceKeyPath();-- // Use loadServiceKey for consistent loading (supports file paths, JSON strings, base64)const serviceKey = await loadServiceKey(credentialsPath);if (!serviceKey || !serviceKey.project_id) {@@ -131,75 +117,14 @@ async function initializeGeminiClient(options = {}) {);}- // Set GOOGLE_APPLICATION_CREDENTIALS for any Google Cloud SDK dependencies- try {- await fs.promises.access(credentialsPath);- process.env.GOOGLE_APPLICATION_CREDENTIALS = credentialsPath;- } catch {- // File doesn't exist, skip setting env var- }-return new GoogleGenAI({vertexai: true,project: serviceKey.project_id,location: process.env.GOOGLE_LOC || process.env.GOOGLE_CLOUD_LOCATION || 'global',+ googleAuthOptions: { credentials: serviceKey },});}-/**- * Save image to local filesystem- * @param {string} base64Data - Base64 encoded image data- * @param {string} format - Image format- * @param {string} userId - User ID- * @returns {Promise<string>} - The relative URL- */-async function saveImageLocally(base64Data, format, userId) {- const safeFormat = getSafeFormat(format);- const safeUserId = userId ? path.basename(userId) : 'default';- const imageName = `gemini-img-${v4()}.${safeFormat}`;- const userDir = path.join(process.cwd(), 'client/public/images', safeUserId);-- await fs.promises.mkdir(userDir, { recursive: true });-- const filePath = path.join(userDir, imageName);- await fs.promises.writeFile(filePath, Buffer.from(base64Data, 'base64'));-- logger.debug('[GeminiImageGen] Image saved locally to:', filePath);- return `/images/${safeUserId}/${imageName}`;-}--/**- * Save image to cloud storage- * @param {Object} params - Parameters- * @returns {Promise<string|null>} - The storage URL or null- */-async function saveToCloudStorage({ base64Data, format, processFileURL, fileStrategy, userId }) {- if (!processFileURL || !fileStrategy || !userId) {- return null;- }-- try {- const safeFormat = getSafeFormat(format);- const safeUserId = path.basename(userId);- const dataURL = `data:image/${safeFormat};base64,${base64Data}`;- const imageName = `gemini-img-${v4()}.${safeFormat}`;-- const result = await processFileURL({- URL: dataURL,- basePath: 'images',- userId: safeUserId,- fileName: imageName,- fileStrategy,- context: FileContext.image_generation,- });-- return result.filepath;- } catch (error) {- logger.error('[GeminiImageGen] Error saving to cloud storage:', error);- return null;- }-}-/*** Convert image files to Gemini inline data format* @param {Object} params - Parameters@@ -390,34 +315,18 @@ function createGeminiImageTool(fields = {}) {throw new Error('This tool is only available for agents.');}- // Skip validation during tool creation - validation happens at runtime in initializeGeminiClient- // This allows the tool to be added to agents when using Vertex AI without requiring API keys- // The actual credentials check happens when the tool is invoked-- const {- req,- imageFiles = [],- processFileURL,- userId,- fileStrategy,- GEMINI_API_KEY,- GOOGLE_KEY,- // GEMINI_VERTEX_ENABLED is used for auth validation only (not used in code)- // When set as env var, it signals Vertex AI is configured and bypasses API key requirement- } = fields;+ const { req, imageFiles = [], userId, fileStrategy, GEMINI_API_KEY, GOOGLE_KEY } = fields;const imageOutputType = fields.imageOutputType || EImageOutputType.PNG;const geminiImageGenTool = tool(- async ({ prompt, image_ids, aspectRatio, imageSize }, _runnableConfig) => {+ async ({ prompt, image_ids, aspectRatio, imageSize }, runnableConfig) => {if (!prompt) {throw new Error('Missing required field: prompt');}- logger.debug('[GeminiImageGen] Generating image with prompt:', prompt?.substring(0, 100));- logger.debug('[GeminiImageGen] Options:', { aspectRatio, imageSize });+ logger.debug('[GeminiImageGen] Generating image', { aspectRatio, imageSize });- // Initialize Gemini client with user-provided credentialslet ai;try {ai = await initializeGeminiClient({@@ -432,10 +341,8 @@ function createGeminiImageTool(fields = {}) {];}- // Build request contentsconst contents = [{ text: replaceUnwantedChars(prompt) }];- // Add context images if providedif (image_ids?.length > 0) {const contextImages = await convertImagesToInlineData({imageFiles,@@ -447,28 +354,34 @@ function createGeminiImageTool(fields = {}) {logger.debug('[GeminiImageGen] Added', contextImages.length, 'context images');}- // Generate imagelet apiResponse;const geminiModel = process.env.GEMINI_IMAGE_MODEL || 'gemini-2.5-flash-image';- try {- // Build config with optional imageConfig- const config = {- responseModalities: ['TEXT', 'IMAGE'],- };-- // Add imageConfig if aspectRatio or imageSize is specified- // Note: gemini-2.5-flash-image doesn't support imageSize- const supportsImageSize = !geminiModel.includes('gemini-2.5-flash-image');- if (aspectRatio || (imageSize && supportsImageSize)) {- config.imageConfig = {};- if (aspectRatio) {- config.imageConfig.aspectRatio = aspectRatio;- }- if (imageSize && supportsImageSize) {- config.imageConfig.imageSize = imageSize;- }+ const config = {+ responseModalities: ['TEXT', 'IMAGE'],+ };++ const supportsImageSize = !geminiModel.includes('gemini-2.5-flash-image');+ if (aspectRatio || (imageSize && supportsImageSize)) {+ config.imageConfig = {};+ if (aspectRatio) {+ config.imageConfig.aspectRatio = aspectRatio;}+ if (imageSize && supportsImageSize) {+ config.imageConfig.imageSize = imageSize;+ }+ }++ let derivedSignal = null;+ let abortHandler = null;+ if (runnableConfig?.signal) {+ derivedSignal = AbortSignal.any([runnableConfig.signal]);+ abortHandler = () => logger.debug('[GeminiImageGen] Image generation aborted');+ derivedSignal.addEventListener('abort', abortHandler, { once: true });+ config.abortSignal = derivedSignal;+ }++ try {apiResponse = await ai.models.generateContent({model: geminiModel,contents,@@ -480,9 +393,12 @@ function createGeminiImageTool(fields = {}) {[{ type: ContentTypes.TEXT, text: `Image generation failed: ${error.message}` }],{ content: [], file_ids: [] },];+ } finally {+ if (abortHandler && derivedSignal) {+ derivedSignal.removeEventListener('abort', abortHandler);+ }}- // Check for safety blocksconst safetyBlock = checkForSafetyBlock(apiResponse);if (safetyBlock) {logger.warn('[GeminiImageGen] Safety block:', safetyBlock);@@ -509,46 +425,7 @@ function createGeminiImageTool(fields = {}) {const imageData = convertedBuffer.toString('base64');const mimeType = outputFormat === 'jpeg' ? 'image/jpeg' : `image/${outputFormat}`;- logger.debug('[GeminiImageGen] Image format:', { outputFormat, mimeType });-- let imageUrl;- const useLocalStorage = !fileStrategy || fileStrategy === FileSources.local;-- if (useLocalStorage) {- try {- imageUrl = await saveImageLocally(imageData, outputFormat, userId);- } catch (error) {- logger.error('[GeminiImageGen] Local save failed:', error);- imageUrl = `data:${mimeType};base64,${imageData}`;- }- } else {- const cloudUrl = await saveToCloudStorage({- base64Data: imageData,- format: outputFormat,- processFileURL,- fileStrategy,- userId,- });-- if (cloudUrl) {- imageUrl = cloudUrl;- } else {- // Fallback to local- try {- imageUrl = await saveImageLocally(imageData, outputFormat, userId);- } catch (_error) {- imageUrl = `data:${mimeType};base64,${imageData}`;- }- }- }-- logger.debug('[GeminiImageGen] Image URL:', imageUrl);-- // For the artifact, we need a data URL (same as OpenAI)- // The local file save is for persistence, but the response needs a data URLconst dataUrl = `data:${mimeType};base64,${imageData}`;-- // Return in content_and_artifact format (same as OpenAI)const file_ids = [v4()];const content = [{@@ -567,8 +444,7 @@ function createGeminiImageTool(fields = {}) {},];- // Record token usage for balance tracking (don't await to avoid blocking response)- const conversationId = _runnableConfig?.configurable?.thread_id;+ const conversationId = runnableConfig?.configurable?.thread_id;recordTokenUsage({usageMetadata: apiResponse.usageMetadata,req,diff --git a/api/app/clients/tools/util/handleTools.js b/api/app/clients/tools/util/handleTools.jsindex 65c88ce83f..48fcf7cb83 100644--- a/api/app/clients/tools/util/handleTools.js+++ b/api/app/clients/tools/util/handleTools.js@@ -207,7 +207,7 @@ const loadTools = async ({},gemini_image_gen: async (toolContextMap) => {const authFields = getAuthFields('gemini_image_gen');- const authValues = await loadAuthValues({ userId: user, authFields });+ const authValues = await loadAuthValues({ userId: user, authFields, throwError: false });const imageFiles = options.tool_resources?.[EToolResources.image_edit]?.files ?? [];const toolContext = buildImageToolContext({imageFiles,@@ -222,7 +222,6 @@ const loadTools = async ({isAgent: !!agent,req: options.req,imageFiles,- processFileURL: options.processFileURL,userId: user,fileStrategy,});diff --git a/client/src/components/Plugins/Store/PluginAuthForm.tsx b/client/src/components/Plugins/Store/PluginAuthForm.tsxindex 5af1948c11..d304b2eab7 100644--- a/client/src/components/Plugins/Store/PluginAuthForm.tsx+++ b/client/src/components/Plugins/Store/PluginAuthForm.tsx@@ -20,6 +20,7 @@ function PluginAuthForm({ plugin, onSubmit, isEntityTool }: TPluginAuthFormPropsconst localize = useLocalize();const authConfig = plugin?.authConfig ?? [];+ const allFieldsOptional = authConfig.length > 0 && authConfig.every((c) => c.optional === true);return (<div className="flex w-full flex-col items-center gap-2">@@ -38,6 +39,7 @@ function PluginAuthForm({ plugin, onSubmit, isEntityTool }: TPluginAuthFormProps>{authConfig.map((config: TPluginAuthConfig, i: number) => {const authField = config.authField.split('||')[0];+ const isOptional = config.optional === true;return (<div key={`${authField}-${i}`} className="flex w-full flex-col gap-1"><label@@ -55,19 +57,24 @@ function PluginAuthForm({ plugin, onSubmit, isEntityTool }: TPluginAuthFormPropsaria-invalid={!!errors[authField]}aria-describedby={`${authField}-error`}aria-label={config.label}- aria-required="true"+ aria-required={!isOptional}/* autoFocus is generally disabled due to the fact that it can disorient users,* but in this case, the required field must be navigated to anyways, and the component's functionality* emulates that of a new modal opening, where users would expect focus to be shifted to the new content */// eslint-disable-next-line jsx-a11y/no-autofocusautoFocus={i === 0}- {...register(authField, {- required: `${config.label} is required.`,- minLength: {- value: 1,- message: `${config.label} must be at least 1 character long`,- },- })}+ {...register(+ authField,+ isOptional+ ? {}+ : {+ required: `${config.label} is required.`,+ minLength: {+ value: 1,+ message: `${config.label} must be at least 1 character long`,+ },+ },+ )}className="flex h-10 max-h-10 w-full resize-none rounded-md border border-gray-200 bg-transparent px-3 py-2 text-sm text-gray-700 shadow-[0_0_10px_rgba(0,0,0,0.05)] outline-none placeholder:text-gray-400 focus:border-gray-400 focus:bg-gray-50 focus:outline-none focus:ring-0 focus:ring-gray-400 focus:ring-opacity-0 focus:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-50 dark:border-gray-500 dark:bg-gray-700 dark:text-gray-50 dark:shadow-[0_0_15px_rgba(0,0,0,0.10)] dark:focus:border-gray-400 focus:dark:bg-gray-600 dark:focus:outline-none dark:focus:ring-0 dark:focus:ring-gray-400 dark:focus:ring-offset-0"/></HoverCardTrigger>@@ -82,7 +89,7 @@ function PluginAuthForm({ plugin, onSubmit, isEntityTool }: TPluginAuthFormProps);})}<button- disabled={!isDirty || !isValid || isSubmitting}+ disabled={allFieldsOptional ? isSubmitting : !isDirty || !isValid || isSubmitting}type="button"className="btn btn-primary relative"onClick={() => {diff --git a/packages/api/src/tools/format.spec.ts b/packages/api/src/tools/format.spec.tsindex 2119cd724d..68a32090e9 100644--- a/packages/api/src/tools/format.spec.ts+++ b/packages/api/src/tools/format.spec.ts@@ -189,6 +189,77 @@ describe('format.ts helper functions', () => {const result = checkPluginAuth(plugin);expect(result).toBe(true);});++ it('should return true when auth field is marked as optional with no env vars', () => {+ const plugin: TPlugin = {+ name: 'Test',+ pluginKey: 'test',+ description: 'Test plugin',+ authConfig: [+ { authField: 'MISSING_KEY', label: 'API Key', description: 'API Key', optional: true },+ ],+ };++ const result = checkPluginAuth(plugin);+ expect(result).toBe(true);+ });++ it('should return true when all auth fields are optional', () => {+ const plugin: TPlugin = {+ name: 'Test',+ pluginKey: 'test',+ description: 'Test plugin',+ authConfig: [+ { authField: 'MISSING_KEY_A', label: 'Key A', description: 'Key A', optional: true },+ { authField: 'MISSING_KEY_B', label: 'Key B', description: 'Key B', optional: true },+ ],+ };++ const result = checkPluginAuth(plugin);+ expect(result).toBe(true);+ });++ it('should return false when required field is missing even if optional field exists', () => {+ const plugin: TPlugin = {+ name: 'Test',+ pluginKey: 'test',+ description: 'Test plugin',+ authConfig: [+ { authField: 'MISSING_KEY', label: 'Required Key', description: 'Required' },+ {+ authField: 'OPTIONAL_KEY',+ label: 'Optional Key',+ description: 'Optional',+ optional: true,+ },+ ],+ };++ const result = checkPluginAuth(plugin);+ expect(result).toBe(false);+ });++ it('should return true when optional field has no env var but required field is satisfied', () => {+ process.env.REQUIRED_KEY = 'valid-key';++ const plugin: TPlugin = {+ name: 'Test',+ pluginKey: 'test',+ description: 'Test plugin',+ authConfig: [+ { authField: 'REQUIRED_KEY', label: 'Required Key', description: 'Required' },+ {+ authField: 'OPTIONAL_KEY',+ label: 'Optional Key',+ description: 'Optional',+ optional: true,+ },+ ],+ };++ const result = checkPluginAuth(plugin);+ expect(result).toBe(true);+ });});describe('getToolkitKey', () => {diff --git a/packages/api/src/tools/format.ts b/packages/api/src/tools/format.tsindex 2525743ff2..feab29461e 100644--- a/packages/api/src/tools/format.ts+++ b/packages/api/src/tools/format.ts@@ -31,6 +31,10 @@ export const checkPluginAuth = (plugin?: TPlugin): boolean => {}return plugin.authConfig.every((authFieldObj) => {+ if (authFieldObj.optional === true) {+ return true;+ }+const authFieldOptions = authFieldObj.authField.split('||');let isFieldAuthenticated = false;diff --git a/packages/data-provider/src/schemas.ts b/packages/data-provider/src/schemas.tsindex 60a20ecae7..90d5362273 100644--- a/packages/data-provider/src/schemas.ts+++ b/packages/data-provider/src/schemas.ts@@ -544,6 +544,7 @@ export const tPluginAuthConfigSchema = z.object({authField: z.string(),label: z.string(),description: z.string(),+ optional: z.boolean().optional(),});export type TPluginAuthConfig = z.infer<typeof tPluginAuthConfigSchema>;