OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
feat: docling ingest vlms (#1841)
Edwin Joseb0ead4af5795
diff --git a/flows/components/export_docling_document.py b/flows/components/export_docling_document.pyindex 727534435..14c83d41b 100644--- a/flows/components/export_docling_document.py+++ b/flows/components/export_docling_document.py@@ -157,11 +157,18 @@ class ExportDoclingDocumentComponent(Component):continue# Fall back to whole-document export (no page granularity)- content = doc.export_to_markdown(- image_mode=image_mode,- image_placeholder=self.md_image_placeholder,- page_break_placeholder=self.md_page_break_placeholder,- )+ try:+ content = doc.export_to_markdown(+ image_mode=image_mode,+ image_placeholder=self.md_image_placeholder,+ page_break_placeholder=self.md_page_break_placeholder,+ )+ except TypeError:+ # Older docling-core versions lack page_break_placeholder+ content = doc.export_to_markdown(+ image_mode=image_mode,+ image_placeholder=self.md_image_placeholder,+ )elif self.export_format == "HTML":content = doc.export_to_html(image_mode=image_mode)elif self.export_format == "Plaintext":diff --git a/frontend/app/api/mutations/useUpdateSettingsMutation.ts b/frontend/app/api/mutations/useUpdateSettingsMutation.tsindex d2c9130e4..8744bfc54 100644--- a/frontend/app/api/mutations/useUpdateSettingsMutation.ts+++ b/frontend/app/api/mutations/useUpdateSettingsMutation.ts@@ -22,6 +22,17 @@ export interface UpdateSettingsRequest {embedding_model?: string;embedding_provider?: string;+ // Docling VLM pipeline settings+ vlm_enabled?: boolean;+ vlm_provider?: string;+ vlm_model?: string;+ vlm_prompt?: string;+ vlm_response_format?: string;+ vlm_max_tokens?: number;+ vlm_concurrency?: number;+ vlm_timeout?: number;+ vlm_watsonx_api_version?: string;+// Provider-specific settings (for dialogs)model_provider?: string; // Deprecated, kept for backward compatibilityapi_key?: string;diff --git a/frontend/app/api/queries/useGetModelsQuery.ts b/frontend/app/api/queries/useGetModelsQuery.tsindex aaae1f76d..dfe92be37 100644--- a/frontend/app/api/queries/useGetModelsQuery.ts+++ b/frontend/app/api/queries/useGetModelsQuery.ts@@ -9,6 +9,7 @@ export interface ModelOption {value: string;label: string;default?: boolean;+ supports_images?: boolean;}export interface ModelsResponse {diff --git a/frontend/app/api/queries/useGetSearchQuery.ts b/frontend/app/api/queries/useGetSearchQuery.tsindex 33b98d3af..9245d0915 100644--- a/frontend/app/api/queries/useGetSearchQuery.ts+++ b/frontend/app/api/queries/useGetSearchQuery.ts@@ -36,6 +36,8 @@ export interface ChunkResult {parser?: string;chunk_size?: number;chunk_overlap?: number;+ chunk_id?: string;+ id?: string;index?: number;allowed_users?: string[];allowed_groups?: string[];diff --git a/frontend/app/api/queries/useGetSettingsQuery.ts b/frontend/app/api/queries/useGetSettingsQuery.tsindex c51e16816..c06a15ca9 100644--- a/frontend/app/api/queries/useGetSettingsQuery.ts+++ b/frontend/app/api/queries/useGetSettingsQuery.ts@@ -20,6 +20,15 @@ export interface KnowledgeSettings {ocr?: boolean;picture_descriptions?: boolean;disable_ingest_with_langflow?: boolean;+ vlm_enabled?: boolean;+ vlm_provider?: string;+ vlm_model?: string;+ vlm_prompt?: string;+ vlm_response_format?: string;+ vlm_max_tokens?: number;+ vlm_concurrency?: number;+ vlm_timeout?: number;+ vlm_watsonx_api_version?: string;}export interface ProviderSettings {@@ -41,6 +50,9 @@ export interface ProviderSettings {endpoint?: string;configured?: boolean;};+ local?: {+ configured?: boolean;+ };}export interface OnboardingState {@@ -79,6 +91,8 @@ export interface Settings {localhost_url?: string;ingest_via_chat?: boolean;show_provider_ingest_settings?: boolean;+ show_vlm_settings?: boolean;+ local_vlm_models?: string[];show_shared_upload_toggle?: boolean;show_workspace_oauth_overrides?: boolean;segment_write_key?: string;diff --git a/frontend/app/settings/_components/ingest-settings-section.tsx b/frontend/app/settings/_components/ingest-settings-section.tsxindex 2a896be32..97ec84fe8 100644--- a/frontend/app/settings/_components/ingest-settings-section.tsx+++ b/frontend/app/settings/_components/ingest-settings-section.tsx@@ -4,6 +4,7 @@ import { ArrowUpRight, Loader2, Minus, Plus } from "lucide-react";import { useCallback, useEffect, useMemo, useRef, useState } from "react";import { toast } from "sonner";import {+ useGetAnthropicModelsQuery,useGetIBMModelsQuery,useGetOllamaModelsQuery,useGetOpenAIModelsQuery,@@ -12,6 +13,12 @@ import { useGetSettingsQuery } from "@/app/api/queries/useGetSettingsQuery";import { ConfirmationDialog } from "@/components/confirmation-dialog";import { LabelWrapper } from "@/components/label-wrapper";import { RequirePermission } from "@/components/require-permission";+import {+ Accordion,+ AccordionContent,+ AccordionItem,+ AccordionTrigger,+} from "@/components/ui/accordion";import { Button } from "@/components/ui/button";import {Card,@@ -21,8 +28,17 @@ import {CardTitle,} from "@/components/ui/card";import { Input } from "@/components/ui/input";+import { NumberInput } from "@/components/ui/inputs/number-input";import { Label } from "@/components/ui/label";+import {+ Select,+ SelectContent,+ SelectItem,+ SelectTrigger,+ SelectValue,+} from "@/components/ui/select";import { Switch } from "@/components/ui/switch";+import { Textarea } from "@/components/ui/textarea";import { useAuth } from "@/contexts/auth-context";import { useIsCloudBrand } from "@/contexts/brand-context";import { trackButton } from "@/lib/analytics";@@ -34,6 +50,14 @@ import { ModelSelector } from "../../onboarding/_components/model-selector";import { getModelLogo } from "../_helpers/model-helpers";import { LangflowIcon } from "./langflow-icon";+const DEFAULT_WATSONX_API_VERSION = "2023-05-29";++const RESPONSE_FORMATS = [+ { value: "markdown", label: "Markdown (recommended)" },+ { value: "doctags", label: "DocTags" },+ { value: "html", label: "HTML" },+] as const;+export function IngestSettingsSection() {const isCloudBrand = useIsCloudBrand();const { isAuthenticated, isNoAuthMode, isIbmAuthMode, runMode } = useAuth();@@ -52,15 +76,35 @@ export function IngestSettingsSection() {const [disableIngestWithLangflow, setDisableIngestWithLangflow] =useState<boolean>(false);+ const [vlmProvider, setVlmProvider] = useState<string>("openai");+ const [vlmModel, setVlmModel] = useState<string>("");+ const [vlmPrompt, setVlmPrompt] = useState<string>("");+ const [vlmResponseFormat, setVlmResponseFormat] =+ useState<string>("markdown");+ const [vlmMaxTokens, setVlmMaxTokens] = useState<number>(5000);+ const [vlmConcurrency, setVlmConcurrency] = useState<number>(4);+ const [vlmTimeout, setVlmTimeout] = useState<number>(120);+ const [vlmWatsonxApiVersion, setVlmWatsonxApiVersion] = useState<string>(+ DEFAULT_WATSONX_API_VERSION,+ );+ const [validationError, setValidationError] = useState<string | null>(null);+const { data: settings = {} } = useGetSettingsQuery({enabled: isAuthenticated || isNoAuthMode,});+ const showVlmSettings = settings.show_vlm_settings ?? true;+const { data: openaiModels, isLoading: openaiLoading } =useGetOpenAIModelsQuery({ apiKey: "" },{ enabled: settings?.providers?.openai?.configured === true },);+ const { data: anthropicModels, isLoading: anthropicLoading } =+ useGetAnthropicModelsQuery(+ { apiKey: "" },+ { enabled: settings?.providers?.anthropic?.configured === true },+ );const { data: ollamaModels, isLoading: ollamaLoading } =useGetOllamaModelsQuery({ endpoint: settings?.providers?.ollama?.endpoint },@@ -129,6 +173,99 @@ export function IngestSettingsSection() {const isLoadingAnyEmbeddingModels =openaiLoading || ollamaLoading || watsonxLoading;+ const groupedVlmModels = useMemo(() => {+ const list: any[] = [];++ // 1. Local Models+ if (settings.local_vlm_models && settings.local_vlm_models.length > 0) {+ list.push({+ group: "Local Models",+ icon: getModelLogo("", "local"),+ options: settings.local_vlm_models.map((m: string) => ({+ value: m,+ label: m.split("/").pop(),+ provider: "local",+ })),+ });+ }++ // 2. OpenAI+ if (settings.providers?.openai?.configured) {+ const models = (openaiModels?.language_models || [])+ .filter((m: any) => m.supports_images === true)+ .map((m: any) => ({ ...m, provider: "openai" }));+ if (models.length > 0) {+ list.push({+ group: "OpenAI",+ icon: getModelLogo("", "openai"),+ options: models,+ });+ }+ }++ // 3. Anthropic+ if (settings.providers?.anthropic?.configured) {+ const models = (anthropicModels?.language_models || [])+ .filter((m: any) => m.supports_images === true)+ .map((m: any) => ({ ...m, provider: "anthropic" }));+ if (models.length > 0) {+ list.push({+ group: "Anthropic",+ icon: getModelLogo("", "anthropic"),+ options: models,+ });+ }+ }++ // 4. Ollama+ if (settings.providers?.ollama?.configured) {+ const models = (ollamaModels?.language_models || [])+ .filter((m: any) => m.supports_images === true)+ .map((m: any) => ({ ...m, provider: "ollama" }));+ if (models.length > 0) {+ list.push({+ group: "Ollama",+ icon: getModelLogo("", "ollama"),+ options: models,+ });+ }+ }++ // 5. IBM watsonx.ai+ if (settings.providers?.watsonx?.configured) {+ const models = (watsonxModels?.language_models || [])+ .filter((m: any) => m.supports_images === true)+ .map((m: any) => ({ ...m, provider: "watsonx" }));+ if (models.length > 0) {+ list.push({+ group: "IBM watsonx.ai",+ icon: getModelLogo("", "watsonx"),+ options: models,+ });+ }+ }++ return list;+ }, [+ settings.local_vlm_models,+ settings.providers?.openai?.configured,+ settings.providers?.anthropic?.configured,+ settings.providers?.ollama?.configured,+ settings.providers?.watsonx?.configured,+ openaiModels?.language_models,+ anthropicModels?.language_models,+ ollamaModels?.language_models,+ watsonxModels?.language_models,+ ]);++ const isLoadingAnyVlmModels =+ openaiLoading || anthropicLoading || ollamaLoading || watsonxLoading;++ const allVlmOptions = useMemo(+ () => groupedVlmModels.flatMap((g) => g.options),+ [groupedVlmModels],+ );+const updateSettingsMutation = useUpdateSettingsMutation({onSuccess: () => {toast.success("Settings updated successfully");@@ -177,37 +314,80 @@ export function IngestSettingsSection() {]);useEffect(() => {- if (settings.knowledge?.chunk_size !== undefined)- setChunkSize(settings.knowledge.chunk_size);- }, [settings.knowledge?.chunk_size]);+ const k = settings.knowledge;+ if (!k) return;+ if (k.chunk_size !== undefined) setChunkSize(k.chunk_size);+ if (k.chunk_overlap !== undefined) setChunkOverlap(k.chunk_overlap);+ if (k.table_structure !== undefined) setTableStructure(k.table_structure);+ if (k.ocr !== undefined) setOcr(k.ocr);+ if (k.picture_descriptions !== undefined)+ setPictureDescriptions(k.picture_descriptions);+ if (k.disable_ingest_with_langflow !== undefined)+ setDisableIngestWithLangflow(k.disable_ingest_with_langflow);+ if (k.vlm_provider !== undefined) setVlmProvider(k.vlm_provider);+ // Backend defaults vlm_model to ""; an empty value means "not configured",+ // so don't clobber a locally auto-selected model with it.+ if (k.vlm_model) setVlmModel(k.vlm_model);+ if (k.vlm_prompt !== undefined) setVlmPrompt(k.vlm_prompt);+ if (k.vlm_response_format !== undefined)+ setVlmResponseFormat(k.vlm_response_format);+ if (k.vlm_max_tokens !== undefined) setVlmMaxTokens(k.vlm_max_tokens);+ if (k.vlm_concurrency !== undefined) setVlmConcurrency(k.vlm_concurrency);+ if (k.vlm_timeout !== undefined) setVlmTimeout(k.vlm_timeout);+ if (k.vlm_watsonx_api_version !== undefined)+ setVlmWatsonxApiVersion(k.vlm_watsonx_api_version);+ }, [settings.knowledge]);- useEffect(() => {- if (settings.knowledge?.chunk_overlap !== undefined)- setChunkOverlap(settings.knowledge.chunk_overlap);- }, [settings.knowledge?.chunk_overlap]);+ const [vlmAccordionValue, setVlmAccordionValue] = useState<string>("");+ const autoSelectedVlm = useRef(false);useEffect(() => {- if (settings.knowledge?.table_structure !== undefined)- setTableStructure(settings.knowledge.table_structure);- }, [settings.knowledge?.table_structure]);-- useEffect(() => {- if (settings.knowledge?.ocr !== undefined) setOcr(settings.knowledge.ocr);- }, [settings.knowledge?.ocr]);+ if (!showVlmSettings) return;+ if (settings.knowledge?.vlm_model) {+ autoSelectedVlm.current = false;+ return;+ }+ if (autoSelectedVlm.current) return;+ if (settings.local_vlm_models && settings.local_vlm_models.length > 0) {+ setVlmModel(settings.local_vlm_models[0]);+ setVlmProvider("local");+ autoSelectedVlm.current = true;+ } else if (allVlmOptions.length > 0) {+ const fallback = allVlmOptions.find((o) => o.default) || allVlmOptions[0];+ setVlmModel(fallback.value);+ setVlmProvider(fallback.provider || "openai");+ autoSelectedVlm.current = true;+ }+ }, [+ showVlmSettings,+ settings.knowledge?.vlm_model,+ settings.local_vlm_models,+ allVlmOptions,+ ]);- useEffect(() => {- if (settings.knowledge?.picture_descriptions !== undefined)- setPictureDescriptions(settings.knowledge.picture_descriptions);- }, [settings.knowledge?.picture_descriptions]);+ const handleVlmModelChange = (value: string, provider?: string) => {+ setVlmModel(value);+ if (provider) setVlmProvider(provider);+ setValidationError(null);+ };- useEffect(() => {- if (settings.knowledge?.disable_ingest_with_langflow !== undefined)- setDisableIngestWithLangflow(- settings.knowledge.disable_ingest_with_langflow,- );- }, [settings.knowledge?.disable_ingest_with_langflow]);+ const vlmModelPending =+ showVlmSettings && pictureDescriptions && !vlmModel.trim();const k = settings.knowledge;+ const vlmDirty =+ showVlmSettings &&+ (pictureDescriptions !== (k?.vlm_enabled ?? pictureDescriptions) ||+ vlmProvider !== (k?.vlm_provider ?? vlmProvider) ||+ vlmModel !== (k?.vlm_model ?? vlmModel) ||+ vlmPrompt !== (k?.vlm_prompt ?? vlmPrompt) ||+ vlmResponseFormat !== (k?.vlm_response_format ?? vlmResponseFormat) ||+ vlmMaxTokens !== (k?.vlm_max_tokens ?? vlmMaxTokens) ||+ vlmConcurrency !== (k?.vlm_concurrency ?? vlmConcurrency) ||+ vlmTimeout !== (k?.vlm_timeout ?? vlmTimeout) ||+ vlmWatsonxApiVersion !==+ (k?.vlm_watsonx_api_version ?? vlmWatsonxApiVersion));+const knowledgeIngestDirty =chunkSize !== (k?.chunk_size ?? chunkSize) ||chunkOverlap !== (k?.chunk_overlap ?? chunkOverlap) ||@@ -215,7 +395,31 @@ export function IngestSettingsSection() {ocr !== (k?.ocr ?? ocr) ||pictureDescriptions !== (k?.picture_descriptions ?? pictureDescriptions) ||disableIngestWithLangflow !==- (k?.disable_ingest_with_langflow ?? disableIngestWithLangflow);+ (k?.disable_ingest_with_langflow ?? disableIngestWithLangflow) ||+ vlmDirty;++ const providerConfigured =+ settings.providers === undefined || settings.providers === null+ ? undefined+ : vlmProvider === "watsonx"+ ? settings.providers.watsonx?.configured === true+ : vlmProvider === "anthropic"+ ? settings.providers.anthropic?.configured === true+ : vlmProvider === "ollama"+ ? settings.providers.ollama?.configured === true+ : vlmProvider === "local"+ ? true+ : settings.providers.openai?.configured === true;++ const providerWarning = pictureDescriptions && providerConfigured === false;+ const providerLabel =+ vlmProvider === "watsonx"+ ? "IBM watsonx.ai"+ : vlmProvider === "anthropic"+ ? "Anthropic"+ : vlmProvider === "ollama"+ ? "Ollama"+ : "OpenAI";const handleChunkSizeChange = (value: string) => {setChunkSize(Math.max(0, Number.parseInt(value, 10) || 0));@@ -228,6 +432,22 @@ export function IngestSettingsSection() {};const handleKnowledgeIngestSave = () => {+ // Only include VLM fields when the VLM UI is enabled; a hidden section+ // must not drive backend VLM state or trip its validation.+ const vlmPayload = showVlmSettings+ ? {+ vlm_enabled: pictureDescriptions,+ vlm_provider: vlmProvider,+ vlm_model: vlmModel.trim() || undefined,+ vlm_prompt: vlmPrompt,+ vlm_response_format: vlmResponseFormat,+ vlm_max_tokens: vlmMaxTokens,+ vlm_concurrency: vlmConcurrency,+ vlm_timeout: vlmTimeout,+ vlm_watsonx_api_version: vlmWatsonxApiVersion,+ }+ : {};+trackButton({CTA: "Save Ingest Settings",elementId: "save-ingest-settings-button",@@ -239,8 +459,10 @@ export function IngestSettingsSection() {ocr,picture_descriptions: pictureDescriptions,disable_ingest_with_langflow: disableIngestWithLangflow,+ ...vlmPayload,},});+if (chunkSize < 1) {const msg = "Chunk size must be at least 1";setChunkValidationError(msg);@@ -253,6 +475,23 @@ export function IngestSettingsSection() {toast.error("Could not save ingest settings", { description: msg });return;}++ if (showVlmSettings && pictureDescriptions) {+ if (!vlmModel.trim()) {+ const msg =+ "Model name is required when picture descriptions are enabled";+ setValidationError(msg);+ toast.error("Could not save ingest settings", { description: msg });+ return;+ }+ if (vlmMaxTokens < 1 || vlmConcurrency < 1 || vlmTimeout < 1) {+ const msg = "Max tokens, concurrency, and timeout must be at least 1";+ setValidationError(msg);+ toast.error("Could not save ingest settings", { description: msg });+ return;+ }+ }+updateSettingsMutation.mutate({chunk_size: chunkSize,@@ -261,8 +500,14 @@ export function IngestSettingsSection() {ocr,picture_descriptions: pictureDescriptions,disable_ingest_with_langflow: disableIngestWithLangflow,+ ...vlmPayload,+ },+ {+ onSuccess: () => {+ setChunkValidationError(null);+ setValidationError(null);+ },},- { onSuccess: () => setChunkValidationError(null) },);};@@ -579,12 +824,171 @@ export function IngestSettingsSection() {onCheckedChange={setPictureDescriptions}/></div>+ {showVlmSettings && (+ <div+ className={cn(+ "mt-4 border border-border rounded-lg bg-muted/5 overflow-hidden transition-all duration-200",+ !pictureDescriptions &&+ "opacity-50 cursor-not-allowed select-none",+ )}+ >+ <Accordion+ type="single"+ collapsible+ disabled={!pictureDescriptions}+ value={pictureDescriptions ? vlmAccordionValue : ""}+ onValueChange={setVlmAccordionValue}+ >+ <AccordionItem value="vlm-settings" className="border-none">+ <AccordionTrigger+ className={cn(+ "hover:no-underline font-medium text-foreground px-4 py-3 bg-muted/10 border-b border-border",+ !pictureDescriptions && "pointer-events-none",+ )}+ >+ Advanced Vision Model (VLM) Settings+ </AccordionTrigger>+ <AccordionContent className="p-4 space-y-6">+ <div className="space-y-2">+ <LabelWrapper+ id="vlm-model"+ label="Vision model"+ helperText="Pick a vision-capable model; the provider is set from your selection"+ required={pictureDescriptions}+ >+ <ModelSelector+ groupedOptions={groupedVlmModels}+ noOptionsPlaceholder={+ isLoadingAnyVlmModels+ ? "Loading models..."+ : "No models detected. Configure OpenAI, Anthropic, Ollama, or IBM watsonx.ai first."+ }+ value={vlmModel}+ onValueChange={handleVlmModelChange}+ hasError={!!validationError}+ />+ </LabelWrapper>+ {providerWarning && (+ <p className="text-sm text-destructive" role="alert">+ {providerLabel} is not configured. Configure it in+ Settings > Providers first.+ </p>+ )}+ </div>++ {vlmProvider === "watsonx" && (+ <div className="space-y-2">+ <LabelWrapper+ id="vlm-watsonx-api-version"+ label="watsonx API version"+ helperText="API version date sent to watsonx.ai"+ >+ <Input+ id="vlm-watsonx-api-version"+ type="text"+ placeholder={DEFAULT_WATSONX_API_VERSION}+ value={vlmWatsonxApiVersion}+ onChange={(e) =>+ setVlmWatsonxApiVersion(e.target.value)+ }+ />+ </LabelWrapper>+ </div>+ )}++ <div className="space-y-2">+ <LabelWrapper+ id="vlm-prompt"+ label="Prompt"+ helperText="Sent to the VLM for every page"+ >+ <Textarea+ id="vlm-prompt"+ rows={3}+ value={vlmPrompt}+ onChange={(e) => setVlmPrompt(e.target.value)}+ />+ </LabelWrapper>+ </div>++ <div className="space-y-2">+ <LabelWrapper+ id="vlm-response-format"+ label="Response format"+ helperText="Per-page VLM output. Markdown is compatible with the existing pipeline; the final document is always Docling JSON."+ >+ <Select+ value={vlmResponseFormat}+ onValueChange={setVlmResponseFormat}+ >+ <SelectTrigger id="vlm-response-format">+ <SelectValue placeholder="Select a format" />+ </SelectTrigger>+ <SelectContent>+ {RESPONSE_FORMATS.map((format) => (+ <SelectItem+ key={format.value}+ value={format.value}+ >+ {format.label}+ </SelectItem>+ ))}+ </SelectContent>+ </Select>+ </LabelWrapper>+ </div>++ <div className="grid grid-cols-3 gap-4">+ <NumberInput+ id="vlm-max-tokens"+ label="Max tokens per page"+ value={vlmMaxTokens}+ onChange={(value) =>+ setVlmMaxTokens(Math.max(1, value))+ }+ unit="tokens"+ min={1}+ />+ <NumberInput+ id="vlm-concurrency"+ label="Concurrency"+ value={vlmConcurrency}+ onChange={(value) =>+ setVlmConcurrency(Math.max(1, value))+ }+ unit="requests"+ min={1}+ />+ <NumberInput+ id="vlm-timeout"+ label="API timeout"+ value={vlmTimeout}+ onChange={(value) =>+ setVlmTimeout(Math.max(1, value))+ }+ unit="seconds"+ min={1}+ />+ </div>++ {validationError && (+ <p className="text-sm text-destructive" role="alert">+ {validationError}+ </p>+ )}+ </AccordionContent>+ </AccordionItem>+ </Accordion>+ </div>+ )}</div><div className="flex justify-end pt-2"><ButtononClick={handleKnowledgeIngestSave}disabled={- updateSettingsMutation.isPending || !knowledgeIngestDirty+ updateSettingsMutation.isPending ||+ !knowledgeIngestDirty ||+ vlmModelPending}className="min-w-[120px]"size="sm"diff --git a/frontend/app/settings/_components/model-providers.tsx b/frontend/app/settings/_components/model-providers.tsxindex ef1ceb80c..b5cc970de 100644--- a/frontend/app/settings/_components/model-providers.tsx+++ b/frontend/app/settings/_components/model-providers.tsx@@ -94,6 +94,34 @@ export const ModelProviders = () => {logoColor: "text-white",logoBgColor: "bg-[#1063FE]",},+ local: {+ name: "Local",+ logo: (props) => (+ <svg+ xmlns="http://www.w3.org/2000/svg"+ viewBox="0 0 24 24"+ fill="none"+ stroke="currentColor"+ strokeWidth="2"+ strokeLinecap="round"+ strokeLinejoin="round"+ {...props}+ >+ <rect x="4" y="4" width="16" height="16" rx="2" />+ <rect x="9" y="9" width="6" height="6" />+ <path d="M9 1v3" />+ <path d="M15 1v3" />+ <path d="M9 20v3" />+ <path d="M15 20v3" />+ <path d="M20 9h3" />+ <path d="M20 15h3" />+ <path d="M1 9h3" />+ <path d="M1 15h3" />+ </svg>+ ),+ logoColor: "text-muted-foreground",+ logoBgColor: "bg-white",+ },};const currentLlmProvider =diff --git a/frontend/app/settings/_helpers/model-helpers.tsx b/frontend/app/settings/_helpers/model-helpers.tsxindex 216b621b7..fac3d2ee0 100644--- a/frontend/app/settings/_helpers/model-helpers.tsx+++ b/frontend/app/settings/_helpers/model-helpers.tsx@@ -3,7 +3,12 @@ import IBMLogo from "@/components/icons/ibm-logo";import OllamaLogo from "@/components/icons/ollama-logo";import OpenAILogo from "@/components/icons/openai-logo";-export type ModelProvider = "openai" | "anthropic" | "ollama" | "watsonx";+export type ModelProvider =+ | "openai"+ | "anthropic"+ | "ollama"+ | "watsonx"+ | "local";// Full ordered list of providers for settings / cardsexport const ALL_PROVIDERS: ModelProvider[] = [@@ -47,6 +52,30 @@ export function getModelLogo(modelValue: string, provider?: ModelProvider) {return <OllamaLogo className="w-4 h-4" />;} else if (provider === "watsonx") {return <IBMLogo className="w-4 h-4" />;+ } else if (provider === "local") {+ return (+ <svg+ xmlns="http://www.w3.org/2000/svg"+ viewBox="0 0 24 24"+ fill="none"+ stroke="currentColor"+ strokeWidth="2"+ strokeLinecap="round"+ strokeLinejoin="round"+ className="w-4 h-4 text-muted-foreground"+ >+ <rect x="4" y="4" width="16" height="16" rx="2" />+ <rect x="9" y="9" width="6" height="6" />+ <path d="M9 1v3" />+ <path d="M15 1v3" />+ <path d="M9 20v3" />+ <path d="M15 20v3" />+ <path d="M20 9h3" />+ <path d="M20 15h3" />+ <path d="M1 9h3" />+ <path d="M1 15h3" />+ </svg>+ );}// Fallback to model name analysisdiff --git a/frontend/components/provider-health-banner.tsx b/frontend/components/provider-health-banner.tsxindex 6061a7f9d..722cb8ed9 100644--- a/frontend/components/provider-health-banner.tsx+++ b/frontend/components/provider-health-banner.tsx@@ -51,6 +51,7 @@ const providerTitleMap: Record<ModelProvider, string> = {anthropic: "Anthropic",ollama: "Ollama",watsonx: "IBM watsonx.ai",+ local: "Local",};export function ProviderHealthBanner({ className }: ProviderHealthBannerProps) {diff --git a/frontend/lib/chat-stream-parsers.ts b/frontend/lib/chat-stream-parsers.tsindex 6a27427cf..ab9a8d580 100644--- a/frontend/lib/chat-stream-parsers.ts+++ b/frontend/lib/chat-stream-parsers.ts@@ -36,8 +36,9 @@ function findFunctionCallByEvent(): FunctionCall | undefined {const itemId = c.item_id || c.call_id || c.id;if (typeof itemId === "string") {- const match = calls.find((fc) => fc.id === itemId);- if (match) return match;+ // The event names a specific call; if we don't know it yet, let the+ // caller create it rather than guessing an existing one.+ return calls.find((fc) => fc.id === itemId);}const outputIndex =@@ -50,7 +51,9 @@ function findFunctionCallByEvent(return calls[outputIndex];}- return calls[calls.length - 1];+ // No identifying info on the event: continue an in-progress call that+ // hasn't been assigned an id yet, if there is one.+ return [...calls].reverse().find((fc) => fc.status === "pending" && !fc.id);}export function parseOpenAIChatChunk(diff --git a/kubernetes/helm/openrag/values.yaml b/kubernetes/helm/openrag/values.yamlindex ecdcb5b23..800bf8be3 100644--- a/kubernetes/helm/openrag/values.yaml+++ b/kubernetes/helm/openrag/values.yaml@@ -34,6 +34,11 @@ global:port: 5001scheme: "http"verifySsl: false+ # Note: this chart does not deploy docling-serve. To use OpenRAG's Docling+ # VLM pipeline (remote OpenAI/watsonx vision models), the external+ # docling-serve deployment must set:+ # DOCLING_SERVE_ENABLE_REMOTE_SERVICES=true+ # DOCLING_SERVE_ALLOW_CUSTOM_VLM_CONFIG=true# Shared OAuth credentials (same across all tenants)oauth:diff --git a/kubernetes/operator/config/samples/docling-complete.yaml b/kubernetes/operator/config/samples/docling-complete.yamlindex b170558cc..6bdb3bf33 100644--- a/kubernetes/operator/config/samples/docling-complete.yaml+++ b/kubernetes/operator/config/samples/docling-complete.yaml@@ -73,6 +73,16 @@ spec:serviceAnnotations:prometheus.io/scrape: "true"prometheus.io/port: "5001"++ # Docling configuration for the serve API+ config:+ # Required for OpenRAG's Docling VLM pipeline (remote OpenAI/watsonx+ # vision models). Harmless when VLM is not used.+ extraEnv:+ - name: "DOCLING_SERVE_ENABLE_REMOTE_SERVICES"+ value: "true"+ - name: "DOCLING_SERVE_ALLOW_CUSTOM_VLM_CONFIG"+ value: "true"# Node placementnodeSelector:diff --git a/kubernetes/operator/config/samples/openrag_v1alpha1_openrag.yaml b/kubernetes/operator/config/samples/openrag_v1alpha1_openrag.yamlindex a3365dbb6..9e91435b9 100644--- a/kubernetes/operator/config/samples/openrag_v1alpha1_openrag.yaml+++ b/kubernetes/operator/config/samples/openrag_v1alpha1_openrag.yaml@@ -105,6 +105,13 @@ spec:# app.kubernetes.io/component: "document-processor"# podAnnotations:# sidecar.istio.io/inject: "true"+ # config:+ # # Required for OpenRAG's Docling VLM pipeline (remote vision models)+ # extraEnv:+ # - name: "DOCLING_SERVE_ENABLE_REMOTE_SERVICES"+ # value: "true"+ # - name: "DOCLING_SERVE_ALLOW_CUSTOM_VLM_CONFIG"+ # value: "true"# hpa:# enabled: true# minReplicas: 1diff --git a/src/api/connectors.py b/src/api/connectors.pyindex c074b87f3..ba83bf843 100644--- a/src/api/connectors.py+++ b/src/api/connectors.py@@ -1,3 +1,4 @@+import copyfrom datetime import UTC, datetimefrom typing import Any@@ -675,20 +676,19 @@ async def _expand_selected_connector_files(expanded_files_info: list[dict[str, Any]] = []if file_ids and hasattr(connector, "cfg"):- original_file_ids = getattr(connector.cfg, "file_ids", None)- original_folder_ids = getattr(connector.cfg, "folder_ids", None)+ # The connector is cached and shared across requests by+ # ConnectionManager, so scope the listing to a per-call copy with its+ # own cfg instead of mutating (and racing on) the shared config.+ scoped_connector = copy.copy(connector)+ scoped_connector.cfg = copy.copy(connector.cfg)+ scoped_connector.cfg.file_ids = file_ids+ scoped_connector.cfg.folder_ids = Nonetry:- connector.cfg.file_ids = file_ids- connector.cfg.folder_ids = None-- result = await connector.list_files()+ result = await scoped_connector.list_files()for f in result.get("files", []):expanded_files_info.append(_connector_file_response(f))except Exception as e:logger.error("Failed to expand files in duplicate check", error=str(e))- finally:- connector.cfg.file_ids = original_file_ids- connector.cfg.folder_ids = original_folder_idsif not expanded_files_info:for f in selected_files_raw:@@ -774,6 +774,18 @@ async def _classify_connector_duplicates(}+def _connector_scoped_to_buckets(connector, bucket_names: list[str]):+ """Shallow copy of a (cached, shared) connector with bucket_names overridden.++ connector_service.get_connector() returns a per-connection cached instance;+ mutating bucket_names on it directly would leak the override into concurrent+ requests using the same connection.+ """+ scoped = copy.copy(connector)+ scoped.bucket_names = list(bucket_names)+ return scoped++async def _classify_bucket_connector_duplicates(connector,connector_type: str,@@ -790,19 +802,15 @@ async def _classify_bucket_connector_duplicates(``non_duplicate_files`` so the caller can sync just those when the userchooses to skip duplicates."""- original_buckets = connector.bucket_names- connector.bucket_names = bucket_filter- try:- all_files: list[dict[str, Any]] = []- page_token = None- while True:- result = await connector.list_files(page_token=page_token)- all_files.extend(result.get("files", []))- page_token = result.get("next_page_token")- if not page_token:- break- finally:- connector.bucket_names = original_buckets+ scoped_connector = _connector_scoped_to_buckets(connector, bucket_filter)+ all_files: list[dict[str, Any]] = []+ page_token = None+ while True:+ result = await scoped_connector.list_files(page_token=page_token)+ all_files.extend(result.get("files", []))+ page_token = result.get("next_page_token")+ if not page_token:+ breakif not all_files:return {@@ -1279,19 +1287,15 @@ async def connector_sync(# container every time. Per-file dedup in ConnectorFileProcessor is a# backstop, but it runs after download — this pre-filter avoids the# redundant fetch/reprocess and the misleading "all files" task view.- original_buckets = connector.bucket_names- connector.bucket_names = body.bucket_filter- try:- all_files: list[dict[str, Any]] = []- page_token = None- while True:- result = await connector.list_files(page_token=page_token)- all_files.extend(result.get("files", []))- page_token = result.get("next_page_token")- if not page_token:- break- finally:- connector.bucket_names = original_buckets+ scoped_connector = _connector_scoped_to_buckets(connector, body.bucket_filter)+ all_files: list[dict[str, Any]] = []+ page_token = None+ while True:+ result = await scoped_connector.list_files(page_token=page_token)+ all_files.extend(result.get("files", []))+ page_token = result.get("next_page_token")+ if not page_token:+ breakif not all_files:return JSONResponse(@@ -2465,17 +2469,15 @@ async def browse_connection_files(status_code=401,)- # Temporarily override bucket filter if specified- original_buckets = None+ # Scope the listing to the requested bucket without mutating the+ # shared cached connector instance.+ listing_connector = connectorif bucket and hasattr(connector, "bucket_names"):- original_buckets = connector.bucket_names- connector.bucket_names = [bucket]+ listing_connector = _connector_scoped_to_buckets(connector, [bucket])- try:- files_result = await connector.list_files(page_token=page_token, max_files=max_files)- finally:- if original_buckets is not None:- connector.bucket_names = original_buckets+ files_result = await listing_connector.list_files(+ page_token=page_token, max_files=max_files+ )remote_files = files_result.get("files", [])next_page_token = files_result.get("next_page_token")diff --git a/src/api/settings/endpoints.py b/src/api/settings/endpoints.pyindex 2d4f2297f..0ef41c8d3 100644--- a/src/api/settings/endpoints.py+++ b/src/api/settings/endpoints.py@@ -60,6 +60,7 @@ from config.config_manager import ALLOWED_INDEX_NAME_PREFIXES, is_permitted_indefrom config.settings import (DEFAULT_DOCS_URL,ENVIRONMENT,+ HF_HOME,INGEST_SAMPLE_DATA,LANGFLOW_CHAT_FLOW_ID,LANGFLOW_INGEST_FLOW_ID,@@ -70,6 +71,7 @@ from config.settings import (OPENRAG_INGEST_VIA_CHAT,OPENRAG_SHOW_PROVIDER_INGEST_SETTINGS,OPENRAG_SHOW_SHARED_UPLOAD_TOGGLE,+ OPENRAG_SHOW_VLM_SETTINGS,SEGMENT_WRITE_KEY,clients,config_manager,@@ -101,6 +103,34 @@ from utils.version_utils import OPENRAG_VERSIONlogger = get_logger(__name__)+def _detect_local_vlm_models() -> list[str]:+ from pathlib import Path++ local_models = []+ hf_cache = Path.home() / ".cache" / "huggingface" / "hub"+ if HF_HOME:+ hf_cache = Path(HF_HOME) / "hub"+ if hf_cache.exists():+ for p in hf_cache.glob("models--*"):+ if (+ p.is_dir()+ and ("smolvlm" in p.name.lower() or "granite-docling" in p.name.lower())+ and "mlx" not in p.name.lower()+ ):+ parts = p.name.split("--")+ if len(parts) >= 3:+ owner = parts[1]+ model = "--".join(parts[2:])+ model_id = f"{owner}/{model}"+ if model_id not in local_models:+ local_models.append(model_id)+ # Ensure HuggingFaceTB/SmolVLM-256M-Instruct is first if present+ if "HuggingFaceTB/SmolVLM-256M-Instruct" in local_models:+ local_models.remove("HuggingFaceTB/SmolVLM-256M-Instruct")+ local_models.insert(0, "HuggingFaceTB/SmolVLM-256M-Instruct")+ return local_models++async def get_settings(request: Request,session_manager=Depends(get_session_manager),@@ -232,6 +262,15 @@ async def get_settings(picture_descriptions=knowledge_config.picture_descriptions,index_name=knowledge_config.index_name,disable_ingest_with_langflow=knowledge_config.disable_ingest_with_langflow,+ vlm_enabled=knowledge_config.vlm_enabled,+ vlm_provider=knowledge_config.vlm_provider,+ vlm_model=knowledge_config.vlm_model,+ vlm_prompt=knowledge_config.vlm_prompt,+ vlm_response_format=knowledge_config.vlm_response_format,+ vlm_max_tokens=knowledge_config.vlm_max_tokens,+ vlm_concurrency=knowledge_config.vlm_concurrency,+ vlm_timeout=knowledge_config.vlm_timeout,+ vlm_watsonx_api_version=knowledge_config.vlm_watsonx_api_version,),agent=AgentConfig(llm_model=agent_config.llm_model,@@ -244,6 +283,8 @@ async def get_settings(ingestion_defaults=ingestion_defaults_obj,ingest_via_chat=OPENRAG_INGEST_VIA_CHAT,show_provider_ingest_settings=OPENRAG_SHOW_PROVIDER_INGEST_SETTINGS,+ show_vlm_settings=OPENRAG_SHOW_VLM_SETTINGS,+ local_vlm_models=await asyncio.to_thread(_detect_local_vlm_models),show_shared_upload_toggle=OPENRAG_SHOW_SHARED_UPLOAD_TOGGLE,show_workspace_oauth_overrides=is_workspace_oauth_overrides_enabled(),segment_write_key=SEGMENT_WRITE_KEY or None,@@ -292,10 +333,29 @@ async def update_settings(should_validate = any(getattr(body, field) is not None for field in provider_fields)+ # Docling VLM settings reuse provider credentials, so they are gated+ # like the Providers page. They don't trigger provider validation.+ vlm_update_fields = [+ "vlm_enabled",+ "vlm_provider",+ "vlm_model",+ "vlm_prompt",+ "vlm_response_format",+ "vlm_max_tokens",+ "vlm_concurrency",+ "vlm_timeout",+ "vlm_watsonx_api_version",+ ]+ vlm_update = any(getattr(body, field) is not None for field in vlm_update_fields)+# Provider changes are admin-only. The outer gate only requires# config:write; require providers:write specifically when any# provider field is being touched (defends custom roles too).- if should_validate and is_rbac_enforced() and hasattr(rbac, "has_permission"):+ if (+ (should_validate or vlm_update)+ and is_rbac_enforced()+ and hasattr(rbac, "has_permission")+ ):uid = user.db_user_id or user.user_idif not await rbac.has_permission(uid, "providers:write"):await rbac.audit_denied(uid, "providers:write")@@ -597,6 +657,67 @@ async def update_settings(# The config will still be saved+ # Update Docling VLM settings (knowledge.vlm_*). Credentials are reused+ # from the providers config; these fields carry no secrets and are+ # intentionally NOT synced into the Langflow flow JSON.+ if body.vlm_enabled:+ effective_vlm_provider = body.vlm_provider or current_config.knowledge.vlm_provider+ if effective_vlm_provider in ("openai", "watsonx", "anthropic"):+ vlm_provider_config = current_config.providers.get_provider_config(+ effective_vlm_provider+ )+ vlm_provider_missing = (+ not getattr(vlm_provider_config, "api_key", "")+ or not vlm_provider_config.configured+ )+ if effective_vlm_provider == "watsonx":+ vlm_provider_missing = (+ vlm_provider_missing+ or not vlm_provider_config.endpoint+ or not vlm_provider_config.project_id+ )+ if vlm_provider_missing:+ return JSONResponse(+ {+ "error": (+ f"Cannot enable Docling VLM: provider '{effective_vlm_provider}' "+ "is not configured. Configure it in Settings > Providers first."+ )+ },+ status_code=400,+ )+ elif effective_vlm_provider == "local":+ # Local provider does not require any credentials+ pass+ elif effective_vlm_provider == "ollama":+ vlm_provider_config = current_config.providers.ollama+ vlm_provider_missing = (+ not getattr(vlm_provider_config, "endpoint", "")+ or not vlm_provider_config.configured+ )+ if vlm_provider_missing:+ return JSONResponse(+ {+ "error": (+ "Cannot enable Docling VLM: provider 'ollama' "+ "is not configured. Configure it in Settings > Providers first."+ )+ },+ status_code=400,+ )+ effective_vlm_model = body.vlm_model or current_config.knowledge.vlm_model+ if not effective_vlm_model.strip():+ return JSONResponse(+ {"error": "Cannot enable Docling VLM: model name is required."},+ status_code=400,+ )++ for vlm_field in vlm_update_fields:+ value = getattr(body, vlm_field)+ if value is not None:+ setattr(working_config.knowledge, vlm_field, value)+ config_updated = True+# Update provider-specific settingsprovider_updated = Falseif body.openai_api_key is not None and body.openai_api_key.strip():diff --git a/src/api/settings/models.py b/src/api/settings/models.pyindex ae9a6374c..183849fd3 100644--- a/src/api/settings/models.py+++ b/src/api/settings/models.py@@ -22,6 +22,15 @@ class SettingsUpdateBody(BaseModel):ocr: bool | None = Nonepicture_descriptions: bool | None = Nonedisable_ingest_with_langflow: bool | None = None+ vlm_enabled: bool | None = None+ vlm_provider: str | None = Field(None, pattern="^(openai|watsonx|anthropic|local|ollama)$")+ vlm_model: str | None = Field(None, min_length=1)+ vlm_prompt: str | None = None+ vlm_response_format: str | None = Field(None, pattern="^(markdown|doctags|html)$")+ vlm_max_tokens: int | None = Field(None, gt=0)+ vlm_concurrency: int | None = Field(None, gt=0)+ vlm_timeout: int | None = Field(None, gt=0)+ vlm_watsonx_api_version: str | None = Field(None, min_length=1)embedding_model: str | None = Field(None, min_length=1)embedding_provider: str | None = Field(None, pattern="^(openai|watsonx|ollama)$")index_name: str | None = Field(None, min_length=1)@@ -163,6 +172,15 @@ class KnowledgeConfig(BaseModel):picture_descriptions: bool | Noneindex_name: str | Nonedisable_ingest_with_langflow: bool | None+ vlm_enabled: bool | None = None+ vlm_provider: str | None = None+ vlm_model: str | None = None+ vlm_prompt: str | None = None+ vlm_response_format: str | None = None+ vlm_max_tokens: int | None = None+ vlm_concurrency: int | None = None+ vlm_timeout: int | None = None+ vlm_watsonx_api_version: str | None = Noneclass AgentConfig(BaseModel):@@ -195,6 +213,8 @@ class SettingsResponse(BaseModel):ingestion_defaults: IngestionDefaultsConfig | None = Noneingest_via_chat: bool = Falseshow_provider_ingest_settings: bool = False+ show_vlm_settings: bool = True+ local_vlm_models: list[str] = Field(default_factory=list)show_shared_upload_toggle: bool = Falseshow_workspace_oauth_overrides: bool = Falsesegment_write_key: str | None = Nonediff --git a/src/config/config_manager.py b/src/config/config_manager.pyindex 53b17c5bd..49244bc36 100644--- a/src/config/config_manager.py+++ b/src/config/config_manager.py@@ -178,6 +178,22 @@ class KnowledgeConfig:picture_descriptions: bool = Falseindex_name: str = "documents" # OpenSearch index namedisable_ingest_with_langflow: bool = False+ # Docling VLM pipeline. Credentials come from ProvidersConfig — never stored here.+ vlm_enabled: bool = False+ vlm_provider: str = "openai" # "openai" | "watsonx" | "anthropic" | "local" | "ollama"+ vlm_model: str = "" # e.g. "gpt-4o" or a watsonx model_id+ vlm_prompt: str = (+ "Extract ALL the text from the page, ensuring no words are omitted, "+ "and present it as accurately as possible. "+ "Then describe the content of the page in English."+ )+ # Per-page VLM response format only; the docling-serve output stays+ # to_formats="json" so downstream json_content consumers are unaffected.+ vlm_response_format: str = "markdown" # "markdown" | "doctags" | "html"+ vlm_max_tokens: int = 5000+ vlm_concurrency: int = 4+ vlm_timeout: int = 120 # per-page VLM API timeout, seconds+ vlm_watsonx_api_version: str = "2023-05-29"@dataclassdiff --git a/src/config/settings.py b/src/config/settings.pyindex 2f39b511b..ad37b1320 100644--- a/src/config/settings.py+++ b/src/config/settings.py@@ -402,6 +402,10 @@ def get_jwt_verify_signature() -> bool:DOCLING_OCR_ENGINE = os.getenv("DOCLING_OCR_ENGINE")++# Hugging Face cache root override (standard HF_HOME env var). Used to locate+# locally downloaded VLM weights; falls back to ~/.cache/huggingface when unset.+HF_HOME = os.getenv("HF_HOME")SEGMENT_WRITE_KEY = os.getenv("SEGMENT_WRITE_KEY", "")ENVIRONMENT = os.getenv("ENVIRONMENT", "")PLATFORM_AUTH_DEV_MODE = os.getenv("PLATFORM_AUTH_DEV_MODE", "false").lower() in (@@ -563,6 +567,15 @@ OPENRAG_SHOW_PROVIDER_INGEST_SETTINGS = os.getenv("OPENRAG_SHOW_PROVIDER_INGEST_SETTINGS", "false").lower() in ("true", "1", "yes")+# Show the "Advanced Vision Model (VLM) Settings" section in ingest settings.+# On by default; set to "false" to hide the VLM UI (kill switch — the backend+# VLM settings endpoints stay functional either way).+OPENRAG_SHOW_VLM_SETTINGS = os.getenv("OPENRAG_SHOW_VLM_SETTINGS", "true").lower() in (+ "true",+ "1",+ "yes",+)+# Show the "Make documents available to all users" (shared) toggle for COS bucket# ingestion, independent of OPENRAG_SHOW_PROVIDER_INGEST_SETTINGS. Deployments that# hide the general per-upload ingest tuning knobs (e.g. SaaS) still get just thisdiff --git a/src/services/docling_service.py b/src/services/docling_service.pyindex 5dcb199a5..141194b77 100644--- a/src/services/docling_service.py+++ b/src/services/docling_service.py@@ -15,6 +15,7 @@ from config.settings import (DOCLING_SERVE_VERIFY_SSL,get_openrag_config,)+from utils.container_utils import transform_localhost_urlfrom utils.logging_config import get_loggerfrom utils.run_mode_utils import is_run_mode_on_prem, is_run_mode_saas@@ -147,30 +148,127 @@ class DoclingService:)return DoclingService._default_client- def _build_docling_options(+ async def _build_docling_options_async(self,*,ocr_override: bool | None = None,picture_descriptions_override: bool | None = None,) -> dict[str, Any]:- """Build the options payload for docling from OpenRAG configs.+ """Build the options payload for docling from OpenRAG configs, incorporating VLM settings if enabled."""+ from services.watsonx_iam import WatsonxIamError, get_iam_token- Per-request overrides take precedence over saved Knowledge settings.- """config = get_openrag_config()knowledge_config = config.knowledge+ is_ocr_enabled = ocr_override if ocr_override is not None else knowledge_config.ocr+ is_pic_desc_enabled = (+ picture_descriptions_override+ if picture_descriptions_override is not None+ else knowledge_config.picture_descriptions+ )+preset = get_docling_preset_configs(table_structure=knowledge_config.table_structure,- ocr=ocr_override if ocr_override is not None else knowledge_config.ocr,- picture_descriptions=(- picture_descriptions_override- if picture_descriptions_override is not None- else knowledge_config.picture_descriptions- ),+ ocr=is_ocr_enabled,+ picture_descriptions=is_pic_desc_enabled,)options = {"to_formats": "json", "image_export_mode": "placeholder", **preset}++ # If picture descriptions are enabled, configure custom/local VLM model+ if is_pic_desc_enabled and knowledge_config.vlm_enabled:+ provider = knowledge_config.vlm_provider+ vlm_model = knowledge_config.vlm_model+ prompt = knowledge_config.vlm_prompt++ if provider == "local":+ options["picture_description_local"] = {+ "repo_id": vlm_model,+ "prompt": prompt,+ }+ elif provider == "watsonx":+ watsonx = config.providers.watsonx+ if not (watsonx.api_key and watsonx.endpoint and watsonx.project_id):+ raise DoclingServeError(+ "Docling VLM is enabled but the watsonx provider is not fully "+ "configured (api key, endpoint, and project id are required)"+ )+ try:+ token = await get_iam_token(watsonx.api_key)+ except httpx.RequestError as e:+ raise DoclingTransientError(+ f"watsonx IAM token exchange network error: {str(e)}"+ ) from e+ except WatsonxIamError as e:+ raise DoclingServeError(str(e)) from e++ url = (+ f"{watsonx.endpoint.rstrip('/')}/ml/v1/text/chat"+ f"?version={knowledge_config.vlm_watsonx_api_version}"+ )+ options["picture_description_api"] = {+ "url": url,+ "headers": {"Authorization": f"Bearer {token}"},+ "params": {+ "model_id": vlm_model,+ "project_id": watsonx.project_id,+ "max_tokens": knowledge_config.vlm_max_tokens,+ },+ "prompt": prompt,+ }+ elif provider == "anthropic":+ anthropic = config.providers.anthropic+ if not anthropic.api_key:+ raise DoclingServeError(+ "Docling VLM is enabled but the Anthropic provider is not configured"+ )+ url = "https://api.anthropic.com/v1/messages"+ options["picture_description_api"] = {+ "url": url,+ "headers": {+ "x-api-key": anthropic.api_key,+ "anthropic-version": "2023-06-01",+ "content-type": "application/json",+ },+ "params": {+ "model": vlm_model,+ "max_tokens": knowledge_config.vlm_max_tokens,+ },+ "prompt": prompt,+ }+ elif provider == "ollama":+ ollama = config.providers.ollama+ if not ollama.endpoint:+ raise DoclingServeError(+ "Docling VLM is enabled but the Ollama provider is not configured"+ )+ url = f"{transform_localhost_url(ollama.endpoint).rstrip('/')}/v1/chat/completions"+ options["picture_description_api"] = {+ "url": url,+ "headers": {},+ "params": {+ "model": vlm_model,+ "max_completion_tokens": knowledge_config.vlm_max_tokens,+ },+ "prompt": prompt,+ }+ else: # openai or default+ openai = config.providers.openai+ if not openai.api_key:+ raise DoclingServeError(+ "Docling VLM is enabled but the OpenAI provider is not configured"+ )+ url = "https://api.openai.com/v1/chat/completions"+ options["picture_description_api"] = {+ "url": url,+ "headers": {"Authorization": f"Bearer {openai.api_key}"},+ "params": {+ "model": vlm_model,+ "max_completion_tokens": knowledge_config.vlm_max_tokens,+ },+ "prompt": prompt,+ }+return optionsdef _get_auth_headers(@@ -195,22 +293,24 @@ class DoclingService:"""Upload a file to Docling Serve asynchronously using direct multipart/form-data upload."""- options = self._build_docling_options(+ options = await self._build_docling_options_async(ocr_override=ocr,picture_descriptions_override=picture_descriptions,)+headers = self._get_auth_headers(user_id, auth_header)# Docling serve async multipart endpoint /v1/convert/file/async- # Options are passed as form data+ # Options are passed as form data; dict-valued options (e.g.+ # picture_description_local, vlm_pipeline_model_api) go as JSON strings.data = {- k: str(v).lower() if isinstance(v, bool) else v+ k: json.dumps(v)+ if isinstance(v, dict)+ else str(v).lower()+ if isinstance(v, bool)+ else vfor k, v in options.items()- if not isinstance(v, dict)- } # picture_description_local needs to be JSON if it's a dict-- if "picture_description_local" in options:- data["picture_description_local"] = json.dumps(options["picture_description_local"])+ }files = {"files": (filename, file_content)}diff --git a/src/services/models_service.py b/src/services/models_service.pyindex 3311e55d8..0c498018b 100644--- a/src/services/models_service.py+++ b/src/services/models_service.py@@ -1,14 +1,13 @@-import httpximport asyncio-from typing import Dict, List, Optional++import httpx++from config.embedding_constants import OPENAI_DEFAULT_EMBEDDING_MODEL, OPENAI_EMBEDDING_MODEL_PREFIXfrom config.model_constants import (ANTHROPIC_DEFAULT_LANGUAGE_MODEL,- ANTHROPIC_VALIDATION_MODELS,OLLAMA_DEFAULT_LANGUAGE_MODEL_PATTERN,OPENAI_DEFAULT_LANGUAGE_MODEL,- OPENAI_VALIDATION_MODELS,)-from config.embedding_constants import OPENAI_DEFAULT_EMBEDDING_MODEL, OPENAI_EMBEDDING_MODEL_PREFIXfrom utils.container_utils import transform_localhost_urlfrom utils.logging_config import get_logger@@ -23,16 +22,15 @@ class UnknownEmbeddingProvider(Exception):unroutable request into LiteLLM's retry loop."""def __init__(self, model_name: str):- super().__init__(- f"No configured provider can serve embedding model '{model_name}'"- )+ super().__init__(f"No configured provider can serve embedding model '{model_name}'")self.model_name = model_name+class ModelsService:"""Service for fetching available models from different AI providers and managing a model registry."""# Registry for caching model-to-provider mapping- _model_provider_registry: Dict[str, str] = {}+ _model_provider_registry: dict[str, str] = {}_registry_lock = asyncio.Lock()def __init__(self):@@ -58,23 +56,25 @@ class ModelsService:async def update_model_registry(self):"""Fetch all models from all providers and update the internal registry.-+This method calls provider-specific methods to get the list of availablemodels and stores the mapping in a registry for fast lookup."""from config.config_manager import config_manager-+async with self._registry_lock:try:config = config_manager.get_config()new_registry = {}# Fetch from providers-+# OpenAIif config.providers.openai.api_key:try:- res = await self.get_openai_models(config.providers.openai.api_key, update_index=False)+ res = await self.get_openai_models(+ config.providers.openai.api_key, update_index=False+ )self.add_models(res, "openai", new_registry)except Exception as e:logger.debug(f"Could not fetch OpenAI models for registry: {str(e)}")@@ -82,7 +82,9 @@ class ModelsService:# Anthropicif config.providers.anthropic.api_key:try:- res = await self.get_anthropic_models(config.providers.anthropic.api_key, update_index=False)+ res = await self.get_anthropic_models(+ config.providers.anthropic.api_key, update_index=False+ )self.add_models(res, "anthropic", new_registry)except Exception as e:logger.debug(f"Could not fetch Anthropic models for registry: {str(e)}")@@ -90,11 +92,13 @@ class ModelsService:# Ollamaif config.providers.ollama.endpoint:try:- res = await self.get_ollama_models(config.providers.ollama.endpoint, update_index=False)+ res = await self.get_ollama_models(+ config.providers.ollama.endpoint, update_index=False+ )self.add_models(res, "ollama", new_registry)except Exception as e:logger.debug(f"Could not fetch Ollama models for registry: {str(e)}")-+# WatsonXif config.providers.watsonx.api_key:try:@@ -102,22 +106,24 @@ class ModelsService:config.providers.watsonx.endpoint,config.providers.watsonx.api_key,config.providers.watsonx.project_id,- update_index=False+ update_index=False,)self.add_models(res, "watsonx", new_registry)except Exception as e:logger.debug(f"Could not fetch WatsonX models for registry: {str(e)}")ModelsService._model_provider_registry = new_registry- logger.info(f"Model registry updated: {len(ModelsService._model_provider_registry)} models registered")-+ logger.info(+ f"Model registry updated: {len(ModelsService._model_provider_registry)} models registered"+ )+except Exception as e:logger.error(f"Error updating model registry: {str(e)}")async def get_litellm_model_name(self,model_name: str,- provider: Optional[str] = None,+ provider: str | None = None,strict: bool = False,) -> str:"""Resolve ``model_name`` to a LiteLLM-routable string.@@ -160,7 +166,51 @@ class ModelsService:return f"{provider_lower}/{model_name}" if provider_lower != "openai" else model_name- async def get_openai_models(self, api_key: str, update_index: bool = True) -> Dict[str, List[Dict[str, str]]]:+ def _openai_supports_images(self, model_id: str) -> bool:+ model_lower = model_id.lower()+ if "text-embedding" in model_lower:+ return False+ if "o1-mini" in model_lower or "o1-preview" in model_lower:+ return False+ return any(+ x in model_lower for x in ["gpt-4o", "gpt-5", "vision", "o3", "o4", "gpt-4-turbo", "o1"]+ )++ def _anthropic_supports_images(self, model_data: dict) -> bool:+ capabilities = model_data.get("capabilities", {})+ if isinstance(capabilities, dict):+ image_input = capabilities.get("image_input", {})+ if isinstance(image_input, dict) and image_input.get("supported") is True:+ return True+ # Fallback to model name matching if capabilities is not populated+ model_id = model_data.get("id", "").lower()+ return any(x in model_id for x in ["claude-3", "claude-4", "opus", "sonnet", "haiku"])++ def _watsonx_supports_images(self, model_data: dict) -> bool:+ model_id = model_data.get("model_id", "").lower()+ if any(x in model_id for x in ["vision", "pixtral", "qwen-vl", "qwen2-vl", "multimodal"]):+ return True+ short_desc = model_data.get("short_description", "").lower()+ if "vision" in short_desc or "multimodal" in short_desc or "image" in short_desc:+ return True+ return False++ def _ollama_supports_images(self, json_data: dict) -> bool:+ capabilities = json_data.get("capabilities", [])+ if "vision" in capabilities:+ return True+ model_info = json_data.get("model_info", {})+ if any("vision" in key.lower() or "projector" in key.lower() for key in model_info.keys()):+ return True+ details = json_data.get("details", {})+ families = details.get("families", []) or []+ if any("clip" in str(fam).lower() or "vision" in str(fam).lower() for fam in families):+ return True+ return False++ async def get_openai_models(+ self, api_key: str, update_index: bool = True+ ) -> dict[str, list[dict[str, str]]]:"""Fetch available models from OpenAI API with lightweight validation"""try:headers = {@@ -186,41 +236,35 @@ class ModelsService:for model in models:model_id = model.get("id", "")- # Language models (GPT models)- if model_id in OPENAI_VALIDATION_MODELS:- language_models.append(+ # Embedding models+ if OPENAI_EMBEDDING_MODEL_PREFIX in model_id or "text-similarity-" in model_id:+ embedding_models.append({"value": model_id,"label": model_id,- "default": model_id == OPENAI_DEFAULT_LANGUAGE_MODEL,+ "default": model_id == OPENAI_DEFAULT_EMBEDDING_MODEL,})-- # Embedding models- elif OPENAI_EMBEDDING_MODEL_PREFIX in model_id:- embedding_models.append(+ # Language models (GPT and o1/o3/chatgpt models)+ elif (+ model_id.startswith(("gpt-", "o1-", "o3-", "chatgpt-"))+ and "-moderation" not in model_id+ ):+ language_models.append({"value": model_id,"label": model_id,- "default": model_id == OPENAI_DEFAULT_EMBEDDING_MODEL,+ "default": model_id == OPENAI_DEFAULT_LANGUAGE_MODEL,+ "supports_images": self._openai_supports_images(model_id),})# Sort by name and ensure defaults are first- language_models.sort(- key=lambda x: (not x.get("default", False), x["value"])- )- embedding_models.sort(- key=lambda x: (not x.get("default", False), x["value"])- )+ language_models.sort(key=lambda x: (not x.get("default", False), x["value"]))+ embedding_models.sort(key=lambda x: (not x.get("default", False), x["value"]))if not language_models:- logger.warning(- "OpenAI API key is valid but no language models matched the validation list. "- "The API returned %d models, none matched OPENAI_VALIDATION_MODELS. "- "This may indicate a model naming scheme change.",- len(models),- )+ logger.warning("OpenAI API key is valid but no language models were found.")if not embedding_models:logger.warning("OpenAI API key is valid but no embedding models were found matching prefix '%s'.",@@ -248,7 +292,9 @@ class ModelsService:logger.error(f"Error fetching OpenAI models: {str(e)}")raise- async def get_anthropic_models(self, api_key: str, update_index: bool = True) -> Dict[str, List[Dict[str, str]]]:+ async def get_anthropic_models(+ self, api_key: str, update_index: bool = True+ ) -> dict[str, list[dict[str, str]]]:"""Fetch available models from Anthropic API"""try:headers = {@@ -274,27 +320,21 @@ class ModelsService:for model in models:model_id = model.get("id", "")-- if model_id in ANTHROPIC_VALIDATION_MODELS:- language_models.append(- {- "value": model_id,- "label": model_id,- "default": model_id == ANTHROPIC_DEFAULT_LANGUAGE_MODEL,- }- )+ language_models.append(+ {+ "value": model_id,+ "label": model.get("display_name", model_id),+ "default": model_id == ANTHROPIC_DEFAULT_LANGUAGE_MODEL,+ "supports_images": self._anthropic_supports_images(model),+ }+ )# Sort by default first, then by name- language_models.sort(- key=lambda x: (not x.get("default", False), x["value"])- )+ language_models.sort(key=lambda x: (not x.get("default", False), x["value"]))if not language_models:logger.warning(- "Anthropic API key is valid but no models matched the validation list. "- "The API returned %d models, none matched ANTHROPIC_VALIDATION_MODELS. "- "This may indicate a model naming scheme change.",- len(models),+ "Anthropic API key is valid but no models were returned.",)result = {@@ -318,7 +358,7 @@ class ModelsService:async def get_ollama_models(self, endpoint: str = None, update_index: bool = True- ) -> Dict[str, List[Dict[str, str]]]:+ ) -> dict[str, list[dict[str, str]]]:"""Fetch available models from Ollama API with tool calling capabilities for language models"""try:ollama_url = transform_localhost_url(endpoint)@@ -359,16 +399,12 @@ class ModelsService:# Check model capabilitiespayload = {"model": model_name}try:- show_response = await client.post(- show_url, json=payload, timeout=10.0- )+ show_response = await client.post(show_url, json=payload, timeout=10.0)show_response.raise_for_status()json_data = show_response.json()capabilities = json_data.get(JSON_CAPABILITIES_KEY, [])- logger.debug(- f"Model: {model_name}, Capabilities: {capabilities}"- )+ logger.debug(f"Model: {model_name}, Capabilities: {capabilities}")# Check if model has embedding capabilityhas_embedding = "embedding" in capabilities@@ -393,6 +429,7 @@ class ModelsService:"label": model_name,"default": OLLAMA_DEFAULT_LANGUAGE_MODEL_PATTERNin model_name.lower(),+ "supports_images": self._ollama_supports_images(json_data),})if not capabilities and not has_embedding:@@ -414,16 +451,10 @@ class ModelsService:continue# Remove duplicates and sort- language_models = list(- {m["value"]: m for m in language_models}.values()- )- embedding_models = list(- {m["value"]: m for m in embedding_models}.values()- )+ language_models = list({m["value"]: m for m in language_models}.values())+ embedding_models = list({m["value"]: m for m in embedding_models}.values())- language_models.sort(- key=lambda x: (not x.get("default", False), x["value"])- )+ language_models.sort(key=lambda x: (not x.get("default", False), x["value"]))embedding_models.sort(key=lambda x: x["value"])logger.info(@@ -445,8 +476,12 @@ class ModelsService:raiseasync def get_ibm_models(- self, endpoint: str = None, api_key: str = None, project_id: str = None, update_index: bool = True- ) -> Dict[str, List[Dict[str, str]]]:+ self,+ endpoint: str = None,+ api_key: str = None,+ project_id: str = None,+ update_index: bool = True,+ ) -> dict[str, list[dict[str, str]]]:"""Fetch available models from IBM Watson API"""try:# Use provided endpoint or default@@ -522,6 +557,7 @@ class ModelsService:"value": model_id,"label": model_name or model_id,"default": i == 0, # First model is default+ "supports_images": self._watsonx_supports_images(model),})else:diff --git a/src/services/search_service.py b/src/services/search_service.pyindex 0a49c5e23..0f9585518 100644--- a/src/services/search_service.py+++ b/src/services/search_service.py@@ -25,11 +25,11 @@ _global_search_service = Nonedef _is_exact_token_query(query: str) -> bool:- """Return True for code/token-like queries that should not allow partial fuzzy matches."""- if not query or len(query.strip()) < 3:- return False-+ """Return True for code/token-like queries (identifiers, versions, SKUs)+ that warrant exact-file narrowing; ordinary prose returns False."""query = query.strip()+ if not query or len(query) < 3:+ return Falseif re.search(r"[^a-zA-Z0-9\s]", query):return True@@ -37,6 +37,75 @@ def _is_exact_token_query(query: str) -> bool:return bool(re.search(r"[a-zA-Z]", query) and re.search(r"\d", query))+def _apply_exact_match_file_filter(+ query: str,+ chunks: list[dict[str, Any]],+ aggregations: dict[str, Any],+ *,+ is_wildcard_match_all: bool,+) -> tuple[list[dict[str, Any]], dict[str, Any]]:+ """If a token-like query appears verbatim in a subset of files, prefer those files.++ This avoids broad semantic spillover for unique lookups. Narrowing only+ applies to token-like queries (see _is_exact_token_query); multi-word prose+ searches keep their full hybrid-ranked results. When no file contains the+ query verbatim, the ranked results are returned untouched — hits must never+ be discarded just because the query looks token-like (e.g. "chunk_overlap",+ "v1.2", "SKU-1234").+ """+ normalized_query = query.strip().lower()+ if (+ not normalized_query+ or is_wildcard_match_all+ or len(normalized_query) < 4+ or not _is_exact_token_query(normalized_query)+ ):+ return chunks, aggregations++ exact_files = {+ filename+ for chunk in chunks+ for filename in [chunk.get("filename")]+ if isinstance(filename, str)+ and (+ normalized_query in filename.lower()+ or (+ isinstance(chunk.get("text"), str)+ and normalized_query in chunk.get("text", "").lower()+ )+ )+ }+ if not exact_files:+ return chunks, aggregations++ # Filter to only chunks from files with exact matches+ chunks = [chunk for chunk in chunks if chunk.get("filename") in exact_files]++ def _build_terms_agg(field: str) -> dict[str, Any]:+ counts = Counter(+ value+ for chunk in chunks+ for value in [chunk.get(field)]+ if isinstance(value, str) and value+ )+ return {+ "doc_count_error_upper_bound": 0,+ "sum_other_doc_count": 0,+ "buckets": [{"key": key, "doc_count": count} for key, count in counts.most_common()],+ }++ # Keep aggregations consistent with the post-filtered result set.+ aggregations = {+ **aggregations,+ "data_sources": _build_terms_agg("filename"),+ "document_types": _build_terms_agg("mimetype"),+ "owners": _build_terms_agg("owner"),+ "connector_types": _build_terms_agg("connector_type"),+ "embedding_models": _build_terms_agg("embedding_model"),+ }+ return chunks, aggregations++def register_search_service(service: "SearchService") -> None:"""Explicitly register the active search service for the @tool wrapper.@@ -587,58 +656,12 @@ class SearchService:# If query text appears verbatim in one subset of files, prefer those files# to avoid broad semantic spillover for unique lookups.- normalized_query = query.strip().lower()- aggregations = results.get("aggregations", {})- if normalized_query and not is_wildcard_match_all and len(normalized_query) >= 4:- exact_files = {- filename- for chunk in chunks- for filename in [chunk.get("filename")]- if isinstance(filename, str)- and (- normalized_query in filename.lower()- or (- isinstance(chunk.get("text"), str)- and normalized_query in chunk.get("text", "").lower()- )- )- }-- # Determine if we should apply exact-token filtering- should_filter_exact = bool(exact_files) or _is_exact_token_query(query)-- if should_filter_exact:- if exact_files:- # Filter to only chunks from files with exact matches- chunks = [chunk for chunk in chunks if chunk.get("filename") in exact_files]- else:- # No exact matches found for a token-like query - return empty results- chunks = []-- def _build_terms_agg(field: str) -> dict[str, Any]:- counts = Counter(- value- for chunk in chunks- for value in [chunk.get(field)]- if isinstance(value, str) and value- )- return {- "doc_count_error_upper_bound": 0,- "sum_other_doc_count": 0,- "buckets": [- {"key": key, "doc_count": count} for key, count in counts.most_common()- ],- }-- # Keep aggregations consistent with the post-filtered result set.- aggregations = {- **aggregations,- "data_sources": _build_terms_agg("filename"),- "document_types": _build_terms_agg("mimetype"),- "owners": _build_terms_agg("owner"),- "connector_types": _build_terms_agg("connector_type"),- "embedding_models": _build_terms_agg("embedding_model"),- }+ chunks, aggregations = _apply_exact_match_file_filter(+ query,+ chunks,+ results.get("aggregations", {}),+ is_wildcard_match_all=is_wildcard_match_all,+ )# Return both transformed results and aggregations. Surface degraded# semantic-search signals so the UI can show a non-fatal warningdiff --git a/src/services/watsonx_iam.py b/src/services/watsonx_iam.pynew file mode 100644index 000000000..73fc8ac8c--- /dev/null+++ b/src/services/watsonx_iam.py@@ -0,0 +1,69 @@+"""Cached IBM Cloud IAM token exchange for watsonx.ai.++watsonx.ai endpoints require a short-lived IAM bearer token (~1h TTL) obtained+from an IBM Cloud API key. This module exchanges and caches tokens so callers+(e.g. the Docling VLM pipeline) don't hit the IAM endpoint on every request.+"""++import asyncio+import time++import httpx++from utils.logging_config import get_logger++logger = get_logger(__name__)++IAM_TOKEN_URL = "https://iam.cloud.ibm.com/identity/token"++# Refresh tokens that expire within this margin so in-flight conversions+# don't start with a nearly-expired token.+_REFRESH_MARGIN_SECONDS = 300++_cache: dict[str, tuple[str, float]] = {} # api_key -> (token, expiry_epoch)+_lock = asyncio.Lock()+++class WatsonxIamError(Exception):+ """Raised when the IAM token exchange fails with a non-transient error."""+++async def get_iam_token(api_key: str, http_timeout: float = 15.0) -> str:+ """Exchange an IBM Cloud API key for an IAM bearer token, with caching.++ Raises:+ httpx.RequestError: network-level failure (caller may retry).+ WatsonxIamError: IAM rejected the exchange or returned no token.+ """+ async with _lock:+ cached = _cache.get(api_key)+ if cached and cached[1] - time.time() > _REFRESH_MARGIN_SECONDS:+ return cached[0]++ async with httpx.AsyncClient(timeout=http_timeout) as client:+ response = await client.post(+ IAM_TOKEN_URL,+ headers={+ "Content-Type": "application/x-www-form-urlencoded",+ "Accept": "application/json",+ },+ data={+ "grant_type": "urn:ibm:params:oauth:grant-type:apikey",+ "apikey": api_key,+ },+ )++ if response.status_code != 200:+ raise WatsonxIamError(+ f"IBM IAM token exchange failed: HTTP {response.status_code} {response.text[:200]}"+ )++ data = response.json()+ token = data.get("access_token")+ if not token:+ raise WatsonxIamError("IBM IAM response did not contain access_token")++ expiry = data.get("expiration") or (time.time() + float(data.get("expires_in", 3600)))+ _cache[api_key] = (token, float(expiry))+ logger.debug("Obtained IBM IAM token", expires_at=expiry)+ return tokendiff --git a/src/tui/managers/docling_manager.py b/src/tui/managers/docling_manager.pyindex 441c803a2..06a033a3c 100644--- a/src/tui/managers/docling_manager.py+++ b/src/tui/managers/docling_manager.py@@ -338,10 +338,17 @@ class DoclingManager:env = os.environ.copy()- # Add docling serve environment variable to change Tenant Id header name- # so that it's not used to enforce ownership locally.+ # Disable PyTorch model compilation to prevent Dynamo/AttributeError issues on CPU/Mac+ env["TORCH_COMPILE_DISABLE"] = "1"+ env["TORCH_DYNAMO_DISABLE"] = "1"++ # Add docling serve environment variablesif "DOCLING_SERVE_ENG_RAY_TENANT_ID_HEADER" not in env:env["DOCLING_SERVE_ENG_RAY_TENANT_ID_HEADER"] = "X-Docling-Tenant-Id"+ if "DOCLING_SERVE_ENABLE_REMOTE_SERVICES" not in env:+ env["DOCLING_SERVE_ENABLE_REMOTE_SERVICES"] = "true"+ if "DOCLING_SERVE_ALLOW_CUSTOM_VLM_CONFIG" not in env:+ env["DOCLING_SERVE_ALLOW_CUSTOM_VLM_CONFIG"] = "true"self._process = subprocess.Popen(cmd,diff --git a/tests/unit/test_docling_service.py b/tests/unit/test_docling_service.pyindex c474aebd0..78f84d536 100644--- a/tests/unit/test_docling_service.py+++ b/tests/unit/test_docling_service.py@@ -147,6 +147,7 @@ async def test_upload_success(docling_service, mock_httpx_client):mock_config.knowledge.table_structure = Falsemock_config.knowledge.ocr = Falsemock_config.knowledge.picture_descriptions = False+ mock_config.knowledge.vlm_enabled = Falsemock_get_config.return_value = mock_configtask_id = await docling_service.upload_to_docling_direct_async("test.pdf", b"data")@@ -165,7 +166,14 @@ async def test_upload_http_error(docling_service, mock_httpx_client):"""Raises exception if upload returns non-200."""mock_httpx_client.post.return_value = _make_response(400)- with patch("services.docling_service.get_openrag_config"):+ with patch("services.docling_service.get_openrag_config") as mock_get_config:+ mock_config = MagicMock()+ mock_config.knowledge.table_structure = False+ mock_config.knowledge.ocr = False+ mock_config.knowledge.picture_descriptions = False+ mock_config.knowledge.vlm_enabled = False+ mock_get_config.return_value = mock_config+with pytest.raises(httpx.HTTPStatusError):await docling_service.upload_to_docling_direct_async("test.pdf", b"data")@@ -173,15 +181,17 @@ async def test_upload_http_error(docling_service, mock_httpx_client):# ── Configuration Logic ─────────────────────────────────────────────-def test_build_docling_options_toggles(docling_service):+async def test_build_docling_options_toggles(docling_service):"""Correctly maps OpenRAG config to Docling options."""mock_config = MagicMock()mock_config.knowledge.table_structure = Truemock_config.knowledge.ocr = Truemock_config.knowledge.picture_descriptions = False+ mock_config.knowledge.vlm_enabled = Falsewith patch("services.docling_service.get_openrag_config", return_value=mock_config):- options = docling_service._build_docling_options()+ options = await docling_service._build_docling_options_async()assert options["do_table_structure"] is Trueassert options["do_ocr"] is True@@ -212,3 +222,113 @@ def test_init_default_url():with patch("services.docling_service.DOCLING_SERVE_URL", "http://default:5001"):service = DoclingService()assert service.docling_url == "http://default:5001"+++# ── VLM Pipeline Options ────────────────────────────────────────────+++def _vlm_mock_config(provider: str) -> MagicMock:+ mock_config = MagicMock()+ k = mock_config.knowledge+ k.vlm_enabled = True+ k.vlm_provider = provider+ k.vlm_model = "gpt-4o" if provider == "openai" else "meta-llama/llama-vision"+ k.vlm_prompt = "Extract all text."+ k.vlm_response_format = "markdown"+ k.vlm_max_tokens = 5000+ k.vlm_concurrency = 4+ k.vlm_timeout = 120+ k.vlm_openai_url = "https://api.openai.com/v1/chat/completions"+ k.vlm_watsonx_api_version = "2023-05-29"+ k.table_structure = False+ k.ocr = False+ k.picture_descriptions = True+ mock_config.providers.openai.api_key = "sk-test"+ mock_config.providers.watsonx.api_key = "wx-key"+ mock_config.providers.watsonx.endpoint = "https://us-south.ml.cloud.ibm.com/"+ mock_config.providers.watsonx.project_id = "proj-123"+ return mock_config+++async def test_build_vlm_options_openai(docling_service):+ """OpenAI VLM options carry the provider key and chat-completions params."""+ mock_config = _vlm_mock_config("openai")+ with patch("services.docling_service.get_openrag_config", return_value=mock_config):+ options = await docling_service._build_docling_options_async()++ assert options["do_picture_description"] is True+ assert options["to_formats"] == "json"+ api = options["picture_description_api"]+ assert api["url"] == "https://api.openai.com/v1/chat/completions"+ assert api["headers"]["Authorization"] == "Bearer sk-test"+ assert api["params"] == {"model": "gpt-4o", "max_completion_tokens": 5000}+ assert api["prompt"] == "Extract all text."+++async def test_build_vlm_options_watsonx(docling_service):+ """watsonx VLM options exchange the API key for an IAM bearer token."""+ mock_config = _vlm_mock_config("watsonx")+ with (+ patch("services.docling_service.get_openrag_config", return_value=mock_config),+ patch("services.watsonx_iam.get_iam_token", new_callable=AsyncMock) as mock_token,+ ):+ mock_token.return_value = "iam-token"+ options = await docling_service._build_docling_options_async()++ mock_token.assert_awaited_once_with("wx-key")+ api = options["picture_description_api"]+ assert api["url"] == "https://us-south.ml.cloud.ibm.com/ml/v1/text/chat?version=2023-05-29"+ assert api["headers"]["Authorization"] == "Bearer iam-token"+ assert api["params"] == {+ "model_id": "meta-llama/llama-vision",+ "project_id": "proj-123",+ "max_tokens": 5000,+ }+++async def test_build_vlm_options_watsonx_unconfigured(docling_service):+ """Raises DoclingServeError when watsonx provider is incomplete."""+ mock_config = _vlm_mock_config("watsonx")+ mock_config.providers.watsonx.project_id = ""+ with patch("services.docling_service.get_openrag_config", return_value=mock_config):+ with pytest.raises(DoclingServeError, match="watsonx provider is not fully"):+ await docling_service._build_docling_options_async()+++async def test_build_vlm_options_ollama(docling_service):+ """Ollama VLM options carry the provider endpoint and chat-completions params."""+ mock_config = _vlm_mock_config("ollama")+ mock_config.providers.ollama.endpoint = "http://localhost:11434"+ mock_config.providers.ollama.configured = True+ with patch("services.docling_service.get_openrag_config", return_value=mock_config):+ options = await docling_service._build_docling_options_async()++ assert options["do_picture_description"] is True+ assert options["to_formats"] == "json"+ api = options["picture_description_api"]+ assert api["url"] == "http://localhost:11434/v1/chat/completions"+ assert api["headers"] == {}+ assert api["params"] == {"model": "meta-llama/llama-vision", "max_completion_tokens": 5000}+ assert api["prompt"] == "Extract all text."+++async def test_upload_vlm_enabled_sends_vlm_form_fields(docling_service, mock_httpx_client):+ """VLM upload sends custom picture description parameters."""+ import json as json_lib++ mock_httpx_client.post.return_value = _make_response(200, {"task_id": "vlm-task"})+ mock_config = _vlm_mock_config("openai")+ with patch("services.docling_service.get_openrag_config", return_value=mock_config):+ task_id = await docling_service.upload_to_docling_direct_async("test.pdf", b"data")++ assert task_id == "vlm-task"+ _, kwargs = mock_httpx_client.post.call_args+ data = kwargs.get("data", {})+ assert data["do_picture_description"] == "true"+ api = json_lib.loads(data["picture_description_api"])+ assert api["params"]["model"] == "gpt-4o"diff --git a/tests/unit/test_search_service_exact_filter.py b/tests/unit/test_search_service_exact_filter.pynew file mode 100644index 000000000..b4545de2f--- /dev/null+++ b/tests/unit/test_search_service_exact_filter.py@@ -0,0 +1,100 @@+"""+Unit tests for the exact-match file filter in services/search_service.py.++Regression: token-like queries (e.g. "chunk_overlap", "v1.2", "SKU-1234")+must not zero out ranked hybrid results when no file contains the query+verbatim.+"""++import pytest++from services.search_service import _apply_exact_match_file_filter+++def _chunk(filename: str, text: str, **extra) -> dict:+ return {+ "filename": filename,+ "text": text,+ "mimetype": "application/pdf",+ "owner": "user-1",+ "connector_type": "local",+ "embedding_model": "text-embedding-3-small",+ **extra,+ }+++BASE_AGGS = {+ "data_sources": {"buckets": [{"key": "report.pdf", "doc_count": 2}]},+ "document_types": {"buckets": [{"key": "application/pdf", "doc_count": 2}]},+}++[email protected]("query", ["chunk_overlap", "v1.2", "SKU-1234"])+def test_token_like_query_without_exact_match_keeps_hybrid_results(query):+ """Token-like queries must not clear ranked hits when nothing matches verbatim."""+ chunks = [+ _chunk("report.pdf", "Discussion of chunking strategies and overlap."),+ _chunk("notes.md", "Semantic search configuration notes."),+ ]++ filtered, aggs = _apply_exact_match_file_filter(+ query, chunks, BASE_AGGS, is_wildcard_match_all=False+ )++ assert filtered == chunks+ assert aggs == BASE_AGGS+++def test_verbatim_match_in_text_restricts_to_matching_files():+ chunks = [+ _chunk("config-guide.pdf", "Set chunk_overlap to 200 for best results."),+ _chunk("unrelated.pdf", "Quarterly financial summary."),+ ]++ filtered, aggs = _apply_exact_match_file_filter(+ "chunk_overlap", chunks, BASE_AGGS, is_wildcard_match_all=False+ )++ assert [c["filename"] for c in filtered] == ["config-guide.pdf"]+ assert aggs["data_sources"]["buckets"] == [{"key": "config-guide.pdf", "doc_count": 1}]+++def test_verbatim_match_in_filename_restricts_to_matching_files():+ chunks = [+ _chunk("SKU-1234-spec.pdf", "Product specification."),+ _chunk("other.pdf", "Different product."),+ ]++ filtered, _ = _apply_exact_match_file_filter(+ "SKU-1234", chunks, BASE_AGGS, is_wildcard_match_all=False+ )++ assert [c["filename"] for c in filtered] == ["SKU-1234-spec.pdf"]+++def test_multiword_prose_query_is_not_narrowed():+ """Ordinary prose queries keep hybrid-ranked results even when one chunk+ happens to contain the phrase verbatim — narrowing is for token-like+ lookups only."""+ chunks = [+ _chunk("guide.pdf", "The quarterly revenue summary is attached below."),+ _chunk("other.pdf", "Revenue grew on strong quarterly performance."),+ ]++ filtered, aggs = _apply_exact_match_file_filter(+ "quarterly revenue summary", chunks, BASE_AGGS, is_wildcard_match_all=False+ )++ assert filtered == chunks+ assert aggs == BASE_AGGS+++def test_wildcard_and_short_queries_are_untouched():+ chunks = [_chunk("report.pdf", "text"), _chunk("notes.md", "more text")]++ for query, wildcard in [("*", True), ("ab", False), ("", False)]:+ filtered, aggs = _apply_exact_match_file_filter(+ query, chunks, BASE_AGGS, is_wildcard_match_all=wildcard+ )+ assert filtered == chunks+ assert aggs == BASE_AGGS