Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

feat: docling ingest vlms (#1841)

Edwin Joseb0ead4af5795
diff --git a/flows/components/export_docling_document.py b/flows/components/export_docling_document.py
index 727534435..14c83d41b 100644
--- a/flows/components/export_docling_document.py
+++ b/flows/components/export_docling_document.py
@@ -157,11 +157,18 @@ class ExportDoclingDocumentComponent(Component):
continue
# Fall back to whole-document export (no page granularity)
- content = doc.export_to_markdown(
- image_mode=image_mode,
- image_placeholder=self.md_image_placeholder,
- page_break_placeholder=self.md_page_break_placeholder,
- )
+ try:
+ content = doc.export_to_markdown(
+ image_mode=image_mode,
+ image_placeholder=self.md_image_placeholder,
+ page_break_placeholder=self.md_page_break_placeholder,
+ )
+ except TypeError:
+ # Older docling-core versions lack page_break_placeholder
+ content = doc.export_to_markdown(
+ image_mode=image_mode,
+ image_placeholder=self.md_image_placeholder,
+ )
elif self.export_format == "HTML":
content = doc.export_to_html(image_mode=image_mode)
elif self.export_format == "Plaintext":
diff --git a/frontend/app/api/mutations/useUpdateSettingsMutation.ts b/frontend/app/api/mutations/useUpdateSettingsMutation.ts
index d2c9130e4..8744bfc54 100644
--- a/frontend/app/api/mutations/useUpdateSettingsMutation.ts
+++ b/frontend/app/api/mutations/useUpdateSettingsMutation.ts
@@ -22,6 +22,17 @@ export interface UpdateSettingsRequest {
embedding_model?: string;
embedding_provider?: string;
+ // Docling VLM pipeline settings
+ vlm_enabled?: boolean;
+ vlm_provider?: string;
+ vlm_model?: string;
+ vlm_prompt?: string;
+ vlm_response_format?: string;
+ vlm_max_tokens?: number;
+ vlm_concurrency?: number;
+ vlm_timeout?: number;
+ vlm_watsonx_api_version?: string;
+
// Provider-specific settings (for dialogs)
model_provider?: string; // Deprecated, kept for backward compatibility
api_key?: string;
diff --git a/frontend/app/api/queries/useGetModelsQuery.ts b/frontend/app/api/queries/useGetModelsQuery.ts
index aaae1f76d..dfe92be37 100644
--- a/frontend/app/api/queries/useGetModelsQuery.ts
+++ b/frontend/app/api/queries/useGetModelsQuery.ts
@@ -9,6 +9,7 @@ export interface ModelOption {
value: string;
label: string;
default?: boolean;
+ supports_images?: boolean;
}
export interface ModelsResponse {
diff --git a/frontend/app/api/queries/useGetSearchQuery.ts b/frontend/app/api/queries/useGetSearchQuery.ts
index 33b98d3af..9245d0915 100644
--- a/frontend/app/api/queries/useGetSearchQuery.ts
+++ b/frontend/app/api/queries/useGetSearchQuery.ts
@@ -36,6 +36,8 @@ export interface ChunkResult {
parser?: string;
chunk_size?: number;
chunk_overlap?: number;
+ chunk_id?: string;
+ id?: string;
index?: number;
allowed_users?: string[];
allowed_groups?: string[];
diff --git a/frontend/app/api/queries/useGetSettingsQuery.ts b/frontend/app/api/queries/useGetSettingsQuery.ts
index c51e16816..c06a15ca9 100644
--- a/frontend/app/api/queries/useGetSettingsQuery.ts
+++ b/frontend/app/api/queries/useGetSettingsQuery.ts
@@ -20,6 +20,15 @@ export interface KnowledgeSettings {
ocr?: boolean;
picture_descriptions?: boolean;
disable_ingest_with_langflow?: boolean;
+ vlm_enabled?: boolean;
+ vlm_provider?: string;
+ vlm_model?: string;
+ vlm_prompt?: string;
+ vlm_response_format?: string;
+ vlm_max_tokens?: number;
+ vlm_concurrency?: number;
+ vlm_timeout?: number;
+ vlm_watsonx_api_version?: string;
}
export interface ProviderSettings {
@@ -41,6 +50,9 @@ export interface ProviderSettings {
endpoint?: string;
configured?: boolean;
};
+ local?: {
+ configured?: boolean;
+ };
}
export interface OnboardingState {
@@ -79,6 +91,8 @@ export interface Settings {
localhost_url?: string;
ingest_via_chat?: boolean;
show_provider_ingest_settings?: boolean;
+ show_vlm_settings?: boolean;
+ local_vlm_models?: string[];
show_shared_upload_toggle?: boolean;
show_workspace_oauth_overrides?: boolean;
segment_write_key?: string;
diff --git a/frontend/app/settings/_components/ingest-settings-section.tsx b/frontend/app/settings/_components/ingest-settings-section.tsx
index 2a896be32..97ec84fe8 100644
--- a/frontend/app/settings/_components/ingest-settings-section.tsx
+++ b/frontend/app/settings/_components/ingest-settings-section.tsx
@@ -4,6 +4,7 @@ import { ArrowUpRight, Loader2, Minus, Plus } from "lucide-react";
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
import { toast } from "sonner";
import {
+ useGetAnthropicModelsQuery,
useGetIBMModelsQuery,
useGetOllamaModelsQuery,
useGetOpenAIModelsQuery,
@@ -12,6 +13,12 @@ import { useGetSettingsQuery } from "@/app/api/queries/useGetSettingsQuery";
import { ConfirmationDialog } from "@/components/confirmation-dialog";
import { LabelWrapper } from "@/components/label-wrapper";
import { RequirePermission } from "@/components/require-permission";
+import {
+ Accordion,
+ AccordionContent,
+ AccordionItem,
+ AccordionTrigger,
+} from "@/components/ui/accordion";
import { Button } from "@/components/ui/button";
import {
Card,
@@ -21,8 +28,17 @@ import {
CardTitle,
} from "@/components/ui/card";
import { Input } from "@/components/ui/input";
+import { NumberInput } from "@/components/ui/inputs/number-input";
import { Label } from "@/components/ui/label";
+import {
+ Select,
+ SelectContent,
+ SelectItem,
+ SelectTrigger,
+ SelectValue,
+} from "@/components/ui/select";
import { Switch } from "@/components/ui/switch";
+import { Textarea } from "@/components/ui/textarea";
import { useAuth } from "@/contexts/auth-context";
import { useIsCloudBrand } from "@/contexts/brand-context";
import { trackButton } from "@/lib/analytics";
@@ -34,6 +50,14 @@ import { ModelSelector } from "../../onboarding/_components/model-selector";
import { getModelLogo } from "../_helpers/model-helpers";
import { LangflowIcon } from "./langflow-icon";
+const DEFAULT_WATSONX_API_VERSION = "2023-05-29";
+
+const RESPONSE_FORMATS = [
+ { value: "markdown", label: "Markdown (recommended)" },
+ { value: "doctags", label: "DocTags" },
+ { value: "html", label: "HTML" },
+] as const;
+
export function IngestSettingsSection() {
const isCloudBrand = useIsCloudBrand();
const { isAuthenticated, isNoAuthMode, isIbmAuthMode, runMode } = useAuth();
@@ -52,15 +76,35 @@ export function IngestSettingsSection() {
const [disableIngestWithLangflow, setDisableIngestWithLangflow] =
useState<boolean>(false);
+ const [vlmProvider, setVlmProvider] = useState<string>("openai");
+ const [vlmModel, setVlmModel] = useState<string>("");
+ const [vlmPrompt, setVlmPrompt] = useState<string>("");
+ const [vlmResponseFormat, setVlmResponseFormat] =
+ useState<string>("markdown");
+ const [vlmMaxTokens, setVlmMaxTokens] = useState<number>(5000);
+ const [vlmConcurrency, setVlmConcurrency] = useState<number>(4);
+ const [vlmTimeout, setVlmTimeout] = useState<number>(120);
+ const [vlmWatsonxApiVersion, setVlmWatsonxApiVersion] = useState<string>(
+ DEFAULT_WATSONX_API_VERSION,
+ );
+ const [validationError, setValidationError] = useState<string | null>(null);
+
const { data: settings = {} } = useGetSettingsQuery({
enabled: isAuthenticated || isNoAuthMode,
});
+ const showVlmSettings = settings.show_vlm_settings ?? true;
+
const { data: openaiModels, isLoading: openaiLoading } =
useGetOpenAIModelsQuery(
{ apiKey: "" },
{ enabled: settings?.providers?.openai?.configured === true },
);
+ const { data: anthropicModels, isLoading: anthropicLoading } =
+ useGetAnthropicModelsQuery(
+ { apiKey: "" },
+ { enabled: settings?.providers?.anthropic?.configured === true },
+ );
const { data: ollamaModels, isLoading: ollamaLoading } =
useGetOllamaModelsQuery(
{ endpoint: settings?.providers?.ollama?.endpoint },
@@ -129,6 +173,99 @@ export function IngestSettingsSection() {
const isLoadingAnyEmbeddingModels =
openaiLoading || ollamaLoading || watsonxLoading;
+ const groupedVlmModels = useMemo(() => {
+ const list: any[] = [];
+
+ // 1. Local Models
+ if (settings.local_vlm_models && settings.local_vlm_models.length > 0) {
+ list.push({
+ group: "Local Models",
+ icon: getModelLogo("", "local"),
+ options: settings.local_vlm_models.map((m: string) => ({
+ value: m,
+ label: m.split("/").pop(),
+ provider: "local",
+ })),
+ });
+ }
+
+ // 2. OpenAI
+ if (settings.providers?.openai?.configured) {
+ const models = (openaiModels?.language_models || [])
+ .filter((m: any) => m.supports_images === true)
+ .map((m: any) => ({ ...m, provider: "openai" }));
+ if (models.length > 0) {
+ list.push({
+ group: "OpenAI",
+ icon: getModelLogo("", "openai"),
+ options: models,
+ });
+ }
+ }
+
+ // 3. Anthropic
+ if (settings.providers?.anthropic?.configured) {
+ const models = (anthropicModels?.language_models || [])
+ .filter((m: any) => m.supports_images === true)
+ .map((m: any) => ({ ...m, provider: "anthropic" }));
+ if (models.length > 0) {
+ list.push({
+ group: "Anthropic",
+ icon: getModelLogo("", "anthropic"),
+ options: models,
+ });
+ }
+ }
+
+ // 4. Ollama
+ if (settings.providers?.ollama?.configured) {
+ const models = (ollamaModels?.language_models || [])
+ .filter((m: any) => m.supports_images === true)
+ .map((m: any) => ({ ...m, provider: "ollama" }));
+ if (models.length > 0) {
+ list.push({
+ group: "Ollama",
+ icon: getModelLogo("", "ollama"),
+ options: models,
+ });
+ }
+ }
+
+ // 5. IBM watsonx.ai
+ if (settings.providers?.watsonx?.configured) {
+ const models = (watsonxModels?.language_models || [])
+ .filter((m: any) => m.supports_images === true)
+ .map((m: any) => ({ ...m, provider: "watsonx" }));
+ if (models.length > 0) {
+ list.push({
+ group: "IBM watsonx.ai",
+ icon: getModelLogo("", "watsonx"),
+ options: models,
+ });
+ }
+ }
+
+ return list;
+ }, [
+ settings.local_vlm_models,
+ settings.providers?.openai?.configured,
+ settings.providers?.anthropic?.configured,
+ settings.providers?.ollama?.configured,
+ settings.providers?.watsonx?.configured,
+ openaiModels?.language_models,
+ anthropicModels?.language_models,
+ ollamaModels?.language_models,
+ watsonxModels?.language_models,
+ ]);
+
+ const isLoadingAnyVlmModels =
+ openaiLoading || anthropicLoading || ollamaLoading || watsonxLoading;
+
+ const allVlmOptions = useMemo(
+ () => groupedVlmModels.flatMap((g) => g.options),
+ [groupedVlmModels],
+ );
+
const updateSettingsMutation = useUpdateSettingsMutation({
onSuccess: () => {
toast.success("Settings updated successfully");
@@ -177,37 +314,80 @@ export function IngestSettingsSection() {
]);
useEffect(() => {
- if (settings.knowledge?.chunk_size !== undefined)
- setChunkSize(settings.knowledge.chunk_size);
- }, [settings.knowledge?.chunk_size]);
+ const k = settings.knowledge;
+ if (!k) return;
+ if (k.chunk_size !== undefined) setChunkSize(k.chunk_size);
+ if (k.chunk_overlap !== undefined) setChunkOverlap(k.chunk_overlap);
+ if (k.table_structure !== undefined) setTableStructure(k.table_structure);
+ if (k.ocr !== undefined) setOcr(k.ocr);
+ if (k.picture_descriptions !== undefined)
+ setPictureDescriptions(k.picture_descriptions);
+ if (k.disable_ingest_with_langflow !== undefined)
+ setDisableIngestWithLangflow(k.disable_ingest_with_langflow);
+ if (k.vlm_provider !== undefined) setVlmProvider(k.vlm_provider);
+ // Backend defaults vlm_model to ""; an empty value means "not configured",
+ // so don't clobber a locally auto-selected model with it.
+ if (k.vlm_model) setVlmModel(k.vlm_model);
+ if (k.vlm_prompt !== undefined) setVlmPrompt(k.vlm_prompt);
+ if (k.vlm_response_format !== undefined)
+ setVlmResponseFormat(k.vlm_response_format);
+ if (k.vlm_max_tokens !== undefined) setVlmMaxTokens(k.vlm_max_tokens);
+ if (k.vlm_concurrency !== undefined) setVlmConcurrency(k.vlm_concurrency);
+ if (k.vlm_timeout !== undefined) setVlmTimeout(k.vlm_timeout);
+ if (k.vlm_watsonx_api_version !== undefined)
+ setVlmWatsonxApiVersion(k.vlm_watsonx_api_version);
+ }, [settings.knowledge]);
- useEffect(() => {
- if (settings.knowledge?.chunk_overlap !== undefined)
- setChunkOverlap(settings.knowledge.chunk_overlap);
- }, [settings.knowledge?.chunk_overlap]);
+ const [vlmAccordionValue, setVlmAccordionValue] = useState<string>("");
+ const autoSelectedVlm = useRef(false);
useEffect(() => {
- if (settings.knowledge?.table_structure !== undefined)
- setTableStructure(settings.knowledge.table_structure);
- }, [settings.knowledge?.table_structure]);
-
- useEffect(() => {
- if (settings.knowledge?.ocr !== undefined) setOcr(settings.knowledge.ocr);
- }, [settings.knowledge?.ocr]);
+ if (!showVlmSettings) return;
+ if (settings.knowledge?.vlm_model) {
+ autoSelectedVlm.current = false;
+ return;
+ }
+ if (autoSelectedVlm.current) return;
+ if (settings.local_vlm_models && settings.local_vlm_models.length > 0) {
+ setVlmModel(settings.local_vlm_models[0]);
+ setVlmProvider("local");
+ autoSelectedVlm.current = true;
+ } else if (allVlmOptions.length > 0) {
+ const fallback = allVlmOptions.find((o) => o.default) || allVlmOptions[0];
+ setVlmModel(fallback.value);
+ setVlmProvider(fallback.provider || "openai");
+ autoSelectedVlm.current = true;
+ }
+ }, [
+ showVlmSettings,
+ settings.knowledge?.vlm_model,
+ settings.local_vlm_models,
+ allVlmOptions,
+ ]);
- useEffect(() => {
- if (settings.knowledge?.picture_descriptions !== undefined)
- setPictureDescriptions(settings.knowledge.picture_descriptions);
- }, [settings.knowledge?.picture_descriptions]);
+ const handleVlmModelChange = (value: string, provider?: string) => {
+ setVlmModel(value);
+ if (provider) setVlmProvider(provider);
+ setValidationError(null);
+ };
- useEffect(() => {
- if (settings.knowledge?.disable_ingest_with_langflow !== undefined)
- setDisableIngestWithLangflow(
- settings.knowledge.disable_ingest_with_langflow,
- );
- }, [settings.knowledge?.disable_ingest_with_langflow]);
+ const vlmModelPending =
+ showVlmSettings && pictureDescriptions && !vlmModel.trim();
const k = settings.knowledge;
+ const vlmDirty =
+ showVlmSettings &&
+ (pictureDescriptions !== (k?.vlm_enabled ?? pictureDescriptions) ||
+ vlmProvider !== (k?.vlm_provider ?? vlmProvider) ||
+ vlmModel !== (k?.vlm_model ?? vlmModel) ||
+ vlmPrompt !== (k?.vlm_prompt ?? vlmPrompt) ||
+ vlmResponseFormat !== (k?.vlm_response_format ?? vlmResponseFormat) ||
+ vlmMaxTokens !== (k?.vlm_max_tokens ?? vlmMaxTokens) ||
+ vlmConcurrency !== (k?.vlm_concurrency ?? vlmConcurrency) ||
+ vlmTimeout !== (k?.vlm_timeout ?? vlmTimeout) ||
+ vlmWatsonxApiVersion !==
+ (k?.vlm_watsonx_api_version ?? vlmWatsonxApiVersion));
+
const knowledgeIngestDirty =
chunkSize !== (k?.chunk_size ?? chunkSize) ||
chunkOverlap !== (k?.chunk_overlap ?? chunkOverlap) ||
@@ -215,7 +395,31 @@ export function IngestSettingsSection() {
ocr !== (k?.ocr ?? ocr) ||
pictureDescriptions !== (k?.picture_descriptions ?? pictureDescriptions) ||
disableIngestWithLangflow !==
- (k?.disable_ingest_with_langflow ?? disableIngestWithLangflow);
+ (k?.disable_ingest_with_langflow ?? disableIngestWithLangflow) ||
+ vlmDirty;
+
+ const providerConfigured =
+ settings.providers === undefined || settings.providers === null
+ ? undefined
+ : vlmProvider === "watsonx"
+ ? settings.providers.watsonx?.configured === true
+ : vlmProvider === "anthropic"
+ ? settings.providers.anthropic?.configured === true
+ : vlmProvider === "ollama"
+ ? settings.providers.ollama?.configured === true
+ : vlmProvider === "local"
+ ? true
+ : settings.providers.openai?.configured === true;
+
+ const providerWarning = pictureDescriptions && providerConfigured === false;
+ const providerLabel =
+ vlmProvider === "watsonx"
+ ? "IBM watsonx.ai"
+ : vlmProvider === "anthropic"
+ ? "Anthropic"
+ : vlmProvider === "ollama"
+ ? "Ollama"
+ : "OpenAI";
const handleChunkSizeChange = (value: string) => {
setChunkSize(Math.max(0, Number.parseInt(value, 10) || 0));
@@ -228,6 +432,22 @@ export function IngestSettingsSection() {
};
const handleKnowledgeIngestSave = () => {
+ // Only include VLM fields when the VLM UI is enabled; a hidden section
+ // must not drive backend VLM state or trip its validation.
+ const vlmPayload = showVlmSettings
+ ? {
+ vlm_enabled: pictureDescriptions,
+ vlm_provider: vlmProvider,
+ vlm_model: vlmModel.trim() || undefined,
+ vlm_prompt: vlmPrompt,
+ vlm_response_format: vlmResponseFormat,
+ vlm_max_tokens: vlmMaxTokens,
+ vlm_concurrency: vlmConcurrency,
+ vlm_timeout: vlmTimeout,
+ vlm_watsonx_api_version: vlmWatsonxApiVersion,
+ }
+ : {};
+
trackButton({
CTA: "Save Ingest Settings",
elementId: "save-ingest-settings-button",
@@ -239,8 +459,10 @@ export function IngestSettingsSection() {
ocr,
picture_descriptions: pictureDescriptions,
disable_ingest_with_langflow: disableIngestWithLangflow,
+ ...vlmPayload,
},
});
+
if (chunkSize < 1) {
const msg = "Chunk size must be at least 1";
setChunkValidationError(msg);
@@ -253,6 +475,23 @@ export function IngestSettingsSection() {
toast.error("Could not save ingest settings", { description: msg });
return;
}
+
+ if (showVlmSettings && pictureDescriptions) {
+ if (!vlmModel.trim()) {
+ const msg =
+ "Model name is required when picture descriptions are enabled";
+ setValidationError(msg);
+ toast.error("Could not save ingest settings", { description: msg });
+ return;
+ }
+ if (vlmMaxTokens < 1 || vlmConcurrency < 1 || vlmTimeout < 1) {
+ const msg = "Max tokens, concurrency, and timeout must be at least 1";
+ setValidationError(msg);
+ toast.error("Could not save ingest settings", { description: msg });
+ return;
+ }
+ }
+
updateSettingsMutation.mutate(
{
chunk_size: chunkSize,
@@ -261,8 +500,14 @@ export function IngestSettingsSection() {
ocr,
picture_descriptions: pictureDescriptions,
disable_ingest_with_langflow: disableIngestWithLangflow,
+ ...vlmPayload,
+ },
+ {
+ onSuccess: () => {
+ setChunkValidationError(null);
+ setValidationError(null);
+ },
},
- { onSuccess: () => setChunkValidationError(null) },
);
};
@@ -579,12 +824,171 @@ export function IngestSettingsSection() {
onCheckedChange={setPictureDescriptions}
/>
</div>
+ {showVlmSettings && (
+ <div
+ className={cn(
+ "mt-4 border border-border rounded-lg bg-muted/5 overflow-hidden transition-all duration-200",
+ !pictureDescriptions &&
+ "opacity-50 cursor-not-allowed select-none",
+ )}
+ >
+ <Accordion
+ type="single"
+ collapsible
+ disabled={!pictureDescriptions}
+ value={pictureDescriptions ? vlmAccordionValue : ""}
+ onValueChange={setVlmAccordionValue}
+ >
+ <AccordionItem value="vlm-settings" className="border-none">
+ <AccordionTrigger
+ className={cn(
+ "hover:no-underline font-medium text-foreground px-4 py-3 bg-muted/10 border-b border-border",
+ !pictureDescriptions && "pointer-events-none",
+ )}
+ >
+ Advanced Vision Model (VLM) Settings
+ </AccordionTrigger>
+ <AccordionContent className="p-4 space-y-6">
+ <div className="space-y-2">
+ <LabelWrapper
+ id="vlm-model"
+ label="Vision model"
+ helperText="Pick a vision-capable model; the provider is set from your selection"
+ required={pictureDescriptions}
+ >
+ <ModelSelector
+ groupedOptions={groupedVlmModels}
+ noOptionsPlaceholder={
+ isLoadingAnyVlmModels
+ ? "Loading models..."
+ : "No models detected. Configure OpenAI, Anthropic, Ollama, or IBM watsonx.ai first."
+ }
+ value={vlmModel}
+ onValueChange={handleVlmModelChange}
+ hasError={!!validationError}
+ />
+ </LabelWrapper>
+ {providerWarning && (
+ <p className="text-sm text-destructive" role="alert">
+ {providerLabel} is not configured. Configure it in
+ Settings &gt; Providers first.
+ </p>
+ )}
+ </div>
+
+ {vlmProvider === "watsonx" && (
+ <div className="space-y-2">
+ <LabelWrapper
+ id="vlm-watsonx-api-version"
+ label="watsonx API version"
+ helperText="API version date sent to watsonx.ai"
+ >
+ <Input
+ id="vlm-watsonx-api-version"
+ type="text"
+ placeholder={DEFAULT_WATSONX_API_VERSION}
+ value={vlmWatsonxApiVersion}
+ onChange={(e) =>
+ setVlmWatsonxApiVersion(e.target.value)
+ }
+ />
+ </LabelWrapper>
+ </div>
+ )}
+
+ <div className="space-y-2">
+ <LabelWrapper
+ id="vlm-prompt"
+ label="Prompt"
+ helperText="Sent to the VLM for every page"
+ >
+ <Textarea
+ id="vlm-prompt"
+ rows={3}
+ value={vlmPrompt}
+ onChange={(e) => setVlmPrompt(e.target.value)}
+ />
+ </LabelWrapper>
+ </div>
+
+ <div className="space-y-2">
+ <LabelWrapper
+ id="vlm-response-format"
+ label="Response format"
+ helperText="Per-page VLM output. Markdown is compatible with the existing pipeline; the final document is always Docling JSON."
+ >
+ <Select
+ value={vlmResponseFormat}
+ onValueChange={setVlmResponseFormat}
+ >
+ <SelectTrigger id="vlm-response-format">
+ <SelectValue placeholder="Select a format" />
+ </SelectTrigger>
+ <SelectContent>
+ {RESPONSE_FORMATS.map((format) => (
+ <SelectItem
+ key={format.value}
+ value={format.value}
+ >
+ {format.label}
+ </SelectItem>
+ ))}
+ </SelectContent>
+ </Select>
+ </LabelWrapper>
+ </div>
+
+ <div className="grid grid-cols-3 gap-4">
+ <NumberInput
+ id="vlm-max-tokens"
+ label="Max tokens per page"
+ value={vlmMaxTokens}
+ onChange={(value) =>
+ setVlmMaxTokens(Math.max(1, value))
+ }
+ unit="tokens"
+ min={1}
+ />
+ <NumberInput
+ id="vlm-concurrency"
+ label="Concurrency"
+ value={vlmConcurrency}
+ onChange={(value) =>
+ setVlmConcurrency(Math.max(1, value))
+ }
+ unit="requests"
+ min={1}
+ />
+ <NumberInput
+ id="vlm-timeout"
+ label="API timeout"
+ value={vlmTimeout}
+ onChange={(value) =>
+ setVlmTimeout(Math.max(1, value))
+ }
+ unit="seconds"
+ min={1}
+ />
+ </div>
+
+ {validationError && (
+ <p className="text-sm text-destructive" role="alert">
+ {validationError}
+ </p>
+ )}
+ </AccordionContent>
+ </AccordionItem>
+ </Accordion>
+ </div>
+ )}
</div>
<div className="flex justify-end pt-2">
<Button
onClick={handleKnowledgeIngestSave}
disabled={
- updateSettingsMutation.isPending || !knowledgeIngestDirty
+ updateSettingsMutation.isPending ||
+ !knowledgeIngestDirty ||
+ vlmModelPending
}
className="min-w-[120px]"
size="sm"
diff --git a/frontend/app/settings/_components/model-providers.tsx b/frontend/app/settings/_components/model-providers.tsx
index ef1ceb80c..b5cc970de 100644
--- a/frontend/app/settings/_components/model-providers.tsx
+++ b/frontend/app/settings/_components/model-providers.tsx
@@ -94,6 +94,34 @@ export const ModelProviders = () => {
logoColor: "text-white",
logoBgColor: "bg-[#1063FE]",
},
+ local: {
+ name: "Local",
+ logo: (props) => (
+ <svg
+ xmlns="http://www.w3.org/2000/svg"
+ viewBox="0 0 24 24"
+ fill="none"
+ stroke="currentColor"
+ strokeWidth="2"
+ strokeLinecap="round"
+ strokeLinejoin="round"
+ {...props}
+ >
+ <rect x="4" y="4" width="16" height="16" rx="2" />
+ <rect x="9" y="9" width="6" height="6" />
+ <path d="M9 1v3" />
+ <path d="M15 1v3" />
+ <path d="M9 20v3" />
+ <path d="M15 20v3" />
+ <path d="M20 9h3" />
+ <path d="M20 15h3" />
+ <path d="M1 9h3" />
+ <path d="M1 15h3" />
+ </svg>
+ ),
+ logoColor: "text-muted-foreground",
+ logoBgColor: "bg-white",
+ },
};
const currentLlmProvider =
diff --git a/frontend/app/settings/_helpers/model-helpers.tsx b/frontend/app/settings/_helpers/model-helpers.tsx
index 216b621b7..fac3d2ee0 100644
--- a/frontend/app/settings/_helpers/model-helpers.tsx
+++ b/frontend/app/settings/_helpers/model-helpers.tsx
@@ -3,7 +3,12 @@ import IBMLogo from "@/components/icons/ibm-logo";
import OllamaLogo from "@/components/icons/ollama-logo";
import OpenAILogo from "@/components/icons/openai-logo";
-export type ModelProvider = "openai" | "anthropic" | "ollama" | "watsonx";
+export type ModelProvider =
+ | "openai"
+ | "anthropic"
+ | "ollama"
+ | "watsonx"
+ | "local";
// Full ordered list of providers for settings / cards
export const ALL_PROVIDERS: ModelProvider[] = [
@@ -47,6 +52,30 @@ export function getModelLogo(modelValue: string, provider?: ModelProvider) {
return <OllamaLogo className="w-4 h-4" />;
} else if (provider === "watsonx") {
return <IBMLogo className="w-4 h-4" />;
+ } else if (provider === "local") {
+ return (
+ <svg
+ xmlns="http://www.w3.org/2000/svg"
+ viewBox="0 0 24 24"
+ fill="none"
+ stroke="currentColor"
+ strokeWidth="2"
+ strokeLinecap="round"
+ strokeLinejoin="round"
+ className="w-4 h-4 text-muted-foreground"
+ >
+ <rect x="4" y="4" width="16" height="16" rx="2" />
+ <rect x="9" y="9" width="6" height="6" />
+ <path d="M9 1v3" />
+ <path d="M15 1v3" />
+ <path d="M9 20v3" />
+ <path d="M15 20v3" />
+ <path d="M20 9h3" />
+ <path d="M20 15h3" />
+ <path d="M1 9h3" />
+ <path d="M1 15h3" />
+ </svg>
+ );
}
// Fallback to model name analysis
diff --git a/frontend/components/provider-health-banner.tsx b/frontend/components/provider-health-banner.tsx
index 6061a7f9d..722cb8ed9 100644
--- a/frontend/components/provider-health-banner.tsx
+++ b/frontend/components/provider-health-banner.tsx
@@ -51,6 +51,7 @@ const providerTitleMap: Record<ModelProvider, string> = {
anthropic: "Anthropic",
ollama: "Ollama",
watsonx: "IBM watsonx.ai",
+ local: "Local",
};
export function ProviderHealthBanner({ className }: ProviderHealthBannerProps) {
diff --git a/frontend/lib/chat-stream-parsers.ts b/frontend/lib/chat-stream-parsers.ts
index 6a27427cf..ab9a8d580 100644
--- a/frontend/lib/chat-stream-parsers.ts
+++ b/frontend/lib/chat-stream-parsers.ts
@@ -36,8 +36,9 @@ function findFunctionCallByEvent(
): FunctionCall | undefined {
const itemId = c.item_id || c.call_id || c.id;
if (typeof itemId === "string") {
- const match = calls.find((fc) => fc.id === itemId);
- if (match) return match;
+ // The event names a specific call; if we don't know it yet, let the
+ // caller create it rather than guessing an existing one.
+ return calls.find((fc) => fc.id === itemId);
}
const outputIndex =
@@ -50,7 +51,9 @@ function findFunctionCallByEvent(
return calls[outputIndex];
}
- return calls[calls.length - 1];
+ // No identifying info on the event: continue an in-progress call that
+ // hasn't been assigned an id yet, if there is one.
+ return [...calls].reverse().find((fc) => fc.status === "pending" && !fc.id);
}
export function parseOpenAIChatChunk(
diff --git a/kubernetes/helm/openrag/values.yaml b/kubernetes/helm/openrag/values.yaml
index ecdcb5b23..800bf8be3 100644
--- a/kubernetes/helm/openrag/values.yaml
+++ b/kubernetes/helm/openrag/values.yaml
@@ -34,6 +34,11 @@ global:
port: 5001
scheme: "http"
verifySsl: false
+ # Note: this chart does not deploy docling-serve. To use OpenRAG's Docling
+ # VLM pipeline (remote OpenAI/watsonx vision models), the external
+ # docling-serve deployment must set:
+ # DOCLING_SERVE_ENABLE_REMOTE_SERVICES=true
+ # DOCLING_SERVE_ALLOW_CUSTOM_VLM_CONFIG=true
# Shared OAuth credentials (same across all tenants)
oauth:
diff --git a/kubernetes/operator/config/samples/docling-complete.yaml b/kubernetes/operator/config/samples/docling-complete.yaml
index b170558cc..6bdb3bf33 100644
--- a/kubernetes/operator/config/samples/docling-complete.yaml
+++ b/kubernetes/operator/config/samples/docling-complete.yaml
@@ -73,6 +73,16 @@ spec:
serviceAnnotations:
prometheus.io/scrape: "true"
prometheus.io/port: "5001"
+
+ # Docling configuration for the serve API
+ config:
+ # Required for OpenRAG's Docling VLM pipeline (remote OpenAI/watsonx
+ # vision models). Harmless when VLM is not used.
+ extraEnv:
+ - name: "DOCLING_SERVE_ENABLE_REMOTE_SERVICES"
+ value: "true"
+ - name: "DOCLING_SERVE_ALLOW_CUSTOM_VLM_CONFIG"
+ value: "true"
# Node placement
nodeSelector:
diff --git a/kubernetes/operator/config/samples/openrag_v1alpha1_openrag.yaml b/kubernetes/operator/config/samples/openrag_v1alpha1_openrag.yaml
index a3365dbb6..9e91435b9 100644
--- a/kubernetes/operator/config/samples/openrag_v1alpha1_openrag.yaml
+++ b/kubernetes/operator/config/samples/openrag_v1alpha1_openrag.yaml
@@ -105,6 +105,13 @@ spec:
# app.kubernetes.io/component: "document-processor"
# podAnnotations:
# sidecar.istio.io/inject: "true"
+ # config:
+ # # Required for OpenRAG's Docling VLM pipeline (remote vision models)
+ # extraEnv:
+ # - name: "DOCLING_SERVE_ENABLE_REMOTE_SERVICES"
+ # value: "true"
+ # - name: "DOCLING_SERVE_ALLOW_CUSTOM_VLM_CONFIG"
+ # value: "true"
# hpa:
# enabled: true
# minReplicas: 1
diff --git a/src/api/connectors.py b/src/api/connectors.py
index c074b87f3..ba83bf843 100644
--- a/src/api/connectors.py
+++ b/src/api/connectors.py
@@ -1,3 +1,4 @@
+import copy
from datetime import UTC, datetime
from typing import Any
@@ -675,20 +676,19 @@ async def _expand_selected_connector_files(
expanded_files_info: list[dict[str, Any]] = []
if file_ids and hasattr(connector, "cfg"):
- original_file_ids = getattr(connector.cfg, "file_ids", None)
- original_folder_ids = getattr(connector.cfg, "folder_ids", None)
+ # The connector is cached and shared across requests by
+ # ConnectionManager, so scope the listing to a per-call copy with its
+ # own cfg instead of mutating (and racing on) the shared config.
+ scoped_connector = copy.copy(connector)
+ scoped_connector.cfg = copy.copy(connector.cfg)
+ scoped_connector.cfg.file_ids = file_ids
+ scoped_connector.cfg.folder_ids = None
try:
- connector.cfg.file_ids = file_ids
- connector.cfg.folder_ids = None
-
- result = await connector.list_files()
+ result = await scoped_connector.list_files()
for f in result.get("files", []):
expanded_files_info.append(_connector_file_response(f))
except Exception as e:
logger.error("Failed to expand files in duplicate check", error=str(e))
- finally:
- connector.cfg.file_ids = original_file_ids
- connector.cfg.folder_ids = original_folder_ids
if not expanded_files_info:
for f in selected_files_raw:
@@ -774,6 +774,18 @@ async def _classify_connector_duplicates(
}
+def _connector_scoped_to_buckets(connector, bucket_names: list[str]):
+ """Shallow copy of a (cached, shared) connector with bucket_names overridden.
+
+ connector_service.get_connector() returns a per-connection cached instance;
+ mutating bucket_names on it directly would leak the override into concurrent
+ requests using the same connection.
+ """
+ scoped = copy.copy(connector)
+ scoped.bucket_names = list(bucket_names)
+ return scoped
+
+
async def _classify_bucket_connector_duplicates(
connector,
connector_type: str,
@@ -790,19 +802,15 @@ async def _classify_bucket_connector_duplicates(
``non_duplicate_files`` so the caller can sync just those when the user
chooses to skip duplicates.
"""
- original_buckets = connector.bucket_names
- connector.bucket_names = bucket_filter
- try:
- all_files: list[dict[str, Any]] = []
- page_token = None
- while True:
- result = await connector.list_files(page_token=page_token)
- all_files.extend(result.get("files", []))
- page_token = result.get("next_page_token")
- if not page_token:
- break
- finally:
- connector.bucket_names = original_buckets
+ scoped_connector = _connector_scoped_to_buckets(connector, bucket_filter)
+ all_files: list[dict[str, Any]] = []
+ page_token = None
+ while True:
+ result = await scoped_connector.list_files(page_token=page_token)
+ all_files.extend(result.get("files", []))
+ page_token = result.get("next_page_token")
+ if not page_token:
+ break
if not all_files:
return {
@@ -1279,19 +1287,15 @@ async def connector_sync(
# container every time. Per-file dedup in ConnectorFileProcessor is a
# backstop, but it runs after download — this pre-filter avoids the
# redundant fetch/reprocess and the misleading "all files" task view.
- original_buckets = connector.bucket_names
- connector.bucket_names = body.bucket_filter
- try:
- all_files: list[dict[str, Any]] = []
- page_token = None
- while True:
- result = await connector.list_files(page_token=page_token)
- all_files.extend(result.get("files", []))
- page_token = result.get("next_page_token")
- if not page_token:
- break
- finally:
- connector.bucket_names = original_buckets
+ scoped_connector = _connector_scoped_to_buckets(connector, body.bucket_filter)
+ all_files: list[dict[str, Any]] = []
+ page_token = None
+ while True:
+ result = await scoped_connector.list_files(page_token=page_token)
+ all_files.extend(result.get("files", []))
+ page_token = result.get("next_page_token")
+ if not page_token:
+ break
if not all_files:
return JSONResponse(
@@ -2465,17 +2469,15 @@ async def browse_connection_files(
status_code=401,
)
- # Temporarily override bucket filter if specified
- original_buckets = None
+ # Scope the listing to the requested bucket without mutating the
+ # shared cached connector instance.
+ listing_connector = connector
if bucket and hasattr(connector, "bucket_names"):
- original_buckets = connector.bucket_names
- connector.bucket_names = [bucket]
+ listing_connector = _connector_scoped_to_buckets(connector, [bucket])
- try:
- files_result = await connector.list_files(page_token=page_token, max_files=max_files)
- finally:
- if original_buckets is not None:
- connector.bucket_names = original_buckets
+ files_result = await listing_connector.list_files(
+ page_token=page_token, max_files=max_files
+ )
remote_files = files_result.get("files", [])
next_page_token = files_result.get("next_page_token")
diff --git a/src/api/settings/endpoints.py b/src/api/settings/endpoints.py
index 2d4f2297f..0ef41c8d3 100644
--- a/src/api/settings/endpoints.py
+++ b/src/api/settings/endpoints.py
@@ -60,6 +60,7 @@ from config.config_manager import ALLOWED_INDEX_NAME_PREFIXES, is_permitted_inde
from config.settings import (
DEFAULT_DOCS_URL,
ENVIRONMENT,
+ HF_HOME,
INGEST_SAMPLE_DATA,
LANGFLOW_CHAT_FLOW_ID,
LANGFLOW_INGEST_FLOW_ID,
@@ -70,6 +71,7 @@ from config.settings import (
OPENRAG_INGEST_VIA_CHAT,
OPENRAG_SHOW_PROVIDER_INGEST_SETTINGS,
OPENRAG_SHOW_SHARED_UPLOAD_TOGGLE,
+ OPENRAG_SHOW_VLM_SETTINGS,
SEGMENT_WRITE_KEY,
clients,
config_manager,
@@ -101,6 +103,34 @@ from utils.version_utils import OPENRAG_VERSION
logger = get_logger(__name__)
+def _detect_local_vlm_models() -> list[str]:
+ from pathlib import Path
+
+ local_models = []
+ hf_cache = Path.home() / ".cache" / "huggingface" / "hub"
+ if HF_HOME:
+ hf_cache = Path(HF_HOME) / "hub"
+ if hf_cache.exists():
+ for p in hf_cache.glob("models--*"):
+ if (
+ p.is_dir()
+ and ("smolvlm" in p.name.lower() or "granite-docling" in p.name.lower())
+ and "mlx" not in p.name.lower()
+ ):
+ parts = p.name.split("--")
+ if len(parts) >= 3:
+ owner = parts[1]
+ model = "--".join(parts[2:])
+ model_id = f"{owner}/{model}"
+ if model_id not in local_models:
+ local_models.append(model_id)
+ # Ensure HuggingFaceTB/SmolVLM-256M-Instruct is first if present
+ if "HuggingFaceTB/SmolVLM-256M-Instruct" in local_models:
+ local_models.remove("HuggingFaceTB/SmolVLM-256M-Instruct")
+ local_models.insert(0, "HuggingFaceTB/SmolVLM-256M-Instruct")
+ return local_models
+
+
async def get_settings(
request: Request,
session_manager=Depends(get_session_manager),
@@ -232,6 +262,15 @@ async def get_settings(
picture_descriptions=knowledge_config.picture_descriptions,
index_name=knowledge_config.index_name,
disable_ingest_with_langflow=knowledge_config.disable_ingest_with_langflow,
+ vlm_enabled=knowledge_config.vlm_enabled,
+ vlm_provider=knowledge_config.vlm_provider,
+ vlm_model=knowledge_config.vlm_model,
+ vlm_prompt=knowledge_config.vlm_prompt,
+ vlm_response_format=knowledge_config.vlm_response_format,
+ vlm_max_tokens=knowledge_config.vlm_max_tokens,
+ vlm_concurrency=knowledge_config.vlm_concurrency,
+ vlm_timeout=knowledge_config.vlm_timeout,
+ vlm_watsonx_api_version=knowledge_config.vlm_watsonx_api_version,
),
agent=AgentConfig(
llm_model=agent_config.llm_model,
@@ -244,6 +283,8 @@ async def get_settings(
ingestion_defaults=ingestion_defaults_obj,
ingest_via_chat=OPENRAG_INGEST_VIA_CHAT,
show_provider_ingest_settings=OPENRAG_SHOW_PROVIDER_INGEST_SETTINGS,
+ show_vlm_settings=OPENRAG_SHOW_VLM_SETTINGS,
+ local_vlm_models=await asyncio.to_thread(_detect_local_vlm_models),
show_shared_upload_toggle=OPENRAG_SHOW_SHARED_UPLOAD_TOGGLE,
show_workspace_oauth_overrides=is_workspace_oauth_overrides_enabled(),
segment_write_key=SEGMENT_WRITE_KEY or None,
@@ -292,10 +333,29 @@ async def update_settings(
should_validate = any(getattr(body, field) is not None for field in provider_fields)
+ # Docling VLM settings reuse provider credentials, so they are gated
+ # like the Providers page. They don't trigger provider validation.
+ vlm_update_fields = [
+ "vlm_enabled",
+ "vlm_provider",
+ "vlm_model",
+ "vlm_prompt",
+ "vlm_response_format",
+ "vlm_max_tokens",
+ "vlm_concurrency",
+ "vlm_timeout",
+ "vlm_watsonx_api_version",
+ ]
+ vlm_update = any(getattr(body, field) is not None for field in vlm_update_fields)
+
# Provider changes are admin-only. The outer gate only requires
# config:write; require providers:write specifically when any
# provider field is being touched (defends custom roles too).
- if should_validate and is_rbac_enforced() and hasattr(rbac, "has_permission"):
+ if (
+ (should_validate or vlm_update)
+ and is_rbac_enforced()
+ and hasattr(rbac, "has_permission")
+ ):
uid = user.db_user_id or user.user_id
if not await rbac.has_permission(uid, "providers:write"):
await rbac.audit_denied(uid, "providers:write")
@@ -597,6 +657,67 @@ async def update_settings(
# The config will still be saved
+ # Update Docling VLM settings (knowledge.vlm_*). Credentials are reused
+ # from the providers config; these fields carry no secrets and are
+ # intentionally NOT synced into the Langflow flow JSON.
+ if body.vlm_enabled:
+ effective_vlm_provider = body.vlm_provider or current_config.knowledge.vlm_provider
+ if effective_vlm_provider in ("openai", "watsonx", "anthropic"):
+ vlm_provider_config = current_config.providers.get_provider_config(
+ effective_vlm_provider
+ )
+ vlm_provider_missing = (
+ not getattr(vlm_provider_config, "api_key", "")
+ or not vlm_provider_config.configured
+ )
+ if effective_vlm_provider == "watsonx":
+ vlm_provider_missing = (
+ vlm_provider_missing
+ or not vlm_provider_config.endpoint
+ or not vlm_provider_config.project_id
+ )
+ if vlm_provider_missing:
+ return JSONResponse(
+ {
+ "error": (
+ f"Cannot enable Docling VLM: provider '{effective_vlm_provider}' "
+ "is not configured. Configure it in Settings > Providers first."
+ )
+ },
+ status_code=400,
+ )
+ elif effective_vlm_provider == "local":
+ # Local provider does not require any credentials
+ pass
+ elif effective_vlm_provider == "ollama":
+ vlm_provider_config = current_config.providers.ollama
+ vlm_provider_missing = (
+ not getattr(vlm_provider_config, "endpoint", "")
+ or not vlm_provider_config.configured
+ )
+ if vlm_provider_missing:
+ return JSONResponse(
+ {
+ "error": (
+ "Cannot enable Docling VLM: provider 'ollama' "
+ "is not configured. Configure it in Settings > Providers first."
+ )
+ },
+ status_code=400,
+ )
+ effective_vlm_model = body.vlm_model or current_config.knowledge.vlm_model
+ if not effective_vlm_model.strip():
+ return JSONResponse(
+ {"error": "Cannot enable Docling VLM: model name is required."},
+ status_code=400,
+ )
+
+ for vlm_field in vlm_update_fields:
+ value = getattr(body, vlm_field)
+ if value is not None:
+ setattr(working_config.knowledge, vlm_field, value)
+ config_updated = True
+
# Update provider-specific settings
provider_updated = False
if body.openai_api_key is not None and body.openai_api_key.strip():
diff --git a/src/api/settings/models.py b/src/api/settings/models.py
index ae9a6374c..183849fd3 100644
--- a/src/api/settings/models.py
+++ b/src/api/settings/models.py
@@ -22,6 +22,15 @@ class SettingsUpdateBody(BaseModel):
ocr: bool | None = None
picture_descriptions: bool | None = None
disable_ingest_with_langflow: bool | None = None
+ vlm_enabled: bool | None = None
+ vlm_provider: str | None = Field(None, pattern="^(openai|watsonx|anthropic|local|ollama)$")
+ vlm_model: str | None = Field(None, min_length=1)
+ vlm_prompt: str | None = None
+ vlm_response_format: str | None = Field(None, pattern="^(markdown|doctags|html)$")
+ vlm_max_tokens: int | None = Field(None, gt=0)
+ vlm_concurrency: int | None = Field(None, gt=0)
+ vlm_timeout: int | None = Field(None, gt=0)
+ vlm_watsonx_api_version: str | None = Field(None, min_length=1)
embedding_model: str | None = Field(None, min_length=1)
embedding_provider: str | None = Field(None, pattern="^(openai|watsonx|ollama)$")
index_name: str | None = Field(None, min_length=1)
@@ -163,6 +172,15 @@ class KnowledgeConfig(BaseModel):
picture_descriptions: bool | None
index_name: str | None
disable_ingest_with_langflow: bool | None
+ vlm_enabled: bool | None = None
+ vlm_provider: str | None = None
+ vlm_model: str | None = None
+ vlm_prompt: str | None = None
+ vlm_response_format: str | None = None
+ vlm_max_tokens: int | None = None
+ vlm_concurrency: int | None = None
+ vlm_timeout: int | None = None
+ vlm_watsonx_api_version: str | None = None
class AgentConfig(BaseModel):
@@ -195,6 +213,8 @@ class SettingsResponse(BaseModel):
ingestion_defaults: IngestionDefaultsConfig | None = None
ingest_via_chat: bool = False
show_provider_ingest_settings: bool = False
+ show_vlm_settings: bool = True
+ local_vlm_models: list[str] = Field(default_factory=list)
show_shared_upload_toggle: bool = False
show_workspace_oauth_overrides: bool = False
segment_write_key: str | None = None
diff --git a/src/config/config_manager.py b/src/config/config_manager.py
index 53b17c5bd..49244bc36 100644
--- a/src/config/config_manager.py
+++ b/src/config/config_manager.py
@@ -178,6 +178,22 @@ class KnowledgeConfig:
picture_descriptions: bool = False
index_name: str = "documents" # OpenSearch index name
disable_ingest_with_langflow: bool = False
+ # Docling VLM pipeline. Credentials come from ProvidersConfig — never stored here.
+ vlm_enabled: bool = False
+ vlm_provider: str = "openai" # "openai" | "watsonx" | "anthropic" | "local" | "ollama"
+ vlm_model: str = "" # e.g. "gpt-4o" or a watsonx model_id
+ vlm_prompt: str = (
+ "Extract ALL the text from the page, ensuring no words are omitted, "
+ "and present it as accurately as possible. "
+ "Then describe the content of the page in English."
+ )
+ # Per-page VLM response format only; the docling-serve output stays
+ # to_formats="json" so downstream json_content consumers are unaffected.
+ vlm_response_format: str = "markdown" # "markdown" | "doctags" | "html"
+ vlm_max_tokens: int = 5000
+ vlm_concurrency: int = 4
+ vlm_timeout: int = 120 # per-page VLM API timeout, seconds
+ vlm_watsonx_api_version: str = "2023-05-29"
@dataclass
diff --git a/src/config/settings.py b/src/config/settings.py
index 2f39b511b..ad37b1320 100644
--- a/src/config/settings.py
+++ b/src/config/settings.py
@@ -402,6 +402,10 @@ def get_jwt_verify_signature() -> bool:
DOCLING_OCR_ENGINE = os.getenv("DOCLING_OCR_ENGINE")
+
+# Hugging Face cache root override (standard HF_HOME env var). Used to locate
+# locally downloaded VLM weights; falls back to ~/.cache/huggingface when unset.
+HF_HOME = os.getenv("HF_HOME")
SEGMENT_WRITE_KEY = os.getenv("SEGMENT_WRITE_KEY", "")
ENVIRONMENT = os.getenv("ENVIRONMENT", "")
PLATFORM_AUTH_DEV_MODE = os.getenv("PLATFORM_AUTH_DEV_MODE", "false").lower() in (
@@ -563,6 +567,15 @@ OPENRAG_SHOW_PROVIDER_INGEST_SETTINGS = os.getenv(
"OPENRAG_SHOW_PROVIDER_INGEST_SETTINGS", "false"
).lower() in ("true", "1", "yes")
+# Show the "Advanced Vision Model (VLM) Settings" section in ingest settings.
+# On by default; set to "false" to hide the VLM UI (kill switch — the backend
+# VLM settings endpoints stay functional either way).
+OPENRAG_SHOW_VLM_SETTINGS = os.getenv("OPENRAG_SHOW_VLM_SETTINGS", "true").lower() in (
+ "true",
+ "1",
+ "yes",
+)
+
# Show the "Make documents available to all users" (shared) toggle for COS bucket
# ingestion, independent of OPENRAG_SHOW_PROVIDER_INGEST_SETTINGS. Deployments that
# hide the general per-upload ingest tuning knobs (e.g. SaaS) still get just this
diff --git a/src/services/docling_service.py b/src/services/docling_service.py
index 5dcb199a5..141194b77 100644
--- a/src/services/docling_service.py
+++ b/src/services/docling_service.py
@@ -15,6 +15,7 @@ from config.settings import (
DOCLING_SERVE_VERIFY_SSL,
get_openrag_config,
)
+from utils.container_utils import transform_localhost_url
from utils.logging_config import get_logger
from utils.run_mode_utils import is_run_mode_on_prem, is_run_mode_saas
@@ -147,30 +148,127 @@ class DoclingService:
)
return DoclingService._default_client
- def _build_docling_options(
+ async def _build_docling_options_async(
self,
*,
ocr_override: bool | None = None,
picture_descriptions_override: bool | None = None,
) -> dict[str, Any]:
- """Build the options payload for docling from OpenRAG configs.
+ """Build the options payload for docling from OpenRAG configs, incorporating VLM settings if enabled."""
+ from services.watsonx_iam import WatsonxIamError, get_iam_token
- Per-request overrides take precedence over saved Knowledge settings.
- """
config = get_openrag_config()
knowledge_config = config.knowledge
+ is_ocr_enabled = ocr_override if ocr_override is not None else knowledge_config.ocr
+ is_pic_desc_enabled = (
+ picture_descriptions_override
+ if picture_descriptions_override is not None
+ else knowledge_config.picture_descriptions
+ )
+
preset = get_docling_preset_configs(
table_structure=knowledge_config.table_structure,
- ocr=ocr_override if ocr_override is not None else knowledge_config.ocr,
- picture_descriptions=(
- picture_descriptions_override
- if picture_descriptions_override is not None
- else knowledge_config.picture_descriptions
- ),
+ ocr=is_ocr_enabled,
+ picture_descriptions=is_pic_desc_enabled,
)
options = {"to_formats": "json", "image_export_mode": "placeholder", **preset}
+
+ # If picture descriptions are enabled, configure custom/local VLM model
+ if is_pic_desc_enabled and knowledge_config.vlm_enabled:
+ provider = knowledge_config.vlm_provider
+ vlm_model = knowledge_config.vlm_model
+ prompt = knowledge_config.vlm_prompt
+
+ if provider == "local":
+ options["picture_description_local"] = {
+ "repo_id": vlm_model,
+ "prompt": prompt,
+ }
+ elif provider == "watsonx":
+ watsonx = config.providers.watsonx
+ if not (watsonx.api_key and watsonx.endpoint and watsonx.project_id):
+ raise DoclingServeError(
+ "Docling VLM is enabled but the watsonx provider is not fully "
+ "configured (api key, endpoint, and project id are required)"
+ )
+ try:
+ token = await get_iam_token(watsonx.api_key)
+ except httpx.RequestError as e:
+ raise DoclingTransientError(
+ f"watsonx IAM token exchange network error: {str(e)}"
+ ) from e
+ except WatsonxIamError as e:
+ raise DoclingServeError(str(e)) from e
+
+ url = (
+ f"{watsonx.endpoint.rstrip('/')}/ml/v1/text/chat"
+ f"?version={knowledge_config.vlm_watsonx_api_version}"
+ )
+ options["picture_description_api"] = {
+ "url": url,
+ "headers": {"Authorization": f"Bearer {token}"},
+ "params": {
+ "model_id": vlm_model,
+ "project_id": watsonx.project_id,
+ "max_tokens": knowledge_config.vlm_max_tokens,
+ },
+ "prompt": prompt,
+ }
+ elif provider == "anthropic":
+ anthropic = config.providers.anthropic
+ if not anthropic.api_key:
+ raise DoclingServeError(
+ "Docling VLM is enabled but the Anthropic provider is not configured"
+ )
+ url = "https://api.anthropic.com/v1/messages"
+ options["picture_description_api"] = {
+ "url": url,
+ "headers": {
+ "x-api-key": anthropic.api_key,
+ "anthropic-version": "2023-06-01",
+ "content-type": "application/json",
+ },
+ "params": {
+ "model": vlm_model,
+ "max_tokens": knowledge_config.vlm_max_tokens,
+ },
+ "prompt": prompt,
+ }
+ elif provider == "ollama":
+ ollama = config.providers.ollama
+ if not ollama.endpoint:
+ raise DoclingServeError(
+ "Docling VLM is enabled but the Ollama provider is not configured"
+ )
+ url = f"{transform_localhost_url(ollama.endpoint).rstrip('/')}/v1/chat/completions"
+ options["picture_description_api"] = {
+ "url": url,
+ "headers": {},
+ "params": {
+ "model": vlm_model,
+ "max_completion_tokens": knowledge_config.vlm_max_tokens,
+ },
+ "prompt": prompt,
+ }
+ else: # openai or default
+ openai = config.providers.openai
+ if not openai.api_key:
+ raise DoclingServeError(
+ "Docling VLM is enabled but the OpenAI provider is not configured"
+ )
+ url = "https://api.openai.com/v1/chat/completions"
+ options["picture_description_api"] = {
+ "url": url,
+ "headers": {"Authorization": f"Bearer {openai.api_key}"},
+ "params": {
+ "model": vlm_model,
+ "max_completion_tokens": knowledge_config.vlm_max_tokens,
+ },
+ "prompt": prompt,
+ }
+
return options
def _get_auth_headers(
@@ -195,22 +293,24 @@ class DoclingService:
"""
Upload a file to Docling Serve asynchronously using direct multipart/form-data upload.
"""
- options = self._build_docling_options(
+ options = await self._build_docling_options_async(
ocr_override=ocr,
picture_descriptions_override=picture_descriptions,
)
+
headers = self._get_auth_headers(user_id, auth_header)
# Docling serve async multipart endpoint /v1/convert/file/async
- # Options are passed as form data
+ # Options are passed as form data; dict-valued options (e.g.
+ # picture_description_local, vlm_pipeline_model_api) go as JSON strings.
data = {
- k: str(v).lower() if isinstance(v, bool) else v
+ k: json.dumps(v)
+ if isinstance(v, dict)
+ else str(v).lower()
+ if isinstance(v, bool)
+ else v
for k, v in options.items()
- if not isinstance(v, dict)
- } # picture_description_local needs to be JSON if it's a dict
-
- if "picture_description_local" in options:
- data["picture_description_local"] = json.dumps(options["picture_description_local"])
+ }
files = {"files": (filename, file_content)}
diff --git a/src/services/models_service.py b/src/services/models_service.py
index 3311e55d8..0c498018b 100644
--- a/src/services/models_service.py
+++ b/src/services/models_service.py
@@ -1,14 +1,13 @@
-import httpx
import asyncio
-from typing import Dict, List, Optional
+
+import httpx
+
+from config.embedding_constants import OPENAI_DEFAULT_EMBEDDING_MODEL, OPENAI_EMBEDDING_MODEL_PREFIX
from config.model_constants import (
ANTHROPIC_DEFAULT_LANGUAGE_MODEL,
- ANTHROPIC_VALIDATION_MODELS,
OLLAMA_DEFAULT_LANGUAGE_MODEL_PATTERN,
OPENAI_DEFAULT_LANGUAGE_MODEL,
- OPENAI_VALIDATION_MODELS,
)
-from config.embedding_constants import OPENAI_DEFAULT_EMBEDDING_MODEL, OPENAI_EMBEDDING_MODEL_PREFIX
from utils.container_utils import transform_localhost_url
from utils.logging_config import get_logger
@@ -23,16 +22,15 @@ class UnknownEmbeddingProvider(Exception):
unroutable request into LiteLLM's retry loop."""
def __init__(self, model_name: str):
- super().__init__(
- f"No configured provider can serve embedding model '{model_name}'"
- )
+ super().__init__(f"No configured provider can serve embedding model '{model_name}'")
self.model_name = model_name
+
class ModelsService:
"""Service for fetching available models from different AI providers and managing a model registry."""
# Registry for caching model-to-provider mapping
- _model_provider_registry: Dict[str, str] = {}
+ _model_provider_registry: dict[str, str] = {}
_registry_lock = asyncio.Lock()
def __init__(self):
@@ -58,23 +56,25 @@ class ModelsService:
async def update_model_registry(self):
"""Fetch all models from all providers and update the internal registry.
-
+
This method calls provider-specific methods to get the list of available
models and stores the mapping in a registry for fast lookup.
"""
from config.config_manager import config_manager
-
+
async with self._registry_lock:
try:
config = config_manager.get_config()
new_registry = {}
# Fetch from providers
-
+
# OpenAI
if config.providers.openai.api_key:
try:
- res = await self.get_openai_models(config.providers.openai.api_key, update_index=False)
+ res = await self.get_openai_models(
+ config.providers.openai.api_key, update_index=False
+ )
self.add_models(res, "openai", new_registry)
except Exception as e:
logger.debug(f"Could not fetch OpenAI models for registry: {str(e)}")
@@ -82,7 +82,9 @@ class ModelsService:
# Anthropic
if config.providers.anthropic.api_key:
try:
- res = await self.get_anthropic_models(config.providers.anthropic.api_key, update_index=False)
+ res = await self.get_anthropic_models(
+ config.providers.anthropic.api_key, update_index=False
+ )
self.add_models(res, "anthropic", new_registry)
except Exception as e:
logger.debug(f"Could not fetch Anthropic models for registry: {str(e)}")
@@ -90,11 +92,13 @@ class ModelsService:
# Ollama
if config.providers.ollama.endpoint:
try:
- res = await self.get_ollama_models(config.providers.ollama.endpoint, update_index=False)
+ res = await self.get_ollama_models(
+ config.providers.ollama.endpoint, update_index=False
+ )
self.add_models(res, "ollama", new_registry)
except Exception as e:
logger.debug(f"Could not fetch Ollama models for registry: {str(e)}")
-
+
# WatsonX
if config.providers.watsonx.api_key:
try:
@@ -102,22 +106,24 @@ class ModelsService:
config.providers.watsonx.endpoint,
config.providers.watsonx.api_key,
config.providers.watsonx.project_id,
- update_index=False
+ update_index=False,
)
self.add_models(res, "watsonx", new_registry)
except Exception as e:
logger.debug(f"Could not fetch WatsonX models for registry: {str(e)}")
ModelsService._model_provider_registry = new_registry
- logger.info(f"Model registry updated: {len(ModelsService._model_provider_registry)} models registered")
-
+ logger.info(
+ f"Model registry updated: {len(ModelsService._model_provider_registry)} models registered"
+ )
+
except Exception as e:
logger.error(f"Error updating model registry: {str(e)}")
async def get_litellm_model_name(
self,
model_name: str,
- provider: Optional[str] = None,
+ provider: str | None = None,
strict: bool = False,
) -> str:
"""Resolve ``model_name`` to a LiteLLM-routable string.
@@ -160,7 +166,51 @@ class ModelsService:
return f"{provider_lower}/{model_name}" if provider_lower != "openai" else model_name
- async def get_openai_models(self, api_key: str, update_index: bool = True) -> Dict[str, List[Dict[str, str]]]:
+ def _openai_supports_images(self, model_id: str) -> bool:
+ model_lower = model_id.lower()
+ if "text-embedding" in model_lower:
+ return False
+ if "o1-mini" in model_lower or "o1-preview" in model_lower:
+ return False
+ return any(
+ x in model_lower for x in ["gpt-4o", "gpt-5", "vision", "o3", "o4", "gpt-4-turbo", "o1"]
+ )
+
+ def _anthropic_supports_images(self, model_data: dict) -> bool:
+ capabilities = model_data.get("capabilities", {})
+ if isinstance(capabilities, dict):
+ image_input = capabilities.get("image_input", {})
+ if isinstance(image_input, dict) and image_input.get("supported") is True:
+ return True
+ # Fallback to model name matching if capabilities is not populated
+ model_id = model_data.get("id", "").lower()
+ return any(x in model_id for x in ["claude-3", "claude-4", "opus", "sonnet", "haiku"])
+
+ def _watsonx_supports_images(self, model_data: dict) -> bool:
+ model_id = model_data.get("model_id", "").lower()
+ if any(x in model_id for x in ["vision", "pixtral", "qwen-vl", "qwen2-vl", "multimodal"]):
+ return True
+ short_desc = model_data.get("short_description", "").lower()
+ if "vision" in short_desc or "multimodal" in short_desc or "image" in short_desc:
+ return True
+ return False
+
+ def _ollama_supports_images(self, json_data: dict) -> bool:
+ capabilities = json_data.get("capabilities", [])
+ if "vision" in capabilities:
+ return True
+ model_info = json_data.get("model_info", {})
+ if any("vision" in key.lower() or "projector" in key.lower() for key in model_info.keys()):
+ return True
+ details = json_data.get("details", {})
+ families = details.get("families", []) or []
+ if any("clip" in str(fam).lower() or "vision" in str(fam).lower() for fam in families):
+ return True
+ return False
+
+ async def get_openai_models(
+ self, api_key: str, update_index: bool = True
+ ) -> dict[str, list[dict[str, str]]]:
"""Fetch available models from OpenAI API with lightweight validation"""
try:
headers = {
@@ -186,41 +236,35 @@ class ModelsService:
for model in models:
model_id = model.get("id", "")
- # Language models (GPT models)
- if model_id in OPENAI_VALIDATION_MODELS:
- language_models.append(
+ # Embedding models
+ if OPENAI_EMBEDDING_MODEL_PREFIX in model_id or "text-similarity-" in model_id:
+ embedding_models.append(
{
"value": model_id,
"label": model_id,
- "default": model_id == OPENAI_DEFAULT_LANGUAGE_MODEL,
+ "default": model_id == OPENAI_DEFAULT_EMBEDDING_MODEL,
}
)
-
- # Embedding models
- elif OPENAI_EMBEDDING_MODEL_PREFIX in model_id:
- embedding_models.append(
+ # Language models (GPT and o1/o3/chatgpt models)
+ elif (
+ model_id.startswith(("gpt-", "o1-", "o3-", "chatgpt-"))
+ and "-moderation" not in model_id
+ ):
+ language_models.append(
{
"value": model_id,
"label": model_id,
- "default": model_id == OPENAI_DEFAULT_EMBEDDING_MODEL,
+ "default": model_id == OPENAI_DEFAULT_LANGUAGE_MODEL,
+ "supports_images": self._openai_supports_images(model_id),
}
)
# Sort by name and ensure defaults are first
- language_models.sort(
- key=lambda x: (not x.get("default", False), x["value"])
- )
- embedding_models.sort(
- key=lambda x: (not x.get("default", False), x["value"])
- )
+ language_models.sort(key=lambda x: (not x.get("default", False), x["value"]))
+ embedding_models.sort(key=lambda x: (not x.get("default", False), x["value"]))
if not language_models:
- logger.warning(
- "OpenAI API key is valid but no language models matched the validation list. "
- "The API returned %d models, none matched OPENAI_VALIDATION_MODELS. "
- "This may indicate a model naming scheme change.",
- len(models),
- )
+ logger.warning("OpenAI API key is valid but no language models were found.")
if not embedding_models:
logger.warning(
"OpenAI API key is valid but no embedding models were found matching prefix '%s'.",
@@ -248,7 +292,9 @@ class ModelsService:
logger.error(f"Error fetching OpenAI models: {str(e)}")
raise
- async def get_anthropic_models(self, api_key: str, update_index: bool = True) -> Dict[str, List[Dict[str, str]]]:
+ async def get_anthropic_models(
+ self, api_key: str, update_index: bool = True
+ ) -> dict[str, list[dict[str, str]]]:
"""Fetch available models from Anthropic API"""
try:
headers = {
@@ -274,27 +320,21 @@ class ModelsService:
for model in models:
model_id = model.get("id", "")
-
- if model_id in ANTHROPIC_VALIDATION_MODELS:
- language_models.append(
- {
- "value": model_id,
- "label": model_id,
- "default": model_id == ANTHROPIC_DEFAULT_LANGUAGE_MODEL,
- }
- )
+ language_models.append(
+ {
+ "value": model_id,
+ "label": model.get("display_name", model_id),
+ "default": model_id == ANTHROPIC_DEFAULT_LANGUAGE_MODEL,
+ "supports_images": self._anthropic_supports_images(model),
+ }
+ )
# Sort by default first, then by name
- language_models.sort(
- key=lambda x: (not x.get("default", False), x["value"])
- )
+ language_models.sort(key=lambda x: (not x.get("default", False), x["value"]))
if not language_models:
logger.warning(
- "Anthropic API key is valid but no models matched the validation list. "
- "The API returned %d models, none matched ANTHROPIC_VALIDATION_MODELS. "
- "This may indicate a model naming scheme change.",
- len(models),
+ "Anthropic API key is valid but no models were returned.",
)
result = {
@@ -318,7 +358,7 @@ class ModelsService:
async def get_ollama_models(
self, endpoint: str = None, update_index: bool = True
- ) -> Dict[str, List[Dict[str, str]]]:
+ ) -> dict[str, list[dict[str, str]]]:
"""Fetch available models from Ollama API with tool calling capabilities for language models"""
try:
ollama_url = transform_localhost_url(endpoint)
@@ -359,16 +399,12 @@ class ModelsService:
# Check model capabilities
payload = {"model": model_name}
try:
- show_response = await client.post(
- show_url, json=payload, timeout=10.0
- )
+ show_response = await client.post(show_url, json=payload, timeout=10.0)
show_response.raise_for_status()
json_data = show_response.json()
capabilities = json_data.get(JSON_CAPABILITIES_KEY, [])
- logger.debug(
- f"Model: {model_name}, Capabilities: {capabilities}"
- )
+ logger.debug(f"Model: {model_name}, Capabilities: {capabilities}")
# Check if model has embedding capability
has_embedding = "embedding" in capabilities
@@ -393,6 +429,7 @@ class ModelsService:
"label": model_name,
"default": OLLAMA_DEFAULT_LANGUAGE_MODEL_PATTERN
in model_name.lower(),
+ "supports_images": self._ollama_supports_images(json_data),
}
)
if not capabilities and not has_embedding:
@@ -414,16 +451,10 @@ class ModelsService:
continue
# Remove duplicates and sort
- language_models = list(
- {m["value"]: m for m in language_models}.values()
- )
- embedding_models = list(
- {m["value"]: m for m in embedding_models}.values()
- )
+ language_models = list({m["value"]: m for m in language_models}.values())
+ embedding_models = list({m["value"]: m for m in embedding_models}.values())
- language_models.sort(
- key=lambda x: (not x.get("default", False), x["value"])
- )
+ language_models.sort(key=lambda x: (not x.get("default", False), x["value"]))
embedding_models.sort(key=lambda x: x["value"])
logger.info(
@@ -445,8 +476,12 @@ class ModelsService:
raise
async def get_ibm_models(
- self, endpoint: str = None, api_key: str = None, project_id: str = None, update_index: bool = True
- ) -> Dict[str, List[Dict[str, str]]]:
+ self,
+ endpoint: str = None,
+ api_key: str = None,
+ project_id: str = None,
+ update_index: bool = True,
+ ) -> dict[str, list[dict[str, str]]]:
"""Fetch available models from IBM Watson API"""
try:
# Use provided endpoint or default
@@ -522,6 +557,7 @@ class ModelsService:
"value": model_id,
"label": model_name or model_id,
"default": i == 0, # First model is default
+ "supports_images": self._watsonx_supports_images(model),
}
)
else:
diff --git a/src/services/search_service.py b/src/services/search_service.py
index 0a49c5e23..0f9585518 100644
--- a/src/services/search_service.py
+++ b/src/services/search_service.py
@@ -25,11 +25,11 @@ _global_search_service = None
def _is_exact_token_query(query: str) -> bool:
- """Return True for code/token-like queries that should not allow partial fuzzy matches."""
- if not query or len(query.strip()) < 3:
- return False
-
+ """Return True for code/token-like queries (identifiers, versions, SKUs)
+ that warrant exact-file narrowing; ordinary prose returns False."""
query = query.strip()
+ if not query or len(query) < 3:
+ return False
if re.search(r"[^a-zA-Z0-9\s]", query):
return True
@@ -37,6 +37,75 @@ def _is_exact_token_query(query: str) -> bool:
return bool(re.search(r"[a-zA-Z]", query) and re.search(r"\d", query))
+def _apply_exact_match_file_filter(
+ query: str,
+ chunks: list[dict[str, Any]],
+ aggregations: dict[str, Any],
+ *,
+ is_wildcard_match_all: bool,
+) -> tuple[list[dict[str, Any]], dict[str, Any]]:
+ """If a token-like query appears verbatim in a subset of files, prefer those files.
+
+ This avoids broad semantic spillover for unique lookups. Narrowing only
+ applies to token-like queries (see _is_exact_token_query); multi-word prose
+ searches keep their full hybrid-ranked results. When no file contains the
+ query verbatim, the ranked results are returned untouched — hits must never
+ be discarded just because the query looks token-like (e.g. "chunk_overlap",
+ "v1.2", "SKU-1234").
+ """
+ normalized_query = query.strip().lower()
+ if (
+ not normalized_query
+ or is_wildcard_match_all
+ or len(normalized_query) < 4
+ or not _is_exact_token_query(normalized_query)
+ ):
+ return chunks, aggregations
+
+ exact_files = {
+ filename
+ for chunk in chunks
+ for filename in [chunk.get("filename")]
+ if isinstance(filename, str)
+ and (
+ normalized_query in filename.lower()
+ or (
+ isinstance(chunk.get("text"), str)
+ and normalized_query in chunk.get("text", "").lower()
+ )
+ )
+ }
+ if not exact_files:
+ return chunks, aggregations
+
+ # Filter to only chunks from files with exact matches
+ chunks = [chunk for chunk in chunks if chunk.get("filename") in exact_files]
+
+ def _build_terms_agg(field: str) -> dict[str, Any]:
+ counts = Counter(
+ value
+ for chunk in chunks
+ for value in [chunk.get(field)]
+ if isinstance(value, str) and value
+ )
+ return {
+ "doc_count_error_upper_bound": 0,
+ "sum_other_doc_count": 0,
+ "buckets": [{"key": key, "doc_count": count} for key, count in counts.most_common()],
+ }
+
+ # Keep aggregations consistent with the post-filtered result set.
+ aggregations = {
+ **aggregations,
+ "data_sources": _build_terms_agg("filename"),
+ "document_types": _build_terms_agg("mimetype"),
+ "owners": _build_terms_agg("owner"),
+ "connector_types": _build_terms_agg("connector_type"),
+ "embedding_models": _build_terms_agg("embedding_model"),
+ }
+ return chunks, aggregations
+
+
def register_search_service(service: "SearchService") -> None:
"""
Explicitly register the active search service for the @tool wrapper.
@@ -587,58 +656,12 @@ class SearchService:
# If query text appears verbatim in one subset of files, prefer those files
# to avoid broad semantic spillover for unique lookups.
- normalized_query = query.strip().lower()
- aggregations = results.get("aggregations", {})
- if normalized_query and not is_wildcard_match_all and len(normalized_query) >= 4:
- exact_files = {
- filename
- for chunk in chunks
- for filename in [chunk.get("filename")]
- if isinstance(filename, str)
- and (
- normalized_query in filename.lower()
- or (
- isinstance(chunk.get("text"), str)
- and normalized_query in chunk.get("text", "").lower()
- )
- )
- }
-
- # Determine if we should apply exact-token filtering
- should_filter_exact = bool(exact_files) or _is_exact_token_query(query)
-
- if should_filter_exact:
- if exact_files:
- # Filter to only chunks from files with exact matches
- chunks = [chunk for chunk in chunks if chunk.get("filename") in exact_files]
- else:
- # No exact matches found for a token-like query - return empty results
- chunks = []
-
- def _build_terms_agg(field: str) -> dict[str, Any]:
- counts = Counter(
- value
- for chunk in chunks
- for value in [chunk.get(field)]
- if isinstance(value, str) and value
- )
- return {
- "doc_count_error_upper_bound": 0,
- "sum_other_doc_count": 0,
- "buckets": [
- {"key": key, "doc_count": count} for key, count in counts.most_common()
- ],
- }
-
- # Keep aggregations consistent with the post-filtered result set.
- aggregations = {
- **aggregations,
- "data_sources": _build_terms_agg("filename"),
- "document_types": _build_terms_agg("mimetype"),
- "owners": _build_terms_agg("owner"),
- "connector_types": _build_terms_agg("connector_type"),
- "embedding_models": _build_terms_agg("embedding_model"),
- }
+ chunks, aggregations = _apply_exact_match_file_filter(
+ query,
+ chunks,
+ results.get("aggregations", {}),
+ is_wildcard_match_all=is_wildcard_match_all,
+ )
# Return both transformed results and aggregations. Surface degraded
# semantic-search signals so the UI can show a non-fatal warning
diff --git a/src/services/watsonx_iam.py b/src/services/watsonx_iam.py
new file mode 100644
index 000000000..73fc8ac8c
--- /dev/null
+++ b/src/services/watsonx_iam.py
@@ -0,0 +1,69 @@
+"""Cached IBM Cloud IAM token exchange for watsonx.ai.
+
+watsonx.ai endpoints require a short-lived IAM bearer token (~1h TTL) obtained
+from an IBM Cloud API key. This module exchanges and caches tokens so callers
+(e.g. the Docling VLM pipeline) don't hit the IAM endpoint on every request.
+"""
+
+import asyncio
+import time
+
+import httpx
+
+from utils.logging_config import get_logger
+
+logger = get_logger(__name__)
+
+IAM_TOKEN_URL = "https://iam.cloud.ibm.com/identity/token"
+
+# Refresh tokens that expire within this margin so in-flight conversions
+# don't start with a nearly-expired token.
+_REFRESH_MARGIN_SECONDS = 300
+
+_cache: dict[str, tuple[str, float]] = {} # api_key -> (token, expiry_epoch)
+_lock = asyncio.Lock()
+
+
+class WatsonxIamError(Exception):
+ """Raised when the IAM token exchange fails with a non-transient error."""
+
+
+async def get_iam_token(api_key: str, http_timeout: float = 15.0) -> str:
+ """Exchange an IBM Cloud API key for an IAM bearer token, with caching.
+
+ Raises:
+ httpx.RequestError: network-level failure (caller may retry).
+ WatsonxIamError: IAM rejected the exchange or returned no token.
+ """
+ async with _lock:
+ cached = _cache.get(api_key)
+ if cached and cached[1] - time.time() > _REFRESH_MARGIN_SECONDS:
+ return cached[0]
+
+ async with httpx.AsyncClient(timeout=http_timeout) as client:
+ response = await client.post(
+ IAM_TOKEN_URL,
+ headers={
+ "Content-Type": "application/x-www-form-urlencoded",
+ "Accept": "application/json",
+ },
+ data={
+ "grant_type": "urn:ibm:params:oauth:grant-type:apikey",
+ "apikey": api_key,
+ },
+ )
+
+ if response.status_code != 200:
+ raise WatsonxIamError(
+ f"IBM IAM token exchange failed: HTTP {response.status_code} {response.text[:200]}"
+ )
+
+ data = response.json()
+ token = data.get("access_token")
+ if not token:
+ raise WatsonxIamError("IBM IAM response did not contain access_token")
+
+ expiry = data.get("expiration") or (time.time() + float(data.get("expires_in", 3600)))
+ _cache[api_key] = (token, float(expiry))
+ logger.debug("Obtained IBM IAM token", expires_at=expiry)
+ return token
diff --git a/src/tui/managers/docling_manager.py b/src/tui/managers/docling_manager.py
index 441c803a2..06a033a3c 100644
--- a/src/tui/managers/docling_manager.py
+++ b/src/tui/managers/docling_manager.py
@@ -338,10 +338,17 @@ class DoclingManager:
env = os.environ.copy()
- # Add docling serve environment variable to change Tenant Id header name
- # so that it's not used to enforce ownership locally.
+ # Disable PyTorch model compilation to prevent Dynamo/AttributeError issues on CPU/Mac
+ env["TORCH_COMPILE_DISABLE"] = "1"
+ env["TORCH_DYNAMO_DISABLE"] = "1"
+
+ # Add docling serve environment variables
if "DOCLING_SERVE_ENG_RAY_TENANT_ID_HEADER" not in env:
env["DOCLING_SERVE_ENG_RAY_TENANT_ID_HEADER"] = "X-Docling-Tenant-Id"
+ if "DOCLING_SERVE_ENABLE_REMOTE_SERVICES" not in env:
+ env["DOCLING_SERVE_ENABLE_REMOTE_SERVICES"] = "true"
+ if "DOCLING_SERVE_ALLOW_CUSTOM_VLM_CONFIG" not in env:
+ env["DOCLING_SERVE_ALLOW_CUSTOM_VLM_CONFIG"] = "true"
self._process = subprocess.Popen(
cmd,
diff --git a/tests/unit/test_docling_service.py b/tests/unit/test_docling_service.py
index c474aebd0..78f84d536 100644
--- a/tests/unit/test_docling_service.py
+++ b/tests/unit/test_docling_service.py
@@ -147,6 +147,7 @@ async def test_upload_success(docling_service, mock_httpx_client):
mock_config.knowledge.table_structure = False
mock_config.knowledge.ocr = False
mock_config.knowledge.picture_descriptions = False
+ mock_config.knowledge.vlm_enabled = False
mock_get_config.return_value = mock_config
task_id = await docling_service.upload_to_docling_direct_async("test.pdf", b"data")
@@ -165,7 +166,14 @@ async def test_upload_http_error(docling_service, mock_httpx_client):
"""Raises exception if upload returns non-200."""
mock_httpx_client.post.return_value = _make_response(400)
- with patch("services.docling_service.get_openrag_config"):
+ with patch("services.docling_service.get_openrag_config") as mock_get_config:
+ mock_config = MagicMock()
+ mock_config.knowledge.table_structure = False
+ mock_config.knowledge.ocr = False
+ mock_config.knowledge.picture_descriptions = False
+ mock_config.knowledge.vlm_enabled = False
+ mock_get_config.return_value = mock_config
+
with pytest.raises(httpx.HTTPStatusError):
await docling_service.upload_to_docling_direct_async("test.pdf", b"data")
@@ -173,15 +181,17 @@ async def test_upload_http_error(docling_service, mock_httpx_client):
# ── Configuration Logic ─────────────────────────────────────────────
-def test_build_docling_options_toggles(docling_service):
+async def test_build_docling_options_toggles(docling_service):
"""Correctly maps OpenRAG config to Docling options."""
mock_config = MagicMock()
mock_config.knowledge.table_structure = True
mock_config.knowledge.ocr = True
mock_config.knowledge.picture_descriptions = False
+ mock_config.knowledge.vlm_enabled = False
with patch("services.docling_service.get_openrag_config", return_value=mock_config):
- options = docling_service._build_docling_options()
+ options = await docling_service._build_docling_options_async()
assert options["do_table_structure"] is True
assert options["do_ocr"] is True
@@ -212,3 +222,113 @@ def test_init_default_url():
with patch("services.docling_service.DOCLING_SERVE_URL", "http://default:5001"):
service = DoclingService()
assert service.docling_url == "http://default:5001"
+
+
+# ── VLM Pipeline Options ────────────────────────────────────────────
+
+
+def _vlm_mock_config(provider: str) -> MagicMock:
+ mock_config = MagicMock()
+ k = mock_config.knowledge
+ k.vlm_enabled = True
+ k.vlm_provider = provider
+ k.vlm_model = "gpt-4o" if provider == "openai" else "meta-llama/llama-vision"
+ k.vlm_prompt = "Extract all text."
+ k.vlm_response_format = "markdown"
+ k.vlm_max_tokens = 5000
+ k.vlm_concurrency = 4
+ k.vlm_timeout = 120
+ k.vlm_openai_url = "https://api.openai.com/v1/chat/completions"
+ k.vlm_watsonx_api_version = "2023-05-29"
+ k.table_structure = False
+ k.ocr = False
+ k.picture_descriptions = True
+ mock_config.providers.openai.api_key = "sk-test"
+ mock_config.providers.watsonx.api_key = "wx-key"
+ mock_config.providers.watsonx.endpoint = "https://us-south.ml.cloud.ibm.com/"
+ mock_config.providers.watsonx.project_id = "proj-123"
+ return mock_config
+
+
+async def test_build_vlm_options_openai(docling_service):
+ """OpenAI VLM options carry the provider key and chat-completions params."""
+ mock_config = _vlm_mock_config("openai")
+ with patch("services.docling_service.get_openrag_config", return_value=mock_config):
+ options = await docling_service._build_docling_options_async()
+
+ assert options["do_picture_description"] is True
+ assert options["to_formats"] == "json"
+ api = options["picture_description_api"]
+ assert api["url"] == "https://api.openai.com/v1/chat/completions"
+ assert api["headers"]["Authorization"] == "Bearer sk-test"
+ assert api["params"] == {"model": "gpt-4o", "max_completion_tokens": 5000}
+ assert api["prompt"] == "Extract all text."
+
+
+async def test_build_vlm_options_watsonx(docling_service):
+ """watsonx VLM options exchange the API key for an IAM bearer token."""
+ mock_config = _vlm_mock_config("watsonx")
+ with (
+ patch("services.docling_service.get_openrag_config", return_value=mock_config),
+ patch("services.watsonx_iam.get_iam_token", new_callable=AsyncMock) as mock_token,
+ ):
+ mock_token.return_value = "iam-token"
+ options = await docling_service._build_docling_options_async()
+
+ mock_token.assert_awaited_once_with("wx-key")
+ api = options["picture_description_api"]
+ assert api["url"] == "https://us-south.ml.cloud.ibm.com/ml/v1/text/chat?version=2023-05-29"
+ assert api["headers"]["Authorization"] == "Bearer iam-token"
+ assert api["params"] == {
+ "model_id": "meta-llama/llama-vision",
+ "project_id": "proj-123",
+ "max_tokens": 5000,
+ }
+
+
+async def test_build_vlm_options_watsonx_unconfigured(docling_service):
+ """Raises DoclingServeError when watsonx provider is incomplete."""
+ mock_config = _vlm_mock_config("watsonx")
+ mock_config.providers.watsonx.project_id = ""
+ with patch("services.docling_service.get_openrag_config", return_value=mock_config):
+ with pytest.raises(DoclingServeError, match="watsonx provider is not fully"):
+ await docling_service._build_docling_options_async()
+
+
+async def test_build_vlm_options_ollama(docling_service):
+ """Ollama VLM options carry the provider endpoint and chat-completions params."""
+ mock_config = _vlm_mock_config("ollama")
+ mock_config.providers.ollama.endpoint = "http://localhost:11434"
+ mock_config.providers.ollama.configured = True
+ with patch("services.docling_service.get_openrag_config", return_value=mock_config):
+ options = await docling_service._build_docling_options_async()
+
+ assert options["do_picture_description"] is True
+ assert options["to_formats"] == "json"
+ api = options["picture_description_api"]
+ assert api["url"] == "http://localhost:11434/v1/chat/completions"
+ assert api["headers"] == {}
+ assert api["params"] == {"model": "meta-llama/llama-vision", "max_completion_tokens": 5000}
+ assert api["prompt"] == "Extract all text."
+
+
+async def test_upload_vlm_enabled_sends_vlm_form_fields(docling_service, mock_httpx_client):
+ """VLM upload sends custom picture description parameters."""
+ import json as json_lib
+
+ mock_httpx_client.post.return_value = _make_response(200, {"task_id": "vlm-task"})
+ mock_config = _vlm_mock_config("openai")
+ with patch("services.docling_service.get_openrag_config", return_value=mock_config):
+ task_id = await docling_service.upload_to_docling_direct_async("test.pdf", b"data")
+
+ assert task_id == "vlm-task"
+ _, kwargs = mock_httpx_client.post.call_args
+ data = kwargs.get("data", {})
+ assert data["do_picture_description"] == "true"
+ api = json_lib.loads(data["picture_description_api"])
+ assert api["params"]["model"] == "gpt-4o"
diff --git a/tests/unit/test_search_service_exact_filter.py b/tests/unit/test_search_service_exact_filter.py
new file mode 100644
index 000000000..b4545de2f
--- /dev/null
+++ b/tests/unit/test_search_service_exact_filter.py
@@ -0,0 +1,100 @@
+"""
+Unit tests for the exact-match file filter in services/search_service.py.
+
+Regression: token-like queries (e.g. "chunk_overlap", "v1.2", "SKU-1234")
+must not zero out ranked hybrid results when no file contains the query
+verbatim.
+"""
+
+import pytest
+
+from services.search_service import _apply_exact_match_file_filter
+
+
+def _chunk(filename: str, text: str, **extra) -> dict:
+ return {
+ "filename": filename,
+ "text": text,
+ "mimetype": "application/pdf",
+ "owner": "user-1",
+ "connector_type": "local",
+ "embedding_model": "text-embedding-3-small",
+ **extra,
+ }
+
+
+BASE_AGGS = {
+ "data_sources": {"buckets": [{"key": "report.pdf", "doc_count": 2}]},
+ "document_types": {"buckets": [{"key": "application/pdf", "doc_count": 2}]},
+}
+
+
[email protected]("query", ["chunk_overlap", "v1.2", "SKU-1234"])
+def test_token_like_query_without_exact_match_keeps_hybrid_results(query):
+ """Token-like queries must not clear ranked hits when nothing matches verbatim."""
+ chunks = [
+ _chunk("report.pdf", "Discussion of chunking strategies and overlap."),
+ _chunk("notes.md", "Semantic search configuration notes."),
+ ]
+
+ filtered, aggs = _apply_exact_match_file_filter(
+ query, chunks, BASE_AGGS, is_wildcard_match_all=False
+ )
+
+ assert filtered == chunks
+ assert aggs == BASE_AGGS
+
+
+def test_verbatim_match_in_text_restricts_to_matching_files():
+ chunks = [
+ _chunk("config-guide.pdf", "Set chunk_overlap to 200 for best results."),
+ _chunk("unrelated.pdf", "Quarterly financial summary."),
+ ]
+
+ filtered, aggs = _apply_exact_match_file_filter(
+ "chunk_overlap", chunks, BASE_AGGS, is_wildcard_match_all=False
+ )
+
+ assert [c["filename"] for c in filtered] == ["config-guide.pdf"]
+ assert aggs["data_sources"]["buckets"] == [{"key": "config-guide.pdf", "doc_count": 1}]
+
+
+def test_verbatim_match_in_filename_restricts_to_matching_files():
+ chunks = [
+ _chunk("SKU-1234-spec.pdf", "Product specification."),
+ _chunk("other.pdf", "Different product."),
+ ]
+
+ filtered, _ = _apply_exact_match_file_filter(
+ "SKU-1234", chunks, BASE_AGGS, is_wildcard_match_all=False
+ )
+
+ assert [c["filename"] for c in filtered] == ["SKU-1234-spec.pdf"]
+
+
+def test_multiword_prose_query_is_not_narrowed():
+ """Ordinary prose queries keep hybrid-ranked results even when one chunk
+ happens to contain the phrase verbatim — narrowing is for token-like
+ lookups only."""
+ chunks = [
+ _chunk("guide.pdf", "The quarterly revenue summary is attached below."),
+ _chunk("other.pdf", "Revenue grew on strong quarterly performance."),
+ ]
+
+ filtered, aggs = _apply_exact_match_file_filter(
+ "quarterly revenue summary", chunks, BASE_AGGS, is_wildcard_match_all=False
+ )
+
+ assert filtered == chunks
+ assert aggs == BASE_AGGS
+
+
+def test_wildcard_and_short_queries_are_untouched():
+ chunks = [_chunk("report.pdf", "text"), _chunk("notes.md", "more text")]
+
+ for query, wildcard in [("*", True), ("ab", False), ("", False)]:
+ filtered, aggs = _apply_exact_match_file_filter(
+ query, chunks, BASE_AGGS, is_wildcard_match_all=wildcard
+ )
+ assert filtered == chunks
+ assert aggs == BASE_AGGS