Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

fix: create env var to skip sample data ingestion, fix bugs on onboarding error handling (#963)

Lucas Oliveira7a73f54449e1
diff --git a/.env.example b/.env.example
index 4d36588fe..a18d9b934 100644
--- a/.env.example
+++ b/.env.example
@@ -3,6 +3,10 @@
# If unset or false, Langflow pipeline will be used (default: upload -> ingest -> delete)
DISABLE_INGEST_WITH_LANGFLOW=false
+# Set to false to skip ingesting sample data (pdfs) on startup
+# Default: true
+INGEST_SAMPLE_DATA=true
+
# Langflow HTTP timeout configuration (in seconds)
# For large documents (300+ pages), ingestion can take 30+ minutes
# Increase these values if you experience timeouts with very large PDFs
diff --git a/docker-compose.yml b/docker-compose.yml
index 633dbd21d..11870127c 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -62,6 +62,7 @@ services:
- LANGFLOW_INGEST_FLOW_ID=${LANGFLOW_INGEST_FLOW_ID}
- LANGFLOW_URL_INGEST_FLOW_ID=${LANGFLOW_URL_INGEST_FLOW_ID}
- DISABLE_INGEST_WITH_LANGFLOW=${DISABLE_INGEST_WITH_LANGFLOW:-false}
+ - INGEST_SAMPLE_DATA=${INGEST_SAMPLE_DATA:-true}
- NUDGES_FLOW_ID=${NUDGES_FLOW_ID}
- OPENSEARCH_PORT=9200
- OPENSEARCH_USERNAME=admin
diff --git a/frontend/app/api/mutations/useOnboardingMutation.ts b/frontend/app/api/mutations/useOnboardingMutation.ts
index 911e196ca..4672f4b48 100644
--- a/frontend/app/api/mutations/useOnboardingMutation.ts
+++ b/frontend/app/api/mutations/useOnboardingMutation.ts
@@ -21,9 +21,6 @@ export interface OnboardingVariables {
watsonx_endpoint?: string;
watsonx_project_id?: string;
ollama_endpoint?: string;
-
- // Sample data
- sample_data?: boolean;
}
interface OnboardingResponse {
diff --git a/frontend/app/onboarding/_components/advanced.tsx b/frontend/app/onboarding/_components/advanced.tsx
index b73993b79..39bab81e5 100644
--- a/frontend/app/onboarding/_components/advanced.tsx
+++ b/frontend/app/onboarding/_components/advanced.tsx
@@ -15,8 +15,6 @@ export function AdvancedOnboarding({
embeddingModel,
setLanguageModel,
setEmbeddingModel,
- sampleDataset,
- setSampleDataset,
}: {
icon?: React.ReactNode;
languageModels?: { value: string; label: string }[];
@@ -25,8 +23,6 @@ export function AdvancedOnboarding({
embeddingModel?: string;
setLanguageModel?: (model: string) => void;
setEmbeddingModel?: (model: string) => void;
- sampleDataset: boolean;
- setSampleDataset: (dataset: boolean) => void;
}) {
const hasEmbeddingModels =
embeddingModels !== undefined &&
diff --git a/frontend/app/onboarding/_components/anthropic-onboarding.tsx b/frontend/app/onboarding/_components/anthropic-onboarding.tsx
index d58fc1ba1..6d917c134 100644
--- a/frontend/app/onboarding/_components/anthropic-onboarding.tsx
+++ b/frontend/app/onboarding/_components/anthropic-onboarding.tsx
@@ -18,15 +18,11 @@ import { AdvancedOnboarding } from "./advanced";
export function AnthropicOnboarding({
setSettings,
- sampleDataset,
- setSampleDataset,
setIsLoadingModels,
isEmbedding = false,
hasEnvApiKey = false,
}: {
setSettings: Dispatch<SetStateAction<OnboardingVariables>>;
- sampleDataset: boolean;
- setSampleDataset: (dataset: boolean) => void;
setIsLoadingModels?: (isLoading: boolean) => void;
isEmbedding?: boolean;
hasEnvApiKey?: boolean;
@@ -57,9 +53,6 @@ export function AnthropicOnboarding({
languageModels,
embeddingModels,
} = useModelSelection(modelsData, isEmbedding);
- const handleSampleDatasetChange = (dataset: boolean) => {
- setSampleDataset(dataset);
- };
const handleGetFromEnvChange = (fromEnv: boolean) => {
setGetFromEnv(fromEnv);
@@ -144,9 +137,7 @@ export function AnthropicOnboarding({
embeddingModels={embeddingModels}
languageModel={languageModel}
embeddingModel={embeddingModel}
- sampleDataset={sampleDataset}
setLanguageModel={setLanguageModel}
- setSampleDataset={handleSampleDatasetChange}
setEmbeddingModel={setEmbeddingModel}
/>
</>
diff --git a/frontend/app/onboarding/_components/ibm-onboarding.tsx b/frontend/app/onboarding/_components/ibm-onboarding.tsx
index 023b3467b..ebf455b83 100644
--- a/frontend/app/onboarding/_components/ibm-onboarding.tsx
+++ b/frontend/app/onboarding/_components/ibm-onboarding.tsx
@@ -20,8 +20,6 @@ import { ModelSelector } from "./model-selector";
export function IBMOnboarding({
isEmbedding = false,
setSettings,
- sampleDataset,
- setSampleDataset,
setIsLoadingModels,
alreadyConfigured = false,
existingEndpoint,
@@ -30,8 +28,6 @@ export function IBMOnboarding({
}: {
isEmbedding?: boolean;
setSettings: Dispatch<SetStateAction<OnboardingVariables>>;
- sampleDataset: boolean;
- setSampleDataset: (dataset: boolean) => void;
setIsLoadingModels?: (isLoading: boolean) => void;
alreadyConfigured?: boolean;
existingEndpoint?: string;
@@ -125,10 +121,6 @@ export function IBMOnboarding({
setLanguageModel?.("");
};
- const handleSampleDatasetChange = (dataset: boolean) => {
- setSampleDataset(dataset);
- };
-
useEffect(() => {
setIsLoadingModels?.(isLoadingModels);
}, [isLoadingModels, setIsLoadingModels]);
@@ -161,7 +153,7 @@ export function IBMOnboarding({
options={alreadyConfigured ? [] : options}
value={endpoint}
custom
- onValueChange={alreadyConfigured ? () => {} : setEndpoint}
+ onValueChange={alreadyConfigured ? () => { } : setEndpoint}
searchPlaceholder="Search endpoint..."
noOptionsPlaceholder={
alreadyConfigured
@@ -280,10 +272,8 @@ export function IBMOnboarding({
embeddingModels={embeddingModels}
languageModel={languageModel}
embeddingModel={embeddingModel}
- sampleDataset={sampleDataset}
setLanguageModel={setLanguageModel}
setEmbeddingModel={setEmbeddingModel}
- setSampleDataset={handleSampleDatasetChange}
/>
</>
);
diff --git a/frontend/app/onboarding/_components/ollama-onboarding.tsx b/frontend/app/onboarding/_components/ollama-onboarding.tsx
index d5c7428c0..bcaff27ac 100644
--- a/frontend/app/onboarding/_components/ollama-onboarding.tsx
+++ b/frontend/app/onboarding/_components/ollama-onboarding.tsx
@@ -12,16 +12,12 @@ import { ModelSelector } from "./model-selector";
export function OllamaOnboarding({
setSettings,
- sampleDataset,
- setSampleDataset,
setIsLoadingModels,
isEmbedding = false,
alreadyConfigured = false,
existingEndpoint,
}: {
setSettings: Dispatch<SetStateAction<OnboardingVariables>>;
- sampleDataset: boolean;
- setSampleDataset: (dataset: boolean) => void;
setIsLoadingModels?: (isLoading: boolean) => void;
isEmbedding?: boolean;
alreadyConfigured?: boolean;
diff --git a/frontend/app/onboarding/_components/onboarding-card.tsx b/frontend/app/onboarding/_components/onboarding-card.tsx
index 5324e0504..0371a6f6d 100644
--- a/frontend/app/onboarding/_components/onboarding-card.tsx
+++ b/frontend/app/onboarding/_components/onboarding-card.tsx
@@ -10,7 +10,6 @@ import {
useOnboardingMutation,
} from "@/app/api/mutations/useOnboardingMutation";
import { useOnboardingRollbackMutation } from "@/app/api/mutations/useOnboardingRollbackMutation";
-import { useUpdateOnboardingStateMutation } from "@/app/api/mutations/useUpdateOnboardingStateMutation";
import { useGetSettingsQuery } from "@/app/api/queries/useGetSettingsQuery";
import { useGetTasksQuery } from "@/app/api/queries/useGetTasksQuery";
import type { ProviderHealthResponse } from "@/app/api/queries/useProviderHealthQuery";
@@ -66,8 +65,6 @@ const OnboardingCard = ({
isEmbedding ? "openai" : "anthropic",
);
- const [sampleDataset, setSampleDataset] = useState<boolean>(true);
-
const [isLoadingModels, setIsLoadingModels] = useState<boolean>(false);
const queryClient = useQueryClient();
@@ -211,20 +208,20 @@ const OnboardingCard = ({
task.status === "processing",
);
- // Check if any file failed in completed tasks
- const completedTasks = tasks.filter(
- (task) => task.status === "completed"
+ // Check if any task failed at the top level
+ const failedTask = tasks.find(
+ (task) => task.status === "failed" || task.status === "error",
);
// Check if any completed task has at least one failed file
- const taskWithFailedFile = completedTasks.find((task) => {
+ const completedTaskWithFailedFile = tasks.find((task) => {
// Must have files object
if (!task.files || typeof task.files !== "object") {
return false;
}
const fileEntries = Object.values(task.files);
-
+
// Must have at least one file
if (fileEntries.length === 0) {
return false;
@@ -232,65 +229,69 @@ const OnboardingCard = ({
// Check if any file has failed status
const hasFailedFile = fileEntries.some(
- (file) => file.status === "failed" || file.status === "error"
+ (file) => file.status === "failed" || file.status === "error",
);
return hasFailedFile;
});
+ const taskWithFailure = failedTask || completedTaskWithFailedFile;
+
// If any file failed, show error and jump back one step (like onboardingMutation.onError)
// Only handle if we haven't already handled this task
if (
- taskWithFailedFile &&
- !rollbackMutation.isPending &&
+ taskWithFailure &&
+ !rollbackMutation.isPending &&
!isCompleted &&
- !handledFailedTasksRef.current.has(taskWithFailedFile.task_id)
+ !handledFailedTasksRef.current.has(taskWithFailure.task_id)
) {
- console.error("File failed in task, jumping back one step", taskWithFailedFile);
-
+ console.error(
+ "Task failed, jumping back one step",
+ taskWithFailure,
+ );
+
// Mark this task as handled to prevent infinite loops
- handledFailedTasksRef.current.add(taskWithFailedFile.task_id);
-
+ handledFailedTasksRef.current.add(taskWithFailure.task_id);
+
// Extract error messages from failed files
const errorMessages: string[] = [];
- if (taskWithFailedFile.files) {
- Object.values(taskWithFailedFile.files).forEach((file) => {
- if ((file.status === "failed" || file.status === "error") && file.error) {
+ if (taskWithFailure.files) {
+ Object.values(taskWithFailure.files).forEach((file) => {
+ if (
+ (file.status === "failed" || file.status === "error") &&
+ file.error
+ ) {
errorMessages.push(file.error);
}
});
}
-
+
// Also check task-level error
- if (taskWithFailedFile.error) {
- errorMessages.push(taskWithFailedFile.error);
+ if (taskWithFailure.error) {
+ errorMessages.push(taskWithFailure.error);
}
-
+
// Use the first error message, or a generic message if no errors found
- const errorMessage = errorMessages.length > 0
- ? errorMessages[0]
- : "Sample data file failed to ingest. Please try again with a different configuration.";
-
+ const errorMessage =
+ errorMessages.length > 0
+ ? errorMessages[0]
+ : "Sample data ingestion failed. Please try again.";
+
// Set error message and jump back one step (exactly like onboardingMutation.onError)
setError(errorMessage);
setCurrentStep(totalSteps);
- // Jump back one step after 1 second (go back to the step before ingestion)
- // For embedding: totalSteps is 4, ingestion is step 3, so go back to step 2
- // For LLM: totalSteps is 3, ingestion is step 2, so go back to step 1
- setTimeout(() => {
- // Go back to the step before the last step (which is ingestion)
- const previousStep = totalSteps > 1 ? totalSteps - 2 : 0;
- setCurrentStep(previousStep);
- }, 1000);
+ rollbackMutation.mutate();
return;
}
- // If no active tasks and we've started onboarding, complete it
+ // If at least one processed file, no failures, and we've started onboarding, complete it
if (
- (!activeTasks || (activeTasks.processed_files ?? 0) > 0) &&
- tasks.length > 0 &&
+ (((!activeTasks || (activeTasks.successful_files ?? 0) > 0) &&
+ tasks.length > 0) ||
+ (tasks.length === 0 && currentStep === totalSteps - 1)) && // Complete because no files were ingested
!isCompleted &&
- !taskWithFailedFile
+ !taskWithFailure
+
) {
// Set to final step to show "Done"
setCurrentStep(totalSteps);
@@ -329,10 +330,7 @@ const OnboardingCard = ({
onError: (error) => {
setError(error.message);
setCurrentStep(totalSteps);
- // Reset to provider selection after 1 second
- setTimeout(() => {
- setCurrentStep(null);
- }, 1000);
+ rollbackMutation.mutate();
},
});
@@ -357,7 +355,6 @@ const OnboardingCard = ({
// Prepare onboarding data with provider-specific fields
const onboardingData: OnboardingVariables = {
- sample_data: sampleDataset,
};
// Set the provider field
@@ -447,8 +444,8 @@ const OnboardingCard = ({
value="anthropic"
className={cn(
error &&
- modelProvider === "anthropic" &&
- "data-[state=active]:border-destructive",
+ modelProvider === "anthropic" &&
+ "data-[state=active]:border-destructive",
)}
>
<TabTrigger
@@ -480,8 +477,8 @@ const OnboardingCard = ({
value="openai"
className={cn(
error &&
- modelProvider === "openai" &&
- "data-[state=active]:border-destructive",
+ modelProvider === "openai" &&
+ "data-[state=active]:border-destructive",
)}
>
<TabTrigger
@@ -510,8 +507,8 @@ const OnboardingCard = ({
value="watsonx"
className={cn(
error &&
- modelProvider === "watsonx" &&
- "data-[state=active]:border-destructive",
+ modelProvider === "watsonx" &&
+ "data-[state=active]:border-destructive",
)}
>
<TabTrigger
@@ -542,8 +539,8 @@ const OnboardingCard = ({
value="ollama"
className={cn(
error &&
- modelProvider === "ollama" &&
- "data-[state=active]:border-destructive",
+ modelProvider === "ollama" &&
+ "data-[state=active]:border-destructive",
)}
>
<TabTrigger
@@ -573,8 +570,6 @@ const OnboardingCard = ({
<TabsContent value="anthropic">
<AnthropicOnboarding
setSettings={setSettings}
- sampleDataset={sampleDataset}
- setSampleDataset={setSampleDataset}
setIsLoadingModels={setIsLoadingModels}
isEmbedding={isEmbedding}
hasEnvApiKey={
@@ -587,8 +582,6 @@ const OnboardingCard = ({
<TabsContent value="openai">
<OpenAIOnboarding
setSettings={setSettings}
- sampleDataset={sampleDataset}
- setSampleDataset={setSampleDataset}
setIsLoadingModels={setIsLoadingModels}
isEmbedding={isEmbedding}
hasEnvApiKey={
@@ -600,8 +593,6 @@ const OnboardingCard = ({
<TabsContent value="watsonx">
<IBMOnboarding
setSettings={setSettings}
- sampleDataset={sampleDataset}
- setSampleDataset={setSampleDataset}
setIsLoadingModels={setIsLoadingModels}
isEmbedding={isEmbedding}
alreadyConfigured={providerAlreadyConfigured && modelProvider === "watsonx"}
@@ -613,8 +604,6 @@ const OnboardingCard = ({
<TabsContent value="ollama">
<OllamaOnboarding
setSettings={setSettings}
- sampleDataset={sampleDataset}
- setSampleDataset={setSampleDataset}
setIsLoadingModels={setIsLoadingModels}
isEmbedding={isEmbedding}
alreadyConfigured={providerAlreadyConfigured && modelProvider === "ollama"}
@@ -641,8 +630,8 @@ const OnboardingCard = ({
{isLoadingModels
? "Loading models..."
: !!settings.llm_model &&
- !!settings.embedding_model &&
- !isDoclingHealthy
+ !!settings.embedding_model &&
+ !isDoclingHealthy
? "docling-serve must be running to continue"
: "Please fill in all required fields"}
</TooltipContent>
diff --git a/frontend/app/onboarding/_components/openai-onboarding.tsx b/frontend/app/onboarding/_components/openai-onboarding.tsx
index d7272cde5..be3996901 100644
--- a/frontend/app/onboarding/_components/openai-onboarding.tsx
+++ b/frontend/app/onboarding/_components/openai-onboarding.tsx
@@ -18,16 +18,12 @@ import { AdvancedOnboarding } from "./advanced";
export function OpenAIOnboarding({
setSettings,
- sampleDataset,
- setSampleDataset,
setIsLoadingModels,
isEmbedding = false,
hasEnvApiKey = false,
alreadyConfigured = false,
}: {
setSettings: Dispatch<SetStateAction<OnboardingVariables>>;
- sampleDataset: boolean;
- setSampleDataset: (dataset: boolean) => void;
setIsLoadingModels?: (isLoading: boolean) => void;
isEmbedding?: boolean;
hasEnvApiKey?: boolean;
@@ -48,8 +44,8 @@ export function OpenAIOnboarding({
getFromEnv
? { apiKey: "" }
: debouncedApiKey
- ? { apiKey: debouncedApiKey }
- : undefined,
+ ? { apiKey: debouncedApiKey }
+ : undefined,
{
// Only validate when the user opts in (env) or provides a key.
// If a key was previously configured, let the user decide to reuse or replace it
@@ -66,9 +62,6 @@ export function OpenAIOnboarding({
languageModels,
embeddingModels,
} = useModelSelection(modelsData, isEmbedding);
- const handleSampleDatasetChange = (dataset: boolean) => {
- setSampleDataset(dataset);
- };
const handleGetFromEnvChange = (fromEnv: boolean) => {
setGetFromEnv(fromEnv);
@@ -167,9 +160,7 @@ export function OpenAIOnboarding({
embeddingModels={embeddingModels}
languageModel={languageModel}
embeddingModel={embeddingModel}
- sampleDataset={sampleDataset}
setLanguageModel={setLanguageModel}
- setSampleDataset={handleSampleDatasetChange}
setEmbeddingModel={setEmbeddingModel}
/>
</>
diff --git a/helm/openrag/templates/backend/deployment.yaml b/helm/openrag/templates/backend/deployment.yaml
index 4a1e2a1a6..0b71d3537 100644
--- a/helm/openrag/templates/backend/deployment.yaml
+++ b/helm/openrag/templates/backend/deployment.yaml
@@ -119,6 +119,8 @@ spec:
# Feature flags
- name: DISABLE_INGEST_WITH_LANGFLOW
value: {{ .Values.backend.features.disableIngestWithLangflow | quote }}
+ - name: INGEST_SAMPLE_DATA
+ value: {{ .Values.backend.features.ingestSampleData | quote }}
# LLM Provider keys
{{- if .Values.llmProviders.openai.enabled }}
- name: OPENAI_API_KEY
diff --git a/helm/openrag/values.yaml b/helm/openrag/values.yaml
index b0545984f..3b2d35210 100644
--- a/helm/openrag/values.yaml
+++ b/helm/openrag/values.yaml
@@ -168,6 +168,7 @@ backend:
# Feature flags
features:
disableIngestWithLangflow: false # Set true to use traditional processor instead of Langflow
+ ingestSampleData: true # Set true to ingest sample data on startup
# Langflow public URL (for UI links to Langflow)
langflowPublicUrl: "" # e.g., "https://langflow.example.com"
diff --git a/sdks/python/tests/test_integration.py b/sdks/python/tests/test_integration.py
index 88df89427..2df299491 100644
--- a/sdks/python/tests/test_integration.py
+++ b/sdks/python/tests/test_integration.py
@@ -40,7 +40,6 @@ def ensure_onboarding():
"embedding_provider": "openai",
"embedding_model": "text-embedding-3-small",
"llm_model": "gpt-4o-mini",
- "sample_data": False,
}
try:
diff --git a/sdks/typescript/tests/integration.test.ts b/sdks/typescript/tests/integration.test.ts
index 3baa27bc1..3ef312046 100644
--- a/sdks/typescript/tests/integration.test.ts
+++ b/sdks/typescript/tests/integration.test.ts
@@ -25,7 +25,6 @@ async function ensureOnboarding(): Promise<void> {
embedding_provider: "openai",
embedding_model: "text-embedding-3-small",
llm_model: "gpt-4o-mini",
- sample_data: false,
};
try {
diff --git a/src/api/settings.py b/src/api/settings.py
index 36b558fe7..dedba0f7c 100644
--- a/src/api/settings.py
+++ b/src/api/settings.py
@@ -7,6 +7,7 @@ from utils.logging_config import get_logger
from utils.telemetry import TelemetryClient, Category, MessageId
from config.settings import (
DISABLE_INGEST_WITH_LANGFLOW,
+ INGEST_SAMPLE_DATA,
LANGFLOW_URL,
LANGFLOW_CHAT_FLOW_ID,
LANGFLOW_INGEST_FLOW_ID,
@@ -730,7 +731,6 @@ async def onboarding(request, flows_service, session_manager=None):
"llm_model",
"embedding_provider",
"embedding_model",
- "sample_data",
# Provider-specific fields
"openai_api_key",
"anthropic_api_key",
@@ -920,20 +920,13 @@ async def onboarding(request, flows_service, session_manager=None):
current_config.providers.ollama.configured = True
logger.info("Marked Ollama as configured (chosen as embedding provider)")
- # Handle sample_data
- should_ingest_sample_data = False
- if "sample_data" in body:
- if not isinstance(body["sample_data"], bool):
- return JSONResponse(
- {"error": "sample_data must be a boolean value"}, status_code=400
- )
- should_ingest_sample_data = body["sample_data"]
- if should_ingest_sample_data:
- await TelemetryClient.send_event(
- Category.ONBOARDING,
- MessageId.ORB_ONBOARD_SAMPLE_DATA
- )
- logger.info("Sample data ingestion requested during onboarding")
+ should_ingest_sample_data = INGEST_SAMPLE_DATA
+ if should_ingest_sample_data:
+ await TelemetryClient.send_event(
+ Category.ONBOARDING,
+ MessageId.ORB_ONBOARD_SAMPLE_DATA
+ )
+ logger.info("Sample data ingestion enabled via environment variable")
if not config_updated:
return JSONResponse(
@@ -1025,24 +1018,16 @@ async def onboarding(request, flows_service, session_manager=None):
await init_index()
logger.info("OpenSearch index initialization completed successfully")
except Exception as e:
- if isinstance(e, ValueError):
- logger.error(
- "Failed to initialize OpenSearch index after onboarding",
- error=str(e),
- )
- return JSONResponse(
- {
- "error": str(e),
- "edited": True,
- },
- status_code=400,
- )
logger.error(
"Failed to initialize OpenSearch index after onboarding",
error=str(e),
)
- # Don't fail the entire onboarding process if index creation fails
- # The application can still work, but document operations may fail
+ return JSONResponse(
+ {
+ "error": str(e),
+ },
+ status_code=500,
+ )
# Handle sample data ingestion if requested
if should_ingest_sample_data:
@@ -1069,12 +1054,14 @@ async def onboarding(request, flows_service, session_manager=None):
logger.error(
"Failed to complete sample data ingestion", error=str(e)
)
- # Don't fail the entire onboarding process if sample data fails
+
+ return JSONResponse(
+ {"error": f"Failed to ingest sample documents: {str(e)}"},
+ status_code=500
+ )
if config_manager.save_config_file(current_config):
- updated_fields = [
- k for k in body.keys() if k != "sample_data"
- ] # Exclude sample_data from log
+ updated_fields = list(body.keys())
logger.info(
"Onboarding configuration updated successfully",
updated_fields=updated_fields,
diff --git a/src/config/settings.py b/src/config/settings.py
index c2be7bb90..94a61adf8 100644
--- a/src/config/settings.py
+++ b/src/config/settings.py
@@ -59,6 +59,11 @@ DISABLE_INGEST_WITH_LANGFLOW = os.getenv(
"DISABLE_INGEST_WITH_LANGFLOW", "false"
).lower() in ("true", "1", "yes")
+# Ingest sample data configuration
+INGEST_SAMPLE_DATA = os.getenv(
+ "INGEST_SAMPLE_DATA", "true"
+).lower() in ("true", "1", "yes")
+
# Maximum number of files to upload / ingest (in batch) per task when adding knowledge via folder
UPLOAD_BATCH_SIZE = int(os.getenv("UPLOAD_BATCH_SIZE", "25"))
diff --git a/src/main.py b/src/main.py
index 677601396..c545634f3 100644
--- a/src/main.py
+++ b/src/main.py
@@ -365,11 +365,7 @@ async def ingest_default_documents_when_ready(services):
await TelemetryClient.send_event(Category.DOCUMENT_INGESTION, MessageId.ORB_DOC_DEFAULT_START)
base_dir = _get_documents_dir()
if not os.path.isdir(base_dir):
- logger.info(
- "Default documents directory not found; skipping ingestion",
- base_dir=base_dir,
- )
- return
+ raise FileNotFoundError(f"Default documents directory not found: {base_dir}")
# Collect files recursively, excluding warmup files
file_paths = [
@@ -380,10 +376,7 @@ async def ingest_default_documents_when_ready(services):
]
if not file_paths:
- logger.info(
- "No default documents found; nothing to ingest", base_dir=base_dir
- )
- return
+ raise FileNotFoundError(f"No default documents found in {base_dir}")
if DISABLE_INGEST_WITH_LANGFLOW:
await _ingest_default_documents_openrag(services, file_paths)
@@ -395,6 +388,7 @@ async def ingest_default_documents_when_ready(services):
except Exception as e:
logger.error("Default documents ingestion failed", error=str(e))
await TelemetryClient.send_event(Category.DOCUMENT_INGESTION, MessageId.ORB_DOC_DEFAULT_FAILED)
+ raise
async def _ingest_default_documents_langflow(services, file_paths):
diff --git a/src/tui/managers/env_manager.py b/src/tui/managers/env_manager.py
index bd0b05a4f..1657ea40e 100644
--- a/src/tui/managers/env_manager.py
+++ b/src/tui/managers/env_manager.py
@@ -69,6 +69,7 @@ class EnvConfig:
# Ingestion settings
disable_ingest_with_langflow: str = "False"
nudges_flow_id: str = "ebc01d31-1976-46ce-a385-b0240327226c"
+ ingest_sample_data: str = "True"
# Document paths (comma-separated) - use centralized location by default
openrag_documents_paths: str = "$HOME/.openrag/documents"
@@ -197,6 +198,7 @@ class EnvManager:
"LANGFLOW_NEW_USER_IS_ACTIVE": "langflow_new_user_is_active",
"LANGFLOW_ENABLE_SUPERUSER_CLI": "langflow_enable_superuser_cli",
"DISABLE_INGEST_WITH_LANGFLOW": "disable_ingest_with_langflow",
+ "INGEST_SAMPLE_DATA": "ingest_sample_data",
"OPENRAG_VERSION": "openrag_version",
"LANGFUSE_SECRET_KEY": "langfuse_secret_key", # pragma: allowlist secret
"LANGFUSE_PUBLIC_KEY": "langfuse_public_key", # pragma: allowlist secret
@@ -456,6 +458,7 @@ class EnvManager:
# Ingestion settings
f.write("# Ingestion settings\n")
f.write(f"DISABLE_INGEST_WITH_LANGFLOW={self._quote_env_value(self.config.disable_ingest_with_langflow)}\n")
+ f.write(f"INGEST_SAMPLE_DATA={self._quote_env_value(self.config.ingest_sample_data)}\n")
f.write("\n")
# Langflow auth settings
@@ -600,6 +603,12 @@ class EnvManager:
"False",
False,
),
+ (
+ "ingest_sample_data",
+ "Ingest Sample Data (optional)",
+ "True",
+ False,
+ ),
(
"opensearch_index_name",
"OpenSearch Index Name",
diff --git a/tests/conftest.py b/tests/conftest.py
index 16885fc11..c332d33bf 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -61,7 +61,6 @@ async def onboard_system():
"embedding_provider": "openai",
"embedding_model": "text-embedding-3-small",
"llm_model": "gpt-4o-mini",
- "sample_data": False,
}
resp = await client.post("/onboarding", json=onboarding_payload)
if resp.status_code not in (200, 204):