OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
fix: change factory-reset functions to delete named volume for opensearch-data (#1363)
Lucas Oliveiraa5ed9b66b4ab
diff --git a/.env.example b/.env.exampleindex ba5207354..75b6ea2d9 100644--- a/.env.example+++ b/.env.example@@ -70,10 +70,6 @@ OPENRAG_ENFORCE_PREREQUISITES=false# The password must contain at least 8 characters, and must contain at least one uppercase letter, one lowercase letter, one digit, and one special character.OPENSEARCH_PASSWORD=-# Path to persist OpenSearch data (indices, documents, cluster state)-# Default: ./opensearch-data-OPENSEARCH_DATA_PATH=./opensearch-data-# Path to persist Langflow database and state (flows, credentials, settings)# Without this volume, flow edits will be lost on container restart# Default: ./langflow-datadiff --git a/Makefile b/Makefileindex 05b31b5bd..e030d6cda 100644--- a/Makefile+++ b/Makefile@@ -532,7 +532,6 @@ factory-reset: ## Complete reset (stop, remove volumes, clear data, remove imageecho "$(YELLOW)This will:$(NC)"; \echo " - Stop all containers"; \echo " - Remove all volumes"; \- echo " - Delete opensearch-data directory"; \echo " - Delete langflow-data directory"; \echo " - Delete config directory"; \echo " - Delete JWT keys (private_key.pem, public_key.pem)"; \@@ -550,13 +549,6 @@ factory-reset: ## Complete reset (stop, remove volumes, clear data, remove imageecho "$(YELLOW)Stopping all services and removing volumes...$(NC)"; \$(COMPOSE_CMD) down -v --remove-orphans || true; \echo "$(YELLOW)Removing local data directories...$(NC)"; \- if [ -d "opensearch-data" ]; then \- echo "Removing opensearch-data..."; \- uv run python scripts/clear_opensearch_data.py 2>/dev/null || \- $(CONTAINER_RUNTIME) run --rm -v "$$(pwd)/opensearch-data:/data" alpine sh -c "rm -rf /data/*" 2>/dev/null || \- rm -rf opensearch-data/* 2>/dev/null || true; \- echo "$(PURPLE)opensearch-data removed$(NC)"; \- fi; \if [ -d "langflow-data" ]; then \echo "Removing langflow-data..."; \rm -rf langflow-data; \@@ -1028,10 +1020,10 @@ db-reset: ## Reset OpenSearch indicescurl -X DELETE "http://localhost:9200/knowledge_filters" -u admin:$${OPENSEARCH_PASSWORD} || true@echo "$(PURPLE)Indices reset. Restart backend to recreate.$(NC)"-clear-os-data: ## Clear OpenSearch data directory- @echo "$(YELLOW)Clearing OpenSearch data directory...$(NC)"+clear-os-data: ## Clear OpenSearch data volume+ @echo "$(YELLOW)Clearing OpenSearch data volume...$(NC)"@uv run python scripts/clear_opensearch_data.py- @echo "$(PURPLE)OpenSearch data cleared.$(NC)"+ @echo "$(PURPLE)OpenSearch data volume cleared.$(NC)"####################### FLOW MANAGEMENTdiff --git a/docs/docs/reference/configuration.mdx b/docs/docs/reference/configuration.mdxindex 895892b96..b73cb2047 100644--- a/docs/docs/reference/configuration.mdx+++ b/docs/docs/reference/configuration.mdx@@ -147,7 +147,6 @@ Configure OpenSearch database authentication.| Variable | Default | Description ||----------|---------|-------------|-| `OPENSEARCH_DATA_PATH` | `./opensearch-data` | The path where OpenRAG creates your OpenSearch index data. This persists through updates. || `OPENSEARCH_PASSWORD` | Not set | Required. OpenSearch administrator password. Must adhere to the [OpenSearch password complexity requirements](https://docs.opensearch.org/latest/security/configuration/demo-configuration/#setting-up-a-custom-admin-password). You must set this directly in the `.env` or in the terminal's [**Reconfigure**](/install-uvx#setup) option. || `OPENSEARCH_INDEX_NAME` | `documents` | The name of the [OpenSearch index](/knowledge#index). || `OPENSEARCH_HOST` | `localhost` | OpenSearch service host. |diff --git a/frontend/.env.test.example b/frontend/.env.test.exampleindex c053ca248..ca03461ea 100644--- a/frontend/.env.test.example+++ b/frontend/.env.test.example@@ -6,7 +6,6 @@ INGEST_SAMPLE_DATA=trueOPENSEARCH_PASSWORD=# Paths-OPENSEARCH_DATA_PATH=./opensearch-dataLANGFLOW_DATA_PATH=./langflow-dataOPENSEARCH_INDEX_NAME=documentsdiff --git a/kubernetes/helm/openrag/templates/backend/backend-dotenv.yaml b/kubernetes/helm/openrag/templates/backend/backend-dotenv.yamlindex 7d901a0f2..fb7191924 100644--- a/kubernetes/helm/openrag/templates/backend/backend-dotenv.yaml+++ b/kubernetes/helm/openrag/templates/backend/backend-dotenv.yaml@@ -70,7 +70,6 @@ stringData:{{- if .Values.global.opensearch.password }}OPENSEARCH_PASSWORD={{ .Values.global.opensearch.password | quote }}{{- end }}- OPENSEARCH_DATA_PATH=./opensearch-dataOPENSEARCH_HOST={{ include "openrag.opensearch.host" . | quote }}OPENSEARCH_PORT={{ .Values.global.opensearch.port | quote }}{{- if .Values.global.opensearch.username }}diff --git a/scripts/clear_opensearch_data.py b/scripts/clear_opensearch_data.pyindex 3ad34dd42..91331214b 100644--- a/scripts/clear_opensearch_data.py+++ b/scripts/clear_opensearch_data.py@@ -12,29 +12,20 @@ from src.tui.managers.container_manager import ContainerManagerasync def main():- """Clear OpenSearch data directory."""+ """Clear OpenSearch data volume."""cm = ContainerManager()- # Get opensearch data path from env config (same as container_manager uses)- from src.tui.managers.env_manager import EnvManager- env_manager = EnvManager()- env_manager.load_existing_env()- opensearch_data_path = Path(- env_manager.config.opensearch_data_path.replace("$HOME", str(Path.home()))- ).expanduser()-- if not opensearch_data_path.exists():- print(f"opensearch-data directory does not exist at {opensearch_data_path}")- return 0-- print("Clearing OpenSearch data directory...")-+ if not cm.is_available():+ print("Error: No container runtime available")+ return 1++ print("Clearing OpenSearch data volume...")+async for success, message in cm.clear_opensearch_data_volume():print(message)if not success and "failed" in message.lower():return 1-- print("✅ OpenSearch data cleared successfully")+return 0diff --git a/src/tui/config_fields.py b/src/tui/config_fields.pyindex 19aa01631..f3e0290ac 100644--- a/src/tui/config_fields.py+++ b/src/tui/config_fields.py@@ -87,12 +87,6 @@ CONFIG_SECTIONS: list[ConfigSection] = [placeholder="9200", default="9200",helper_text="Override for remote OpenSearch instances (default: 9200)",),- ConfigField(- "opensearch_data_path", "OPENSEARCH_DATA_PATH", "Data Path",- placeholder="~/.openrag/data/opensearch-data",- default="$HOME/.openrag/data/opensearch-data",- helper_text="Directory to persist OpenSearch indices across upgrades",- ),ConfigField("opensearch_index_name", "OPENSEARCH_INDEX_NAME", "Index Name",placeholder="documents", default="documents",diff --git a/src/tui/main.py b/src/tui/main.pyindex 3899f91e2..0bd1e37f0 100644--- a/src/tui/main.py+++ b/src/tui/main.py@@ -562,7 +562,7 @@ def migrate_legacy_data_directories():"""Migrate data from CWD-based directories to ~/.openrag/.This is a one-time migration for users upgrading from the old layout.- Migrates: documents, flows, keys, config, opensearch-data, langflow-data+ Migrates: documents, flows, keys, config, langflow-dataPrompts user for confirmation before migrating. If user declines,exits with a message to downgrade to v1.52 or earlier.@@ -584,7 +584,8 @@ def migrate_legacy_data_directories():(cwd / "flows", target_base / "flows", "flows"),(cwd / "keys", target_base / "keys", "keys"),(cwd / "config", target_base / "config", "config"),- (cwd / "opensearch-data", target_base / "data" / "opensearch-data", "OpenSearch data"),+ # opensearch-data intentionally omitted: OpenSearch now uses a Docker named+ # volume; old index data is not portable and must be re-ingested.(cwd / "langflow-data", target_base / "data" / "langflow-data", "Langflow data"),]@@ -608,7 +609,6 @@ def migrate_legacy_data_directories():env_manager.config.openrag_flows_path = f"{home}/.openrag/flows"env_manager.config.openrag_config_path = f"{home}/.openrag/config"env_manager.config.openrag_data_path = f"{home}/.openrag/data"- env_manager.config.opensearch_data_path = f"{home}/.openrag/data/opensearch-data"env_manager.config.langflow_data_path = f"{home}/.openrag/data/langflow-data"env_manager.save_env_file()logger.info("Updated .env file with centralized paths")@@ -688,7 +688,6 @@ def migrate_legacy_data_directories():env_manager.config.openrag_flows_path = f"{home}/.openrag/flows"env_manager.config.openrag_config_path = f"{home}/.openrag/config"env_manager.config.openrag_data_path = f"{home}/.openrag/data"- env_manager.config.opensearch_data_path = f"{home}/.openrag/data/opensearch-data"env_manager.config.langflow_data_path = f"{home}/.openrag/data/langflow-data"env_manager.save_env_file()print(" Updated .env with centralized paths")@@ -804,7 +803,6 @@ def setup_host_directories():- ~/.openrag/keys/ (for JWT keys)- ~/.openrag/config/ (for configuration)- ~/.openrag/data/ (for backend data: conversations, OAuth tokens, etc.)- - ~/.openrag/data/opensearch-data/ (for OpenSearch index)- LANGFLOW_DATA_PATH (for Langflow database and state)"""base_dir = Path.home() / ".openrag"@@ -814,7 +812,6 @@ def setup_host_directories():base_dir / "keys",base_dir / "config",base_dir / "data",- base_dir / "data" / "opensearch-data",]for directory in directories:diff --git a/src/tui/managers/container_manager.py b/src/tui/managers/container_manager.pyindex c1cf7e374..321b096be 100644--- a/src/tui/managers/container_manager.py+++ b/src/tui/managers/container_manager.py@@ -1278,23 +1278,37 @@ class ContainerManager:yield False, "Clearing OpenSearch data volume..."- # Get opensearch data path from env config- from .env_manager import EnvManager- env_manager = EnvManager()- env_manager.load_existing_env()- opensearch_data_path = Path(env_manager.config.opensearch_data_path.replace("$HOME", str(Path.home()))).expanduser().absolute()-- if not opensearch_data_path.exists():- yield True, "OpenSearch data directory does not exist, skipping"+ # The volume name is 'opensearch-data' as defined in docker-compose.yml.+ # Docker Compose usually prefixes it with the project name (defaulting to the directory name).+ volume_name = "opensearch-data"++ possible_names = list(dict.fromkeys([+ volume_name,+ f"{Path.cwd().name.lower()}_{volume_name}", # e.g. openrag_opensearch-data+ f"{Path.cwd().name.lower().replace('-', '')}_{volume_name}", # e.g. openrag_opensearch-data (dashes stripped)+ ]))++ # List volumes and find the correct one+ list_success, stdout, _ = await self._run_runtime_command(["volume", "ls", "--format", "{{.Name}}"])+ if not list_success:+ yield False, "Could not list Docker volumes"return- # Use alpine with root to clear container-owned files- success, msg = await self.clear_directory_with_container(opensearch_data_path)+ existing_volumes = set(stdout.splitlines())+ found_name = next((n for n in possible_names if n in existing_volumes), None)- if success:- yield True, "OpenSearch data cleared successfully"+ if found_name is None:+ yield True, "OpenSearch data volume not found — already removed or never created"+ return++ # Only clear the volume we know exists+ cmd = ["run", "--rm", "-v", f"{found_name}:/work:Z", "alpine", "sh", "-c",+ "rm -rf /work/* /work/.[!.]* 2>/dev/null; echo done"]+ success, stdout, stderr = await self._run_runtime_command(cmd)+ if success and "done" in stdout:+ yield True, f"OpenSearch data volume '{found_name}' cleared successfully"else:- yield False, f"Failed to clear OpenSearch data: {msg}"+ yield False, f"Failed to clear OpenSearch data volume '{found_name}': {stderr}"async def reset_services(self) -> AsyncIterator[tuple[bool, str]]:"""Reset all services (stop, remove containers/volumes, clear data) and yield progress updates."""diff --git a/src/tui/managers/env_manager.py b/src/tui/managers/env_manager.pyindex 175c94e45..7f13e66e1 100644--- a/src/tui/managers/env_manager.py+++ b/src/tui/managers/env_manager.py@@ -97,7 +97,6 @@ class EnvConfig:openrag_flows_path: str = "$HOME/.openrag/flows"openrag_config_path: str = "$HOME/.openrag/config"openrag_data_path: str = "$HOME/.openrag/data" # Backend data (conversations, tokens, etc.)- opensearch_data_path: str = "$HOME/.openrag/data/opensearch-data"langflow_data_path: str = "$HOME/.openrag/data/langflow-data"openrag_tui_config_path_legacy: str = "$HOME/.openrag/tui/config"@@ -223,7 +222,6 @@ class EnvManager:"OPENRAG_FLOWS_PATH": "openrag_flows_path","OPENRAG_CONFIG_PATH": "openrag_config_path","OPENRAG_DATA_PATH": "openrag_data_path",- "OPENSEARCH_DATA_PATH": "opensearch_data_path","LANGFLOW_DATA_PATH": "langflow_data_path","LANGFLOW_AUTO_LOGIN": "langflow_auto_login","LANGFLOW_NEW_USER_IS_ACTIVE": "langflow_new_user_is_active",@@ -506,9 +504,6 @@ class EnvManager:f.write(f"OPENRAG_DATA_PATH={self._quote_env_value(expand_path(self.config.openrag_data_path))}\n")- f.write(- f"OPENSEARCH_DATA_PATH={self._quote_env_value(expand_path(self.config.opensearch_data_path))}\n"- )f.write(f"LANGFLOW_DATA_PATH={self._quote_env_value(expand_path(self.config.langflow_data_path))}\n")@@ -783,11 +778,11 @@ class EnvManager:else:new_lines.append(line)- # If not found, add it after OPENSEARCH_DATA_PATH or at the end+ # If not found, add it after LANGFLOW_DATA_PATH or at the endif not updated:insert_pos = len(new_lines)for i, line in enumerate(new_lines):- if "OPENSEARCH_DATA_PATH" in line:+ if "LANGFLOW_DATA_PATH" in line:insert_pos = i + 1breaknew_lines.insert(insert_pos, f"OPENRAG_VERSION={self._quote_env_value(current_version)}")diff --git a/src/tui/screens/config.py b/src/tui/screens/config.pyindex 4ca6c8bd0..5c569d393 100644--- a/src/tui/screens/config.py+++ b/src/tui/screens/config.py@@ -200,7 +200,6 @@ class ConfigScreen(Screen):# Fields that need custom rendering beyond the standard patternSPECIAL_FIELDS = {"opensearch_password",- "opensearch_data_path","langflow_superuser_password","langflow_superuser","langflow_data_path",@@ -288,25 +287,6 @@ class ConfigScreen(Screen):yield Button("👁", id=f"toggle-{field.name}", variant="default")yield Static(" ")- def _render_opensearch_data_path(self, field: ConfigField) -> ComposeResult:- """OpenSearch data path with file picker."""- yield Label(field.label)- yield Static(field.helper_text, classes="helper-text")- current_value = getattr(self.env_manager.config, field.name, field.default)- input_widget = Input(- placeholder=field.placeholder,- value=current_value,- id=f"input-{field.name}",- )- yield input_widget- yield Horizontal(- Button("Pick…", id="pick-opensearch-data-btn"),- id="opensearch-data-path-actions",- classes="controls-row",- )- self.inputs[field.name] = input_widget- yield Static(" ")-def _render_langflow_data_path(self, field: ConfigField) -> ComposeResult:"""Langflow data path with file picker."""yield Label(field.label)@@ -474,8 +454,6 @@ class ConfigScreen(Screen):self.action_back()elif event.button.id == "pick-docs-btn":self.action_pick_documents_path()- elif event.button.id == "pick-opensearch-data-btn":- self.action_pick_opensearch_data_path()elif event.button.id == "pick-langflow-data-btn":self.action_pick_langflow_data_path()elif event.button.id and event.button.id.startswith("toggle-"):@@ -631,62 +609,6 @@ class ConfigScreen(Screen):self._docs_pick_callback = _append_path # type: ignore[attr-defined]self.app.push_screen(picker)- def action_pick_opensearch_data_path(self) -> None:- """Open textual-fspicker to select OpenSearch data directory."""- try:- import importlib-- fsp = importlib.import_module("textual_fspicker")- except Exception:- self.notify("textual-fspicker not available", severity="warning")- return-- # Determine starting path from current input if possible- input_widget = self.inputs.get("opensearch_data_path")- start = Path.home()- if input_widget and input_widget.value:- path_str = input_widget.value.strip()- if path_str:- candidate = Path(path_str).expanduser()- # If path doesn't exist, use parent or fallback to home- if candidate.exists():- start = candidate- elif candidate.parent.exists():- start = candidate.parent-- # Prefer SelectDirectory for directories; fallback to FileOpen- PickerClass = getattr(fsp, "SelectDirectory", None) or getattr(- fsp, "FileOpen", None- )- if PickerClass is None:- self.notify(- "No compatible picker found in textual-fspicker", severity="warning"- )- return- try:- picker = PickerClass(location=start)- except Exception:- try:- picker = PickerClass(start)- except Exception:- self.notify("Could not initialize textual-fspicker", severity="warning")- return-- def _set_path(result) -> None:- if not result:- return- path_str = str(result)- if input_widget is None:- return- input_widget.value = path_str-- # Push with callback when supported; otherwise, use on_screen_dismissed fallback- try:- self.app.push_screen(picker, _set_path) # type: ignore[arg-type]- except TypeError:- self._opensearch_data_pick_callback = _set_path # type: ignore[attr-defined]- self.app.push_screen(picker)-def action_pick_langflow_data_path(self) -> None:"""Open textual-fspicker to select Langflow data directory."""try:diff --git a/src/tui/screens/monitor.py b/src/tui/screens/monitor.pyindex 5c2b6fd81..f26be5485 100644--- a/src/tui/screens/monitor.py+++ b/src/tui/screens/monitor.py@@ -551,23 +551,6 @@ class MonitorScreen(Screen):env_manager = EnvManager()env_manager.load_existing_env()- # Clear opensearch-data using container- yield False, "Clearing OpenSearch data..."- opensearch_data_path = Path(env_manager.config.opensearch_data_path.replace("$HOME", str(Path.home()))).expanduser()- if opensearch_data_path.exists():- async for success, message in self.container_manager.clear_opensearch_data_volume():- yield success, message- if not success and "failed" in message.lower():- return-- # Recreate empty opensearch-data directory- try:- opensearch_data_path.mkdir(parents=True, exist_ok=True)- yield True, "OpenSearch data directory recreated"- except Exception as e:- yield False, f"Error recreating opensearch-data directory: {e}"- return-# Delete langflow-data directory (mirrors Makefile factory-reset behaviour)yield False, "Clearing Langflow data..."from tui.main import _resolve_langflow_data_pathdiff --git a/tests/conftest.py b/tests/conftest.pyindex ffe80467e..e6f058c69 100644--- a/tests/conftest.py+++ b/tests/conftest.py@@ -34,25 +34,19 @@ async def onboard_system():yieldreturn- from pathlib import Path- import shutil-# Delete any existing config to ensure clean onboardingconfig_file = Path("config/config.yaml")if config_file.exists():config_file.unlink()- # Clean up OpenSearch data directory to ensure fresh state for tests- opensearch_data_path = Path(os.getenv("OPENSEARCH_DATA_PATH", "./opensearch-data"))- if opensearch_data_path.exists():- try:- shutil.rmtree(opensearch_data_path)- print(f"[DEBUG] Cleaned up OpenSearch data directory: {opensearch_data_path}")- except Exception as e:- print(f"[DEBUG] Could not clean OpenSearch data directory: {e}")+ # Clean up OpenSearch indices to ensure fresh state for tests+ try:+ await clients.initialize()+ await clients.opensearch.indices.delete(index="_all", ignore_unavailable=True)+ print("[DEBUG] Wiped all OpenSearch indices via API")+ except Exception as e:+ print(f"[DEBUG] Could not wipe OpenSearch indices: {e}")- # Initialize clients- await clients.initialize()# Create app and perform onboarding via APIfrom main import create_app, startup_tasks