Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

Add SDK integration test suite

Edwin Jose34ed56e6f055
diff --git a/tests/integration/sdk/conftest.py b/tests/integration/sdk/conftest.py
new file mode 100644
index 000000000..42678029c
--- /dev/null
+++ b/tests/integration/sdk/conftest.py
@@ -0,0 +1,90 @@
+"""
+Shared fixtures and setup for OpenRAG SDK integration tests.
+
+All tests in this directory require a running OpenRAG instance.
+Set OPENRAG_URL (default: http://localhost:3000) before running.
+"""
+
+import os
+import uuid
+from pathlib import Path
+
+import httpx
+import pytest
+
+_cached_api_key: str | None = None
+_base_url = os.environ.get("OPENRAG_URL", "http://localhost:3000")
+_onboarding_done = False
+
+
[email protected](scope="session", autouse=True)
+def ensure_onboarding():
+ """Ensure the OpenRAG instance is onboarded before running tests."""
+ global _onboarding_done
+ if _onboarding_done:
+ return
+
+ onboarding_payload = {
+ "llm_provider": "openai",
+ "embedding_provider": "openai",
+ "embedding_model": "text-embedding-3-small",
+ "llm_model": "gpt-4o-mini",
+ }
+
+ try:
+ response = httpx.post(
+ f"{_base_url}/api/onboarding",
+ json=onboarding_payload,
+ timeout=30.0,
+ )
+ if response.status_code in (200, 204):
+ print("[SDK Tests] Onboarding completed successfully")
+ else:
+ print(f"[SDK Tests] Onboarding returned {response.status_code}: {response.text[:200]}")
+ except Exception as e:
+ print(f"[SDK Tests] Onboarding request failed: {e}")
+
+ _onboarding_done = True
+
+
+def get_api_key() -> str:
+ """Get or create an API key for testing (cached for the session)."""
+ global _cached_api_key
+ if _cached_api_key is None:
+ response = httpx.post(
+ f"{_base_url}/api/keys",
+ json={"name": "SDK Integration Test"},
+ timeout=30.0,
+ )
+ if response.status_code == 401:
+ pytest.skip("Cannot create API key - authentication required")
+ assert response.status_code == 200, f"Failed to create API key: {response.text}"
+ _cached_api_key = response.json()["api_key"]
+ return _cached_api_key
+
+
+def client():
+ """OpenRAG client authenticated with a valid test API key."""
+ from openrag_sdk import OpenRAGClient
+
+ return OpenRAGClient(api_key=get_api_key(), base_url=_base_url)
+
+
+def base_url() -> str:
+ """The base URL of the running OpenRAG instance."""
+ return _base_url
+
+
+def test_file(tmp_path) -> Path:
+ """A uniquely-named markdown file ready for ingestion."""
+ file_path = tmp_path / f"sdk_test_doc_{uuid.uuid4().hex[:8]}.md"
+ file_path.write_text(
+ f"# SDK Integration Test Document\n\n"
+ f"ID: {uuid.uuid4()}\n\n"
+ "This document tests the OpenRAG Python SDK.\n\n"
+ "It contains unique content about purple elephants dancing.\n"
+ )
+ return file_path
diff --git a/tests/integration/sdk/test_auth.py b/tests/integration/sdk/test_auth.py
new file mode 100644
index 000000000..af233dbca
--- /dev/null
+++ b/tests/integration/sdk/test_auth.py
@@ -0,0 +1,59 @@
+"""Tests for authentication and API key behaviour."""
+
+import os
+
+import pytest
+
+from .conftest import _base_url
+
+pytestmark = pytest.mark.skipif(
+ os.environ.get("SKIP_SDK_INTEGRATION_TESTS") == "true",
+ reason="SDK integration tests skipped",
+)
+
+
+class TestAuth:
+ """Test authentication and API key behaviour."""
+
+ def test_missing_api_key_raises_at_construction(self):
+ """Client must raise AuthenticationError immediately if no api_key is given."""
+ from openrag_sdk import OpenRAGClient
+ from openrag_sdk.exceptions import AuthenticationError
+
+ env_backup = os.environ.pop("OPENRAG_API_KEY", None)
+ try:
+ with pytest.raises(AuthenticationError):
+ OpenRAGClient()
+ finally:
+ if env_backup is not None:
+ os.environ["OPENRAG_API_KEY"] = env_backup
+
+ @pytest.mark.asyncio
+ async def test_invalid_api_key_raises_auth_error(self):
+ """Requests with a bogus key must raise AuthenticationError (401/403)."""
+ from openrag_sdk import OpenRAGClient
+ from openrag_sdk.exceptions import AuthenticationError
+
+ bad_client = OpenRAGClient(api_key="orag_invalid_key_for_testing", base_url=_base_url)
+ try:
+ with pytest.raises(AuthenticationError) as exc_info:
+ await bad_client.settings.get()
+ assert exc_info.value.status_code in (401, 403)
+ finally:
+ await bad_client.close()
+
+ @pytest.mark.asyncio
+ async def test_revoked_api_key_raises_auth_error(self):
+ """A well-formed but non-existent key must be rejected."""
+ from openrag_sdk import OpenRAGClient
+ from openrag_sdk.exceptions import AuthenticationError
+
+ fake_client = OpenRAGClient(
+ api_key="orag_0000000000000000000000000000000000000000",
+ base_url=_base_url,
+ )
+ try:
+ with pytest.raises(AuthenticationError):
+ await fake_client.chat.list()
+ finally:
+ await fake_client.close()
diff --git a/tests/integration/sdk/test_chat.py b/tests/integration/sdk/test_chat.py
new file mode 100644
index 000000000..f3542f138
--- /dev/null
+++ b/tests/integration/sdk/test_chat.py
@@ -0,0 +1,184 @@
+"""Tests for the chat endpoint — non-streaming, streaming, conversations, and RAG."""
+
+import asyncio
+import os
+from pathlib import Path
+
+import pytest
+
+pytestmark = pytest.mark.skipif(
+ os.environ.get("SKIP_SDK_INTEGRATION_TESTS") == "true",
+ reason="SDK integration tests skipped",
+)
+
+
+class TestChat:
+ """Core chat operation tests."""
+
+ @pytest.mark.asyncio
+ async def test_chat_non_streaming(self, client):
+ """Non-streaming chat returns a non-empty response string."""
+ response = await client.chat.create(message="Say hello in exactly 3 words.")
+
+ assert response.response is not None
+ assert isinstance(response.response, str)
+ assert len(response.response) > 0
+
+ @pytest.mark.asyncio
+ async def test_chat_streaming_create(self, client):
+ """create(stream=True) yields content events with text deltas."""
+ collected_text = ""
+
+ async for event in await client.chat.create(
+ message="Say 'test' and nothing else.",
+ stream=True,
+ ):
+ if event.type == "content":
+ collected_text += event.delta
+
+ assert len(collected_text) > 0
+
+ @pytest.mark.asyncio
+ async def test_chat_streaming_context_manager(self, client):
+ """stream() context manager accumulates text in stream.text."""
+ async with client.chat.stream(message="Say 'hello' and nothing else.") as stream:
+ async for _ in stream:
+ pass
+ assert len(stream.text) > 0
+
+ @pytest.mark.asyncio
+ async def test_chat_text_stream(self, client):
+ """text_stream yields plain text deltas."""
+ collected = ""
+
+ async with client.chat.stream(message="Say 'world' and nothing else.") as stream:
+ async for text in stream.text_stream:
+ collected += text
+
+ assert len(collected) > 0
+
+ @pytest.mark.asyncio
+ async def test_chat_final_text(self, client):
+ """final_text() returns the complete accumulated response."""
+ async with client.chat.stream(message="Say 'done' and nothing else.") as stream:
+ text = await stream.final_text()
+
+ assert len(text) > 0
+
+ @pytest.mark.asyncio
+ async def test_chat_conversation_continuation(self, client):
+ """A second message with chat_id continues the same conversation."""
+ response1 = await client.chat.create(message="Remember the number 42.")
+ assert response1.chat_id is not None
+
+ response2 = await client.chat.create(
+ message="What number did I ask you to remember?",
+ chat_id=response1.chat_id,
+ )
+ assert response2.response is not None
+
+ @pytest.mark.asyncio
+ async def test_list_conversations(self, client):
+ """list() returns a ConversationListResponse with a list of conversations."""
+ await client.chat.create(message="Test message for listing.")
+
+ result = await client.chat.list()
+
+ assert result.conversations is not None
+ assert isinstance(result.conversations, list)
+
+ @pytest.mark.asyncio
+ async def test_get_conversation(self, client):
+ """get() returns the full conversation with message history."""
+ response = await client.chat.create(message="Test message for get.")
+ assert response.chat_id is not None
+
+ conversation = await client.chat.get(response.chat_id)
+
+ assert conversation.chat_id == response.chat_id
+ assert conversation.messages is not None
+ assert isinstance(conversation.messages, list)
+ assert len(conversation.messages) >= 1
+
+ @pytest.mark.asyncio
+ async def test_delete_conversation(self, client):
+ """delete() returns True for a conversation that exists."""
+ response = await client.chat.create(message="Test message for delete.")
+ assert response.chat_id is not None
+
+ result = await client.chat.delete(response.chat_id)
+
+ assert result is True
+
+ @pytest.mark.asyncio
+ async def test_chat_with_sources(self, client, test_file: Path):
+ """Chat response must cite the ingested document as a source (RAG)."""
+ result = await client.documents.ingest(file_path=str(test_file))
+ if result.status == "failed" or result.successful_files == 0:
+ pytest.skip("Document ingestion failed — cannot test RAG sources")
+
+ await asyncio.sleep(3)
+
+ response = await client.chat.create(
+ message="What is the color of the dancing animals mentioned in my documents?"
+ )
+
+ assert response.sources is not None
+ assert len(response.sources) > 0
+ source_filenames = [s.filename for s in response.sources]
+ assert any(test_file.name in name for name in source_filenames)
+
+
+class TestChatExtended:
+ """Additional chat edge-case tests."""
+
+ @pytest.mark.asyncio
+ async def test_stream_continuation_with_chat_id(self, client):
+ """Streaming a follow-up message in an existing conversation works."""
+ r1 = await client.chat.create(message="Remember the colour blue.")
+ assert r1.chat_id is not None
+
+ collected = ""
+ async with client.chat.stream(
+ message="What colour did I ask you to remember?",
+ chat_id=r1.chat_id,
+ ) as stream:
+ async for text in stream.text_stream:
+ collected += text
+
+ assert len(collected) > 0
+ await client.chat.delete(r1.chat_id)
+
+ @pytest.mark.asyncio
+ async def test_chat_response_has_chat_id(self, client):
+ """Every non-streaming response must include a chat_id for continuation."""
+ response = await client.chat.create(message="Hello.")
+ assert response.chat_id is not None
+ assert isinstance(response.chat_id, str)
+ assert len(response.chat_id) > 0
+ await client.chat.delete(response.chat_id)
+
+ @pytest.mark.asyncio
+ async def test_stream_chat_id_available_after_iteration(self, client):
+ """chat_id must be populated on ChatStream after the stream is consumed."""
+ async with client.chat.stream(message="Say one word.") as stream:
+ await stream.final_text()
+ assert stream.chat_id is not None
+
+ @pytest.mark.asyncio
+ async def test_chat_sources_field_is_list(self, client):
+ """sources on ChatResponse is always a list (may be empty)."""
+ response = await client.chat.create(message="What time is it?")
+ assert response.sources is not None
+ assert isinstance(response.sources, list)
+ if response.chat_id:
+ await client.chat.delete(response.chat_id)
+
+ @pytest.mark.asyncio
+ async def test_list_conversations_returns_list(self, client):
+ """list() always returns a ConversationListResponse with a list."""
+ r = await client.chat.create(message="List test message.")
+ result = await client.chat.list()
+ assert isinstance(result.conversations, list)
+ if r.chat_id:
+ await client.chat.delete(r.chat_id)
diff --git a/tests/integration/sdk/test_documents.py b/tests/integration/sdk/test_documents.py
new file mode 100644
index 000000000..9fcab83ab
--- /dev/null
+++ b/tests/integration/sdk/test_documents.py
@@ -0,0 +1,121 @@
+"""Tests for document ingestion and deletion."""
+
+import io
+import os
+import uuid
+from pathlib import Path
+
+import pytest
+
+pytestmark = pytest.mark.skipif(
+ os.environ.get("SKIP_SDK_INTEGRATION_TESTS") == "true",
+ reason="SDK integration tests skipped",
+)
+
+
+class TestDocuments:
+ """Core document ingestion and deletion tests."""
+
+ @pytest.mark.asyncio
+ async def test_ingest_document_no_wait(self, client, test_file: Path):
+ """wait=False returns a task_id immediately; polling reaches a terminal state."""
+ result = await client.documents.ingest(file_path=str(test_file), wait=False)
+ assert result.task_id is not None
+
+ final_status = await client.documents.wait_for_task(result.task_id)
+ assert final_status.status is not None
+ assert final_status.successful_files >= 0
+
+ @pytest.mark.asyncio
+ async def test_ingest_document(self, client, test_file: Path):
+ """wait=True polls until completion and returns a terminal status."""
+ result = await client.documents.ingest(file_path=str(test_file))
+ assert result.status is not None
+ assert result.successful_files >= 0
+
+ @pytest.mark.asyncio
+ async def test_delete_document(self, client, test_file: Path):
+ """Deleting an ingested document succeeds when chunks were indexed."""
+ ingest_result = await client.documents.ingest(file_path=str(test_file))
+
+ result = await client.documents.delete(test_file.name)
+
+ if ingest_result.successful_files > 0:
+ assert result.success is True
+ assert result.deleted_chunks > 0
+ else:
+ assert result.success is False
+ assert result.deleted_chunks == 0
+
+ @pytest.mark.asyncio
+ async def test_delete_missing_document_is_idempotent(self, client):
+ """Deleting a never-ingested filename must not raise."""
+ missing_filename = f"never_ingested_{uuid.uuid4().hex}.pdf"
+ result = await client.documents.delete(missing_filename)
+
+ assert result.success is False
+ assert result.deleted_chunks == 0
+ assert result.filename == missing_filename
+ assert result.error is not None
+
+
+class TestDocumentsExtended:
+ """Additional document ingestion scenarios."""
+
+ @pytest.mark.asyncio
+ async def test_ingest_via_file_object(self, client):
+ """Ingest using a file-like object (io.BytesIO) instead of a file path."""
+ unique_token = uuid.uuid4().hex
+ content = (
+ f"# File Object Test\n\n"
+ f"Token: {unique_token}\n\n"
+ f"This document was ingested via a file object.\n"
+ ).encode()
+
+ filename = f"file_obj_{unique_token[:8]}.md"
+ result = await client.documents.ingest(file=io.BytesIO(content), filename=filename)
+ assert result.status is not None
+ assert result.successful_files >= 0
+
+ await client.documents.delete(filename)
+
+ @pytest.mark.asyncio
+ async def test_reingest_same_filename_does_not_raise(self, client, tmp_path):
+ """Ingesting the same filename twice must not raise an error."""
+ unique_token = uuid.uuid4().hex
+ file_path = tmp_path / f"reingest_{unique_token[:8]}.md"
+ file_path.write_text(f"# Reingest Test\n\nToken: {unique_token}\n")
+
+ result1 = await client.documents.ingest(file_path=str(file_path))
+ assert result1.status is not None
+
+ result2 = await client.documents.ingest(file_path=str(file_path))
+ assert result2.status is not None
+
+ await client.documents.delete(file_path.name)
+
+ @pytest.mark.asyncio
+ async def test_ingest_markdown_format(self, client, tmp_path):
+ """Verify .md files are accepted and processed without error."""
+ file_path = tmp_path / f"format_md_{uuid.uuid4().hex[:8]}.md"
+ file_path.write_text("# Markdown Format\n\n## Section\n\nContent here.\n")
+ result = await client.documents.ingest(file_path=str(file_path))
+ assert result.status is not None
+ await client.documents.delete(file_path.name)
+
+ @pytest.mark.asyncio
+ async def test_task_status_polling(self, client, tmp_path):
+ """wait=False returns a task_id that can be polled and waited on manually."""
+ file_path = tmp_path / f"poll_{uuid.uuid4().hex[:8]}.md"
+ file_path.write_text("# Polling Test\n\nContent for polling test.\n")
+
+ task_response = await client.documents.ingest(file_path=str(file_path), wait=False)
+ assert task_response.task_id is not None
+
+ status = await client.documents.get_task_status(task_response.task_id)
+ assert status.status is not None
+
+ final = await client.documents.wait_for_task(task_response.task_id)
+ assert final.status in ("completed", "failed")
+
+ await client.documents.delete(file_path.name)
diff --git a/tests/integration/sdk/test_e2e.py b/tests/integration/sdk/test_e2e.py
new file mode 100644
index 000000000..bc40450cf
--- /dev/null
+++ b/tests/integration/sdk/test_e2e.py
@@ -0,0 +1,107 @@
+"""End-to-end tests covering full multi-step SDK workflows."""
+
+import asyncio
+import os
+import uuid
+
+import pytest
+
+pytestmark = pytest.mark.skipif(
+ os.environ.get("SKIP_SDK_INTEGRATION_TESTS") == "true",
+ reason="SDK integration tests skipped",
+)
+
+
+class TestEndToEnd:
+ """Full pipeline tests that exercise multiple SDK operations together."""
+
+ @pytest.mark.asyncio
+ async def test_full_rag_pipeline(self, client, tmp_path):
+ """Ingest → search → chat: source cited in chat must match the ingested doc."""
+ unique_token = uuid.uuid4().hex
+ file_path = tmp_path / f"rag_e2e_{unique_token[:8]}.md"
+ file_path.write_text(
+ f"# E2E RAG Test\n\n"
+ f"Unique token: {unique_token}\n\n"
+ f"The flamingo named Zephyr lives on planet Xylox-7.\n"
+ )
+
+ ingest_result = await client.documents.ingest(file_path=str(file_path))
+ if ingest_result.successful_files == 0:
+ pytest.skip("Ingestion produced no indexed chunks — skipping E2E RAG test")
+
+ await asyncio.sleep(3)
+
+ search_results = await client.search.query("flamingo Zephyr planet Xylox")
+ assert search_results.results is not None
+
+ chat_response = await client.chat.create(
+ message="What is the name of the flamingo and where does it live?"
+ )
+ assert chat_response.response is not None
+ assert len(chat_response.response) > 0
+ assert chat_response.sources is not None
+ source_names = [s.filename for s in chat_response.sources]
+ assert any(file_path.name in name for name in source_names), (
+ f"Expected {file_path.name} in sources {source_names}"
+ )
+
+ await client.documents.delete(file_path.name)
+
+ @pytest.mark.asyncio
+ async def test_multiturn_rag_conversation(self, client, tmp_path):
+ """Multi-turn conversation: context carries across turns."""
+ unique_token = uuid.uuid4().hex
+ file_path = tmp_path / f"multiturn_{unique_token[:8]}.md"
+ file_path.write_text(
+ f"# Multiturn Test\n\n"
+ f"The capital of the fictional country Valdoria is Sunhaven.\n"
+ f"Token: {unique_token}\n"
+ )
+
+ ingest_result = await client.documents.ingest(file_path=str(file_path))
+ if ingest_result.successful_files == 0:
+ pytest.skip("Ingestion produced no indexed chunks — skipping multiturn test")
+
+ await asyncio.sleep(3)
+
+ r1 = await client.chat.create(message="What is the capital of Valdoria?")
+ assert r1.chat_id is not None
+
+ r2 = await client.chat.create(
+ message="Repeat the capital city you just mentioned.",
+ chat_id=r1.chat_id,
+ )
+ assert r2.response is not None
+ assert r2.chat_id == r1.chat_id
+
+ await client.documents.delete(file_path.name)
+ await client.chat.delete(r1.chat_id)
+
+ @pytest.mark.asyncio
+ async def test_knowledge_filter_scopes_search_results(self, client):
+ """A knowledge filter must constrain search and chat to its configured scope."""
+ unique_token = uuid.uuid4().hex
+ create_result = await client.knowledge_filters.create({
+ "name": f"E2E Scope Filter {unique_token[:8]}",
+ "description": "Filter for E2E scoping test",
+ "queryData": {
+ "query": f"scoped content {unique_token}",
+ "limit": 5,
+ "scoreThreshold": 0.0,
+ },
+ })
+ assert create_result.success is True
+ filter_id = create_result.id
+
+ try:
+ search_results = await client.search.query("test query", filter_id=filter_id)
+ assert search_results.results is not None
+
+ chat_response = await client.chat.create(
+ message="Summarise what you know.",
+ filter_id=filter_id,
+ )
+ assert chat_response.response is not None
+ finally:
+ await client.knowledge_filters.delete(filter_id)
diff --git a/tests/integration/sdk/test_errors.py b/tests/integration/sdk/test_errors.py
new file mode 100644
index 000000000..30ce68dd2
--- /dev/null
+++ b/tests/integration/sdk/test_errors.py
@@ -0,0 +1,70 @@
+"""Tests for SDK error handling and propagation."""
+
+import io
+import os
+import uuid
+
+import pytest
+
+from .conftest import _base_url
+
+pytestmark = pytest.mark.skipif(
+ os.environ.get("SKIP_SDK_INTEGRATION_TESTS") == "true",
+ reason="SDK integration tests skipped",
+)
+
+
+class TestErrorHandling:
+ """Verify the SDK surfaces errors correctly rather than swallowing them."""
+
+ @pytest.mark.asyncio
+ async def test_connection_refused_raises_exception(self):
+ """Pointing the client at a dead port must raise a network exception, not hang."""
+ from openrag_sdk import OpenRAGClient
+
+ dead_client = OpenRAGClient(
+ api_key="orag_test",
+ base_url="http://localhost:19999",
+ timeout=3.0,
+ )
+ try:
+ with pytest.raises(Exception):
+ await dead_client.settings.get()
+ finally:
+ await dead_client.close()
+
+ @pytest.mark.asyncio
+ async def test_get_nonexistent_conversation_raises_not_found(self, client):
+ """Fetching a conversation with a random UUID must raise NotFoundError."""
+ from openrag_sdk.exceptions import NotFoundError
+
+ with pytest.raises(NotFoundError):
+ await client.chat.get(str(uuid.uuid4()))
+
+ @pytest.mark.asyncio
+ async def test_delete_nonexistent_conversation_raises_not_found(self, client):
+ """Deleting a conversation that never existed must raise NotFoundError."""
+ from openrag_sdk.exceptions import NotFoundError
+
+ with pytest.raises(NotFoundError):
+ await client.chat.delete(str(uuid.uuid4()))
+
+ @pytest.mark.asyncio
+ async def test_invalid_settings_value_raises_error(self, client):
+ """Sending an invalid settings value must raise a subclass of OpenRAGError."""
+ from openrag_sdk.exceptions import OpenRAGError
+
+ with pytest.raises(OpenRAGError):
+ await client.settings.update({"chunk_size": -999999})
+
+ @pytest.mark.asyncio
+ async def test_ingest_without_file_raises_value_error(self, client):
+ """Calling ingest() with neither file_path nor file must raise ValueError."""
+ with pytest.raises(ValueError):
+ await client.documents.ingest()
+
+ @pytest.mark.asyncio
+ async def test_ingest_file_object_without_filename_raises_value_error(self, client):
+ """Providing a file object without a filename must raise ValueError."""
+ with pytest.raises(ValueError):
+ await client.documents.ingest(file=io.BytesIO(b"content"))
diff --git a/tests/integration/sdk/test_filters.py b/tests/integration/sdk/test_filters.py
new file mode 100644
index 000000000..5cab4ba6a
--- /dev/null
+++ b/tests/integration/sdk/test_filters.py
@@ -0,0 +1,95 @@
+"""Tests for knowledge filter CRUD and usage in chat/search."""
+
+import os
+
+import pytest
+
+pytestmark = pytest.mark.skipif(
+ os.environ.get("SKIP_SDK_INTEGRATION_TESTS") == "true",
+ reason="SDK integration tests skipped",
+)
+
+
+class TestKnowledgeFilters:
+ """Test knowledge filter create, read, update, delete and usage."""
+
+ @pytest.mark.asyncio
+ async def test_knowledge_filter_crud(self, client):
+ """Full CRUD lifecycle for a knowledge filter."""
+ create_result = await client.knowledge_filters.create({
+ "name": "Python SDK Test Filter",
+ "description": "Filter created by Python SDK integration tests",
+ "queryData": {
+ "query": "test documents",
+ "limit": 10,
+ "scoreThreshold": 0.5,
+ },
+ })
+ assert create_result.success is True
+ assert create_result.id is not None
+ filter_id = create_result.id
+
+ # Search
+ filters = await client.knowledge_filters.search("Python SDK Test")
+ assert isinstance(filters, list)
+ assert any(f.name == "Python SDK Test Filter" for f in filters)
+
+ # Get
+ filter_obj = await client.knowledge_filters.get(filter_id)
+ assert filter_obj is not None
+ assert filter_obj.id == filter_id
+ assert filter_obj.name == "Python SDK Test Filter"
+
+ # Update
+ update_success = await client.knowledge_filters.update(
+ filter_id,
+ {"description": "Updated description from Python SDK test"},
+ )
+ assert update_success is True
+
+ updated_filter = await client.knowledge_filters.get(filter_id)
+ assert updated_filter.description == "Updated description from Python SDK test"
+
+ # Delete
+ delete_success = await client.knowledge_filters.delete(filter_id)
+ assert delete_success is True
+
+ deleted_filter = await client.knowledge_filters.get(filter_id)
+ assert deleted_filter is None
+
+ @pytest.mark.asyncio
+ async def test_filter_id_in_chat(self, client):
+ """A filter_id can be passed to chat without error."""
+ create_result = await client.knowledge_filters.create({
+ "name": "Chat Test Filter Python",
+ "description": "Filter for testing chat with filter_id",
+ "queryData": {"query": "test", "limit": 5},
+ })
+ assert create_result.success is True
+ filter_id = create_result.id
+
+ try:
+ response = await client.chat.create(
+ message="Hello with filter",
+ filter_id=filter_id,
+ )
+ assert response.response is not None
+ finally:
+ await client.knowledge_filters.delete(filter_id)
+
+ @pytest.mark.asyncio
+ async def test_filter_id_in_search(self, client):
+ """A filter_id can be passed to search without error."""
+ create_result = await client.knowledge_filters.create({
+ "name": "Search Test Filter Python",
+ "description": "Filter for testing search with filter_id",
+ "queryData": {"query": "test", "limit": 5},
+ })
+ assert create_result.success is True
+ filter_id = create_result.id
+
+ try:
+ results = await client.search.query("test query", filter_id=filter_id)
+ assert results.results is not None
+ finally:
+ await client.knowledge_filters.delete(filter_id)
diff --git a/tests/integration/sdk/test_models.py b/tests/integration/sdk/test_models.py
new file mode 100644
index 000000000..c86366116
--- /dev/null
+++ b/tests/integration/sdk/test_models.py
@@ -0,0 +1,24 @@
+"""Tests for the models endpoint."""
+
+import os
+
+import pytest
+
+pytestmark = pytest.mark.skipif(
+ os.environ.get("SKIP_SDK_INTEGRATION_TESTS") == "true",
+ reason="SDK integration tests skipped",
+)
+
+
+class TestModels:
+ """Test model listing per provider."""
+
+ @pytest.mark.asyncio
+ async def test_list_models(self, client):
+ """Listing models for a provider must return language and embedding model lists."""
+ models = await client.models.list("openai")
+
+ assert models.language_models is not None
+ assert isinstance(models.language_models, list)
+ assert models.embedding_models is not None
+ assert isinstance(models.embedding_models, list)
diff --git a/tests/integration/sdk/test_sdk.py b/tests/integration/sdk/test_sdk.py
deleted file mode 100644
index c1a202ba5..000000000
--- a/tests/integration/sdk/test_sdk.py
+++ /dev/null
@@ -1,467 +0,0 @@
-"""
-Integration tests for OpenRAG Python SDK.
-
-These tests run against a real OpenRAG instance.
-Requires: OPENRAG_URL environment variable (defaults to http://localhost:3000)
-
-Run with: pytest sdks/python/tests/test_integration.py -v
-"""
-
-import os
-from pathlib import Path
-
-import httpx
-import pytest
-
-# Skip all tests if no OpenRAG instance is available
-pytestmark = pytest.mark.skipif(
- os.environ.get("SKIP_SDK_INTEGRATION_TESTS") == "true",
- reason="SDK integration tests skipped",
-)
-
-# Module-level cache for API key (created once, reused)
-_cached_api_key: str | None = None
-_base_url = os.environ.get("OPENRAG_URL", "http://localhost:3000")
-_onboarding_done = False
-
-
[email protected](scope="session", autouse=True)
-def ensure_onboarding():
- """Ensure the OpenRAG instance is onboarded before running tests.
-
- This marks the config as 'edited' so that settings updates are allowed.
- """
- global _onboarding_done
- if _onboarding_done:
- return
-
- onboarding_payload = {
- "llm_provider": "openai",
- "embedding_provider": "openai",
- "embedding_model": "text-embedding-3-small",
- "llm_model": "gpt-4o-mini",
- }
-
- try:
- response = httpx.post(
- f"{_base_url}/api/onboarding",
- json=onboarding_payload,
- timeout=30.0,
- )
- if response.status_code in (200, 204):
- print(f"[SDK Tests] Onboarding completed successfully")
- else:
- # May already be onboarded, which is fine
- print(f"[SDK Tests] Onboarding returned {response.status_code}: {response.text[:200]}")
- except Exception as e:
- print(f"[SDK Tests] Onboarding request failed: {e}")
-
- _onboarding_done = True
-
-
-def get_api_key() -> str:
- """Get or create an API key for testing."""
- global _cached_api_key
- if _cached_api_key is None:
- # Use /api/keys to go through frontend proxy (frontend at :3000 proxies /api/* to backend)
- response = httpx.post(
- f"{_base_url}/api/keys",
- json={"name": "SDK Integration Test"},
- timeout=30.0,
- )
- if response.status_code == 401:
- pytest.skip("Cannot create API key - authentication required")
- assert response.status_code == 200, f"Failed to create API key: {response.text}"
- _cached_api_key = response.json()["api_key"]
- return _cached_api_key
-
-
-def client():
- """Create an OpenRAG client for each test."""
- from openrag_sdk import OpenRAGClient
-
- return OpenRAGClient(api_key=get_api_key(), base_url=_base_url)
-
-
-def test_file(tmp_path) -> Path:
- """Create a test file for ingestion with unique content."""
- import uuid
- # Use .md extension - Langflow has issues with .txt files
- file_path = tmp_path / f"sdk_test_doc_{uuid.uuid4().hex[:8]}.md"
- file_path.write_text(
- f"# SDK Integration Test Document\n\n"
- f"ID: {uuid.uuid4()}\n\n"
- "This document tests the OpenRAG Python SDK.\n\n"
- "It contains unique content about purple elephants dancing.\n"
- )
- return file_path
-
-
-class TestSettings:
- """Test settings endpoint."""
-
- @pytest.mark.asyncio
- async def test_get_settings(self, client):
- """Test getting settings."""
- settings = await client.settings.get()
-
- assert settings.agent is not None
- assert settings.knowledge is not None
-
- @pytest.mark.asyncio
- async def test_update_settings(self, client):
- """Test updating settings."""
- # Get current settings first
- current_settings = await client.settings.get()
- current_chunk_size = current_settings.knowledge.chunk_size or 1000
-
- # Update with the same value (safe for tests)
- result = await client.settings.update({"chunk_size": current_chunk_size})
-
- assert result.message is not None
-
- # Verify the setting persisted
- updated_settings = await client.settings.get()
- assert updated_settings.knowledge.chunk_size == current_chunk_size
-
-
-class TestModels:
- """Test models endpoint."""
-
- @pytest.mark.asyncio
- async def test_list_models(self, client):
- """Test listing models for a provider."""
- # This tests both the API key auth and the minted JWT
- # since models_service often needs credentials/JWT
- models = await client.models.list("openai")
-
- assert models.language_models is not None
- assert isinstance(models.language_models, list)
- assert models.embedding_models is not None
- assert isinstance(models.embedding_models, list)
-
-
-class TestKnowledgeFilters:
- """Test knowledge filter operations."""
-
- @pytest.mark.asyncio
- async def test_knowledge_filter_crud(self, client):
- """Test create, read, update, delete for knowledge filters."""
- # Create
- create_result = await client.knowledge_filters.create({
- "name": "Python SDK Test Filter",
- "description": "Filter created by Python SDK integration tests",
- "queryData": {
- "query": "test documents",
- "limit": 10,
- "scoreThreshold": 0.5,
- },
- })
-
- assert create_result.success is True
- assert create_result.id is not None
- filter_id = create_result.id
-
- # Search
- filters = await client.knowledge_filters.search("Python SDK Test")
- assert isinstance(filters, list)
- found = any(f.name == "Python SDK Test Filter" for f in filters)
- assert found is True
-
- # Get
- filter_obj = await client.knowledge_filters.get(filter_id)
- assert filter_obj is not None
- assert filter_obj.id == filter_id
- assert filter_obj.name == "Python SDK Test Filter"
-
- # Update
- update_success = await client.knowledge_filters.update(
- filter_id,
- {"description": "Updated description from Python SDK test"},
- )
- assert update_success is True
-
- # Verify update
- updated_filter = await client.knowledge_filters.get(filter_id)
- assert updated_filter.description == "Updated description from Python SDK test"
-
- # Delete
- delete_success = await client.knowledge_filters.delete(filter_id)
- assert delete_success is True
-
- # Verify deletion
- deleted_filter = await client.knowledge_filters.get(filter_id)
- assert deleted_filter is None
-
- @pytest.mark.asyncio
- async def test_filter_id_in_chat(self, client):
- """Test using filter_id in chat."""
- # Create a filter first
- create_result = await client.knowledge_filters.create({
- "name": "Chat Test Filter Python",
- "description": "Filter for testing chat with filter_id",
- "queryData": {
- "query": "test",
- "limit": 5,
- },
- })
- assert create_result.success is True
- filter_id = create_result.id
-
- try:
- # Use filter in chat
- response = await client.chat.create(
- message="Hello with filter",
- filter_id=filter_id,
- )
- assert response.response is not None
- finally:
- # Cleanup
- await client.knowledge_filters.delete(filter_id)
-
- @pytest.mark.asyncio
- async def test_filter_id_in_search(self, client):
- """Test using filter_id in search."""
- # Create a filter first
- create_result = await client.knowledge_filters.create({
- "name": "Search Test Filter Python",
- "description": "Filter for testing search with filter_id",
- "queryData": {
- "query": "test",
- "limit": 5,
- },
- })
- assert create_result.success is True
- filter_id = create_result.id
-
- try:
- # Use filter in search
- results = await client.search.query("test query", filter_id=filter_id)
- assert results.results is not None
- finally:
- # Cleanup
- await client.knowledge_filters.delete(filter_id)
-
-
-class TestDocuments:
- """Test document operations."""
-
- @pytest.mark.asyncio
- async def test_ingest_document_no_wait(self, client, test_file: Path):
- """Test document ingestion without waiting."""
- # wait=False returns immediately with task_id
- result = await client.documents.ingest(file_path=str(test_file), wait=False)
-
- assert result.task_id is not None
-
- # Can poll manually
- final_status = await client.documents.wait_for_task(result.task_id)
- # TODO: Fix Langflow ingestion - status may be "failed" due to flow issues
- assert final_status.status is not None
- assert final_status.successful_files >= 0
-
- @pytest.mark.asyncio
- async def test_ingest_document(self, client, test_file: Path):
- """Test document ingestion."""
- # wait=True (default) polls until completion
- result = await client.documents.ingest(file_path=str(test_file))
-
- # TODO: Fix Langflow ingestion - status may be "failed" due to flow issues
- assert result.status is not None
- assert result.successful_files >= 0
-
-
- @pytest.mark.asyncio
- async def test_delete_document(self, client, test_file: Path):
- """Test document deletion."""
- # First ingest (wait for completion)
- ingest_result = await client.documents.ingest(file_path=str(test_file))
-
- # Then delete
- result = await client.documents.delete(test_file.name)
-
- # If ingestion produced indexed chunks, delete should succeed.
- # In unstable flow environments, ingestion can complete with zero successful files.
- if ingest_result.successful_files > 0:
- assert result.success is True
- assert result.deleted_chunks > 0
- else:
- assert result.success is False
- assert result.deleted_chunks == 0
-
- @pytest.mark.asyncio
- async def test_delete_missing_document_is_idempotent(self, client):
- """Deleting a never-ingested filename should not raise."""
- import uuid
-
- missing_filename = f"never_ingested_{uuid.uuid4().hex}.pdf"
- result = await client.documents.delete(missing_filename)
-
- assert result.success is False
- assert result.deleted_chunks == 0
- assert result.filename == missing_filename
- assert result.error is not None
-
-
-class TestSearch:
- """Test search operations."""
-
- @pytest.mark.asyncio
- async def test_search_query(self, client, test_file: Path):
- """Test search query."""
- # Ensure document is ingested
- await client.documents.ingest(file_path=str(test_file))
-
- # Wait a bit for indexing
- import asyncio
- await asyncio.sleep(2)
-
- # Search for unique content
- results = await client.search.query("purple elephants dancing")
-
- assert results.results is not None
- # Note: might be empty if indexing is slow, that's ok for CI
-
-
-class TestChat:
- """Test chat operations."""
-
- @pytest.mark.asyncio
- async def test_chat_non_streaming(self, client):
- """Test non-streaming chat."""
- response = await client.chat.create(
- message="Say hello in exactly 3 words."
- )
-
- assert response.response is not None
- assert isinstance(response.response, str)
- assert len(response.response) > 0
-
- @pytest.mark.asyncio
- async def test_chat_streaming_create(self, client):
- """Test streaming chat with create(stream=True)."""
- collected_text = ""
-
- async for event in await client.chat.create(
- message="Say 'test' and nothing else.",
- stream=True,
- ):
- if event.type == "content":
- collected_text += event.delta
-
- assert len(collected_text) > 0
-
- @pytest.mark.asyncio
- async def test_chat_streaming_context_manager(self, client):
- """Test streaming chat with stream() context manager."""
- async with client.chat.stream(
- message="Say 'hello' and nothing else."
- ) as stream:
- async for _ in stream:
- pass
-
- # Check aggregated properties
- assert len(stream.text) > 0
-
- @pytest.mark.asyncio
- async def test_chat_text_stream(self, client):
- """Test text_stream helper."""
- collected = ""
-
- async with client.chat.stream(
- message="Say 'world' and nothing else."
- ) as stream:
- async for text in stream.text_stream:
- collected += text
-
- assert len(collected) > 0
-
- @pytest.mark.asyncio
- async def test_chat_final_text(self, client):
- """Test final_text() helper."""
- async with client.chat.stream(
- message="Say 'done' and nothing else."
- ) as stream:
- text = await stream.final_text()
-
- assert len(text) > 0
-
- @pytest.mark.asyncio
- async def test_chat_conversation_continuation(self, client):
- """Test continuing a conversation."""
- # First message
- response1 = await client.chat.create(
- message="Remember the number 42."
- )
- assert response1.chat_id is not None
-
- # Continue conversation
- response2 = await client.chat.create(
- message="What number did I ask you to remember?",
- chat_id=response1.chat_id,
- )
- assert response2.response is not None
-
- @pytest.mark.asyncio
- async def test_list_conversations(self, client):
- """Test listing conversations."""
- # Create a conversation first
- await client.chat.create(message="Test message for listing.")
-
- # List conversations
- result = await client.chat.list()
-
- assert result.conversations is not None
- assert isinstance(result.conversations, list)
-
- @pytest.mark.asyncio
- async def test_get_conversation(self, client):
- """Test getting a specific conversation."""
- # Create a conversation first
- response = await client.chat.create(message="Test message for get.")
- assert response.chat_id is not None
-
- # Get the conversation
- conversation = await client.chat.get(response.chat_id)
-
- assert conversation.chat_id == response.chat_id
- assert conversation.messages is not None
- assert isinstance(conversation.messages, list)
- assert len(conversation.messages) >= 1
-
- @pytest.mark.asyncio
- async def test_delete_conversation(self, client):
- """Test deleting a conversation."""
- # Create a conversation first
- response = await client.chat.create(message="Test message for delete.")
- assert response.chat_id is not None
-
- # Delete the conversation
- result = await client.chat.delete(response.chat_id)
-
- assert result is True
-
- @pytest.mark.asyncio
- async def test_chat_with_sources(self, client, test_file: Path):
- """Test chat uses embedded knowledge (RAG), not just pure LLM."""
- # 1. Ingest document
- result = await client.documents.ingest(file_path=str(test_file))
- if result.status == "failed" or result.successful_files == 0:
- pytest.skip("Document ingestion failed — cannot test RAG sources")
-
- # 2. Wait for indexing
- import asyncio
- await asyncio.sleep(3)
-
- # 3. Chat about document content
- response = await client.chat.create(
- message="What is the color of the dancing animals mentioned in my documents?"
- )
-
- # 4. Verify sources — proves RAG retrieval worked
- assert response.sources is not None
- assert len(response.sources) > 0
- source_filenames = [s.filename for s in response.sources]
- assert any(test_file.name in name for name in source_filenames)
diff --git a/tests/integration/sdk/test_search.py b/tests/integration/sdk/test_search.py
new file mode 100644
index 000000000..f0acf9819
--- /dev/null
+++ b/tests/integration/sdk/test_search.py
@@ -0,0 +1,76 @@
+"""Tests for the search endpoint."""
+
+import asyncio
+import os
+from pathlib import Path
+
+import pytest
+
+pytestmark = pytest.mark.skipif(
+ os.environ.get("SKIP_SDK_INTEGRATION_TESTS") == "true",
+ reason="SDK integration tests skipped",
+)
+
+
+class TestSearch:
+ """Core search query tests."""
+
+ @pytest.mark.asyncio
+ async def test_search_query(self, client, test_file: Path):
+ """A basic search query returns a results list."""
+ await client.documents.ingest(file_path=str(test_file))
+ await asyncio.sleep(2)
+
+ results = await client.search.query("purple elephants dancing")
+ assert results.results is not None
+
+
+class TestSearchExtended:
+ """Additional search parameter and edge-case tests."""
+
+ @pytest.mark.asyncio
+ async def test_search_with_limit(self, client, test_file: Path):
+ """limit parameter caps the number of results returned."""
+ await client.documents.ingest(file_path=str(test_file))
+ await asyncio.sleep(2)
+
+ results = await client.search.query("test", limit=1)
+ assert results.results is not None
+ assert len(results.results) <= 1
+
+ @pytest.mark.asyncio
+ async def test_search_with_high_score_threshold_returns_empty(self, client, test_file: Path):
+ """A score_threshold of 0.99 should filter out most or all results."""
+ await client.documents.ingest(file_path=str(test_file))
+ await asyncio.sleep(2)
+
+ results = await client.search.query("test", score_threshold=0.99)
+ assert results.results is not None
+ assert isinstance(results.results, list)
+
+ @pytest.mark.asyncio
+ async def test_search_no_results_for_obscure_query(self, client):
+ """A nonsense query must return an empty list, not raise an error."""
+ results = await client.search.query(
+ "zzz_xyzzy_nonexistent_content_abc123_qwerty_999"
+ )
+ assert results.results is not None
+ assert isinstance(results.results, list)
+
+ @pytest.mark.asyncio
+ async def test_search_unicode_query(self, client):
+ """Unicode and emoji characters in the query must not cause an error."""
+ results = await client.search.query("こんにちは 🦩 Ñoño résumé")
+ assert results.results is not None
+ assert isinstance(results.results, list)
+
+ @pytest.mark.asyncio
+ async def test_search_returns_result_fields(self, client, test_file: Path):
+ """Each search result must have content populated as a string."""
+ await client.documents.ingest(file_path=str(test_file))
+ await asyncio.sleep(2)
+
+ results = await client.search.query("purple elephants dancing", limit=5)
+ for result in results.results:
+ assert result.content is not None
+ assert isinstance(result.content, str)
diff --git a/tests/integration/sdk/test_settings.py b/tests/integration/sdk/test_settings.py
new file mode 100644
index 000000000..0f28b12d0
--- /dev/null
+++ b/tests/integration/sdk/test_settings.py
@@ -0,0 +1,34 @@
+"""Tests for the settings endpoint."""
+
+import os
+
+import pytest
+
+pytestmark = pytest.mark.skipif(
+ os.environ.get("SKIP_SDK_INTEGRATION_TESTS") == "true",
+ reason="SDK integration tests skipped",
+)
+
+
+class TestSettings:
+ """Test settings get and update operations."""
+
+ @pytest.mark.asyncio
+ async def test_get_settings(self, client):
+ """Settings response must include agent and knowledge sections."""
+ settings = await client.settings.get()
+
+ assert settings.agent is not None
+ assert settings.knowledge is not None
+
+ @pytest.mark.asyncio
+ async def test_update_settings(self, client):
+ """Updating a setting must persist and be readable back."""
+ current_settings = await client.settings.get()
+ current_chunk_size = current_settings.knowledge.chunk_size or 1000
+
+ result = await client.settings.update({"chunk_size": current_chunk_size})
+ assert result.message is not None
+
+ updated_settings = await client.settings.get()
+ assert updated_settings.knowledge.chunk_size == current_chunk_size