Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

feat: Improve SDK integration test coverage (Python + TypeScript) (#2153)

Edwin Josef485358d3e7b
diff --git a/.github/workflows/test-ci.yml b/.github/workflows/test-ci.yml
index 80944e8a1..d03d21ca0 100644
--- a/.github/workflows/test-ci.yml
+++ b/.github/workflows/test-ci.yml
@@ -220,7 +220,7 @@ jobs:
docker compose -f docker-compose.yml down -v --remove-orphans || true
docker system prune -f || true
for i in 1 2 3; do
- docker run --rm -v $(pwd):/work alpine sh -c "rm -rf /work/opensearch-data /work/config /work/langflow-data /work/keys /work/data /work/flows /work/openrag-documents" && break
+ docker run --rm -v $(pwd):/work alpine sh -c "rm -rf /work/opensearch-data /work/config /work/langflow-data /work/keys /work/data /work/flows/backup /work/openrag-documents" && break
echo "Attempt $i failed, retrying in 5s..."
sleep 5
done || true
@@ -332,7 +332,7 @@ jobs:
make clean || true
docker system prune -f || true
for i in 1 2 3; do
- docker run --rm -v $(pwd):/work alpine sh -c "rm -rf /work/opensearch-data /work/config /work/langflow-data /work/keys /work/data /work/flows /work/openrag-documents" && break
+ docker run --rm -v $(pwd):/work alpine sh -c "rm -rf /work/opensearch-data /work/config /work/langflow-data /work/keys /work/data /work/flows/backup /work/openrag-documents" && break
echo "Attempt $i failed, retrying in 5s..."
sleep 5
done || true
@@ -505,7 +505,7 @@ jobs:
make clean || true
docker system prune -f || true
for i in 1 2 3; do
- docker run --rm -v $(pwd):/work alpine sh -c "rm -rf /work/opensearch-data /work/config /work/langflow-data /work/keys /work/data /work/flows /work/openrag-documents" && break
+ docker run --rm -v $(pwd):/work alpine sh -c "rm -rf /work/opensearch-data /work/config /work/langflow-data /work/keys /work/data /work/flows/backup /work/openrag-documents" && break
echo "Attempt $i failed, retrying in 5s..."
sleep 5
done || true
diff --git a/scripts/ci/run_integration_suite.sh b/scripts/ci/run_integration_suite.sh
index 43a8cf00e..d62d616e1 100755
--- a/scripts/ci/run_integration_suite.sh
+++ b/scripts/ci/run_integration_suite.sh
@@ -122,16 +122,20 @@ clean_infra() {
exit_status=1
fi
+ # NOTE: flows/*.json are git-tracked flow definitions, not runtime state -- only
+ # flows/backup is generated at runtime. Deleting the top-level flows/ directory here
+ # would remove those tracked files for the remainder of this job (no re-checkout
+ # happens before a retry), breaking every flow-dependent request afterwards.
if command -v sudo >/dev/null 2>&1; then
- if ! sudo rm -rf langflow-data config data keys opensearch-data openrag-documents flows; then
+ if ! sudo rm -rf langflow-data config data keys opensearch-data openrag-documents flows/backup; then
echo "${red}ERROR: sudo rm -rf failed${nc}"
exit_status=1
fi
else
- if ! rm -rf langflow-data config data keys opensearch-data openrag-documents flows 2>/dev/null; then
+ if ! rm -rf langflow-data config data keys opensearch-data openrag-documents flows/backup 2>/dev/null; then
local docker_rm_success=false
for i in 1 2 3; do
- if docker run --rm -v "$(pwd):/work" alpine sh -c "rm -rf /work/opensearch-data /work/config /work/langflow-data /work/keys /work/data /work/flows /work/openrag-documents"; then
+ if docker run --rm -v "$(pwd):/work" alpine sh -c "rm -rf /work/opensearch-data /work/config /work/langflow-data /work/keys /work/data /work/flows/backup /work/openrag-documents"; then
docker_rm_success=true
break
fi
diff --git a/sdks/typescript/src/types.ts b/sdks/typescript/src/types.ts
index 2d2336ee8..44b7c9acd 100644
--- a/sdks/typescript/src/types.ts
+++ b/sdks/typescript/src/types.ts
@@ -115,6 +115,7 @@ export interface ConversationListResponse {
export interface AgentSettings {
llm_provider?: string | null;
llm_model?: string | null;
+ system_prompt?: string | null;
}
export interface KnowledgeSettings {
@@ -122,6 +123,9 @@ export interface KnowledgeSettings {
embedding_model?: string | null;
chunk_size?: number | null;
chunk_overlap?: number | null;
+ table_structure?: boolean | null;
+ ocr?: boolean | null;
+ picture_descriptions?: boolean | null;
}
export interface SettingsResponse {
diff --git a/sdks/typescript/tests/integration.test.ts b/sdks/typescript/tests/integration.test.ts
index f52110090..b0fff1991 100644
--- a/sdks/typescript/tests/integration.test.ts
+++ b/sdks/typescript/tests/integration.test.ts
@@ -7,13 +7,17 @@
* Run with: npm test
*/
-import { describe, it, expect, beforeAll } from "vitest";
+import { describe, it, expect, beforeAll, afterAll } from "vitest";
import * as fs from "fs";
import * as path from "path";
import * as os from "os";
+import type { DoneEvent, StreamEvent } from "../src";
// Dynamic import to handle the SDK not being built yet
let OpenRAGClient: typeof import("../src").OpenRAGClient;
+let ValidationError: typeof import("../src").ValidationError;
+let NotFoundError: typeof import("../src").NotFoundError;
+let AuthenticationError: typeof import("../src").AuthenticationError;
const BASE_URL = process.env.OPENRAG_URL || "http://localhost:3000";
const SKIP_TESTS = process.env.SKIP_SDK_INTEGRATION_TESTS === "true";
@@ -91,6 +95,9 @@ describe.skipIf(SKIP_TESTS)("OpenRAG TypeScript SDK Integration", () => {
// Import SDK
const sdk = await import("../src");
OpenRAGClient = sdk.OpenRAGClient;
+ ValidationError = sdk.ValidationError;
+ NotFoundError = sdk.NotFoundError;
+ AuthenticationError = sdk.AuthenticationError;
// Create API key and client
const apiKey = await createApiKey();
@@ -100,12 +107,86 @@ describe.skipIf(SKIP_TESTS)("OpenRAG TypeScript SDK Integration", () => {
testFilePath = createTestFile();
});
+ describe("Auth", () => {
+ it("constructs without an apiKey when extraHeaders are provided, and extraHeaders authenticate real requests", async () => {
+ // Mirrors IBM SaaS-style auth, where a gateway injects headers instead of
+ // an SDK-level api key. The TS client has no api-key-or-extraHeaders
+ // construction check (unlike the Python SDK), so this call is expected to
+ // succeed unconditionally; what we're really proving is that a request
+ // authenticated purely via extraHeaders (no `apiKey` option at all) goes
+ // through end-to-end. We forward a real API key via extraHeaders rather
+ // than IBM-specific X-Username/X-Api-Key headers, since those only
+ // authenticate when the server has IBM auth mode enabled, which isn't
+ // guaranteed in a generic test environment — _getHeaders() in client.ts
+ // merges extraHeaders in verbatim, so this exercises the exact same
+ // "auth carried entirely by extraHeaders" code path deterministically.
+ const apiKey = await createApiKey();
+
+ let extraHeadersClient: InstanceType<typeof OpenRAGClient>;
+ expect(() => {
+ extraHeadersClient = new OpenRAGClient({
+ baseUrl: BASE_URL,
+ extraHeaders: { "X-API-Key": apiKey },
+ });
+ }).not.toThrow();
+
+ const settings = await extraHeadersClient!.settings.get();
+ expect(settings.agent).toBeDefined();
+ expect(settings.knowledge).toBeDefined();
+ });
+
+ it("an invalid API key raises AuthenticationError with statusCode 401/403", async () => {
+ const badClient = new OpenRAGClient({
+ apiKey: "orag_invalid_key_for_testing",
+ baseUrl: BASE_URL,
+ });
+
+ let threw = false;
+ try {
+ await badClient.settings.get();
+ } catch (e) {
+ threw = true;
+ expect(e).toBeInstanceOf(AuthenticationError);
+ expect([401, 403]).toContain((e as InstanceType<typeof AuthenticationError>).statusCode);
+ }
+ expect(threw).toBe(true);
+ });
+ });
+
describe("Settings", () => {
it("should get settings", async () => {
const settings = await client.settings.get();
expect(settings.agent).toBeDefined();
expect(settings.knowledge).toBeDefined();
+
+ // Newly-typed fields (types.ts) — type-or-null/undefined checks, mirroring
+ // how chunk_size is treated elsewhere in this block (may not be set yet).
+ expect(
+ settings.knowledge.chunk_overlap === undefined ||
+ settings.knowledge.chunk_overlap === null ||
+ typeof settings.knowledge.chunk_overlap === "number"
+ ).toBe(true);
+ expect(
+ settings.knowledge.table_structure === undefined ||
+ settings.knowledge.table_structure === null ||
+ typeof settings.knowledge.table_structure === "boolean"
+ ).toBe(true);
+ expect(
+ settings.knowledge.ocr === undefined ||
+ settings.knowledge.ocr === null ||
+ typeof settings.knowledge.ocr === "boolean"
+ ).toBe(true);
+ expect(
+ settings.knowledge.picture_descriptions === undefined ||
+ settings.knowledge.picture_descriptions === null ||
+ typeof settings.knowledge.picture_descriptions === "boolean"
+ ).toBe(true);
+ expect(
+ settings.agent.system_prompt === undefined ||
+ settings.agent.system_prompt === null ||
+ typeof settings.agent.system_prompt === "string"
+ ).toBe(true);
});
it("should update settings", async () => {
@@ -154,6 +235,18 @@ describe.skipIf(SKIP_TESTS)("OpenRAG TypeScript SDK Integration", () => {
expect(found).toBe(true);
});
+ it("should return an empty array when a knowledge filter search matches nothing", async () => {
+ const filters = await client.knowledgeFilters.search(
+ `zzzz_no_such_filter_${Date.now()}_qqqq`
+ );
+ expect(filters).toEqual([]);
+ });
+
+ it("should respect the limit param on knowledge filter search", async () => {
+ const filters = await client.knowledgeFilters.search("", 1);
+ expect(filters.length).toBeLessThanOrEqual(1);
+ });
+
it("should get a knowledge filter by ID", async () => {
expect(createdFilterId).toBeDefined();
@@ -162,6 +255,11 @@ describe.skipIf(SKIP_TESTS)("OpenRAG TypeScript SDK Integration", () => {
expect(filter).not.toBeNull();
expect(filter!.id).toBe(createdFilterId);
expect(filter!.name).toBe("SDK Test Filter");
+ expect(filter!.queryData).toBeDefined();
+ expect(filter!.queryData.query).toBe("test documents");
+ expect(typeof filter!.owner).toBe("string");
+ expect(typeof filter!.createdAt).toBe("string");
+ expect(typeof filter!.updatedAt).toBe("string");
});
it("should update a knowledge filter", async () => {
@@ -190,6 +288,26 @@ describe.skipIf(SKIP_TESTS)("OpenRAG TypeScript SDK Integration", () => {
expect(filter).toBeNull();
});
+ it("create() fails when the required name field is missing", async () => {
+ // `name` is required by CreateKnowledgeFilterOptions; bypass the type
+ // system with `as any` to exercise server-side validation. Per
+ // knowledge-filters.ts, create() does not catch request errors, so a
+ // server-side 400 surfaces as a thrown ValidationError rather than a
+ // `{ success: false }` response — but we also accept the latter shape
+ // in case the server instead responds 200 with success:false.
+ try {
+ const result = await client.knowledgeFilters.create({
+ description: "Missing required name field",
+ queryData: { query: "test" },
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ } as any);
+ expect(result.success).toBe(false);
+ expect(result.error).toBeTruthy();
+ } catch (e) {
+ expect((e as Error).message).toBeTruthy();
+ }
+ });
+
it("filterId in chat actually scopes retrieval to data_sources", async () => {
// Ingest two distinguishable docs.
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "sdk-filter-"));
@@ -311,6 +429,9 @@ describe.skipIf(SKIP_TESTS)("OpenRAG TypeScript SDK Integration", () => {
expect(result.success).toBe(true);
expect(result.filenames).toContain(alphaName);
expect(result.filenames ?? []).not.toContain(betaName);
+ // per_file has one entry per resolved data_source (see types.ts).
+ expect(Array.isArray(result.per_file)).toBe(true);
+ expect(result.per_file!.length).toBe(result.filenames!.length);
// Beta still searchable
const remaining = await client.search.query("tigers");
@@ -335,14 +456,34 @@ describe.skipIf(SKIP_TESTS)("OpenRAG TypeScript SDK Integration", () => {
});
describe("Documents", () => {
+ // Filenames ingested by this block that must be removed afterwards so
+ // they don't leak into subsequent test runs.
+ const createdFilenames: string[] = [];
+
+ afterAll(async () => {
+ const uniqueFilenames = Array.from(new Set(createdFilenames));
+ for (const filename of uniqueFilenames) {
+ try {
+ await client.documents.delete(filename);
+ } catch {
+ // Best-effort cleanup; some of these may already be deleted by
+ // the tests themselves.
+ }
+ }
+ });
+
it("should ingest a document (wait for completion)", async () => {
// wait=true (default) polls until completion
const result = await client.documents.ingest({ filePath: testFilePath });
+ createdFilenames.push(path.basename(testFilePath));
// TODO: Fix Langflow ingestion flow - currently returns 0 successful files
// due to embedding model component errors in layer 0
expect(result.status).toBeDefined();
expect((result as any).successful_files).toBeGreaterThanOrEqual(0);
+ expect((result as any).total_files).toBeGreaterThanOrEqual(0);
+ expect((result as any).processed_files).toBeGreaterThanOrEqual(0);
+ expect((result as any).failed_files).toBeGreaterThanOrEqual(0);
}, 120_000);
it("should ingest a document without waiting", async () => {
@@ -351,6 +492,7 @@ describe.skipIf(SKIP_TESTS)("OpenRAG TypeScript SDK Integration", () => {
filePath: testFilePath,
wait: false,
});
+ createdFilenames.push(path.basename(testFilePath));
expect((result as any).task_id).toBeDefined();
@@ -362,6 +504,37 @@ describe.skipIf(SKIP_TESTS)("OpenRAG TypeScript SDK Integration", () => {
expect(finalStatus.status).toBeDefined();
});
+ it("waitForTask throws when timeout elapses before any poll", async () => {
+ // Racing a real task's duration (e.g. a small nonzero timeout) is
+ // inherently flaky: if the task reaches a terminal status on the very
+ // first poll -- which can happen near-instantly, including on failure
+ // -- waitForTask resolves normally instead of throwing, regardless of
+ // how small the timeout was. timeout=0 makes the elapsed<timeoutMs loop
+ // guard false before the first poll, so this is deterministic and
+ // doesn't require a real ingest (the task id is never even queried).
+ await expect(
+ client.documents.waitForTask("nonexistent-id", 0.1, 0)
+ ).rejects.toThrow();
+ });
+
+ it("getTaskStatus throws NotFoundError (404) for a nonexistent task", async () => {
+ let threw = false;
+ try {
+ await client.documents.getTaskStatus("nonexistent-id");
+ } catch (e) {
+ threw = true;
+ expect(e).toBeInstanceOf(NotFoundError);
+ expect((e as InstanceType<typeof NotFoundError>).statusCode).toBe(404);
+ }
+ expect(threw).toBe(true);
+ });
+
+ it("delete({ filterId }) throws NotFoundError for a nonexistent filter id", async () => {
+ await expect(
+ client.documents.delete({ filterId: "nonexistent-uuid-like-value" })
+ ).rejects.toThrow(NotFoundError);
+ });
+
it("should delete a document", async () => {
// Use a uniquely named file so this test doesn't inherit chunks left in
// the index by the two ingest tests above (which share testFilePath /
@@ -380,26 +553,36 @@ describe.skipIf(SKIP_TESTS)("OpenRAG TypeScript SDK Integration", () => {
"This document tests document deletion via the OpenRAG TypeScript SDK.\n" +
"It contains unique content about teal dolphins swimming.\n"
);
+ const deleteFilename = path.basename(deleteFilePath);
+ createdFilenames.push(deleteFilename);
// First ingest (wait for completion)
const ingestResult = await client.documents.ingest({
filePath: deleteFilePath,
});
+ // Make ingestion deterministically succeed before asserting delete
+ // behavior — fail clearly instead of branching the assertions below on
+ // whether ingestion happened to index anything.
+ if (
+ !("successful_files" in ingestResult) ||
+ ingestResult.successful_files <= 0
+ ) {
+ throw new Error(
+ `Ingestion did not index any files for ${deleteFilename} ` +
+ `(successful_files=${
+ "successful_files" in ingestResult
+ ? ingestResult.successful_files
+ : "unknown"
+ }); cannot verify delete behavior deterministically.`
+ );
+ }
+
// Then delete
- const result = await client.documents.delete(
- path.basename(deleteFilePath)
- );
+ const result = await client.documents.delete(deleteFilename);
- // If ingestion produced indexed chunks, delete should succeed.
- // In unstable flow environments, ingestion can complete with zero successful files.
- if ("successful_files" in ingestResult && ingestResult.successful_files > 0) {
- expect(result.success).toBe(true);
- expect(result.deleted_chunks).toBeGreaterThan(0);
- } else {
- expect(result.success).toBe(false);
- expect(result.deleted_chunks).toBe(0);
- }
+ expect(result.success).toBe(true);
+ expect(result.deleted_chunks).toBeGreaterThan(0);
});
it("should treat delete of missing document as idempotent", async () => {
@@ -423,10 +606,101 @@ describe.skipIf(SKIP_TESTS)("OpenRAG TypeScript SDK Integration", () => {
expect(results.results).toBeDefined();
expect(Array.isArray(results.results)).toBe(true);
+ for (const r of results.results) {
+ // score is a raw OpenSearch relevance score (boosted BM25/KNN
+ // hybrid), not normalized to [0, 1] -- it can exceed 1.
+ expect(r.score).toBeGreaterThanOrEqual(0);
+ }
+ });
+
+ it("should return an empty array for a query guaranteed not to match", async () => {
+ // search_service.py only applies OpenSearch's `min_score` filter when
+ // score_threshold > 0 (the default is 0, which applies no filter at
+ // all). Since search is hybrid BM25+KNN, an unfiltered nonsense query
+ // still returns nearest-neighbor hits -- nothing is "guaranteed not to
+ // match" without an explicit threshold. Reuse the same 0.5 cutoff the
+ // "should respect scoreThreshold" test below treats as meaningful.
+ const results = await client.search.query(
+ `zzzz_no_such_content_${Date.now()}_qqqq`,
+ { scoreThreshold: 0.5 }
+ );
+ expect(results.results).toEqual([]);
+ });
+
+ it("should respect scoreThreshold", async () => {
+ const results = await client.search.query("orange kangaroos jumping", {
+ scoreThreshold: 0.5,
+ });
+ expect(results.results.every((r) => r.score >= 0.5)).toBe(true);
+ });
+
+ it("should respect limit", async () => {
+ const results = await client.search.query("orange kangaroos jumping", {
+ limit: 5,
+ });
+ expect(results.results.length).toBeLessThanOrEqual(5);
+ });
+
+ it("should filter by a raw filters dict (data_sources)", async () => {
+ const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "sdk-search-filter-"));
+ const filterDocName = `search_filter_doc_${Date.now()}.md`;
+ const filterDocPath = path.join(tmpDir, filterDocName);
+ fs.writeFileSync(filterDocPath, "# Search Filter Doc\n\nMagenta wombats hopping.\n");
+ await client.documents.ingest({ filePath: filterDocPath });
+
+ try {
+ const results = await client.search.query("wombats", {
+ filters: { data_sources: [filterDocName] },
+ });
+ for (const r of results.results) {
+ expect(r.filename).toBe(filterDocName);
+ }
+ } finally {
+ await client.documents.delete(filterDocName);
+ }
+ }, 120_000);
+
+ it("should reject a whitespace-only query", async () => {
+ // ASSUMPTION (not verified against a live instance in this environment):
+ // the backend rejects blank/whitespace-only search queries with a 400,
+ // which the SDK surfaces as ValidationError. search.ts performs no
+ // client-side validation of its own, so this test's outcome depends
+ // entirely on server-side behavior. If the server does NOT reject
+ // whitespace queries, this test will fail and the validation should be
+ // added server-side (or this expectation revisited).
+ await expect(client.search.query(" ")).rejects.toThrow(ValidationError);
+ });
+
+ it("should include page and mimetype fields with correct optional types", async () => {
+ const results = await client.search.query("orange kangaroos jumping");
+ for (const r of results.results) {
+ expect(
+ r.page === undefined || r.page === null || typeof r.page === "number"
+ ).toBe(true);
+ expect(
+ r.mimetype === undefined ||
+ r.mimetype === null ||
+ typeof r.mimetype === "string"
+ ).toBe(true);
+ }
});
});
describe("Chat", () => {
+ // Conversations created by this block that must be cleaned up.
+ const createdChatIds: string[] = [];
+
+ afterAll(async () => {
+ const uniqueIds = Array.from(new Set(createdChatIds));
+ for (const chatId of uniqueIds) {
+ try {
+ await client.chat.delete(chatId);
+ } catch {
+ // Best-effort cleanup.
+ }
+ }
+ });
+
it("should send non-streaming chat", async () => {
const response = await client.chat.create({
message: "Say hello in exactly 3 words.",
@@ -435,21 +709,64 @@ describe.skipIf(SKIP_TESTS)("OpenRAG TypeScript SDK Integration", () => {
expect(response.response).toBeDefined();
expect(typeof response.response).toBe("string");
expect(response.response.length).toBeGreaterThan(0);
+
+ for (const source of response.sources ?? []) {
+ expect(typeof source.filename).toBe("string");
+ expect(typeof source.text).toBe("string");
+ expect(typeof source.score).toBe("number");
+ // score is a raw OpenSearch relevance score (boosted BM25/KNN
+ // hybrid), not normalized to [0, 1] -- it can exceed 1.
+ expect(source.score).toBeGreaterThanOrEqual(0);
+ }
+
+ if (response.chatId) {
+ createdChatIds.push(response.chatId);
+ }
+ });
+
+ it("should combine filters, limit, and scoreThreshold in one chat.create() call", async () => {
+ const response = await client.chat.create({
+ message: "What documents mention kangaroos?",
+ filters: { data_sources: [path.basename(testFilePath)] },
+ limit: 3,
+ scoreThreshold: 0,
+ });
+
+ expect(response.response).toBeDefined();
+ expect(typeof response.response).toBe("string");
+
+ if (response.chatId) {
+ createdChatIds.push(response.chatId);
+ }
});
it("should stream chat with create({ stream: true })", async () => {
let collectedText = "";
+ const events: StreamEvent[] = [];
for await (const event of await client.chat.create({
message: "Say 'test' and nothing else.",
stream: true,
})) {
+ events.push(event);
if (event.type === "content") {
collectedText += event.delta;
}
}
expect(collectedText.length).toBeGreaterThan(0);
+
+ // A "done" event must be observed and must be the LAST event collected.
+ const doneEvents = events.filter((e) => e.type === "done");
+ expect(doneEvents.length).toBeGreaterThanOrEqual(1);
+ const lastEvent = events[events.length - 1];
+ expect(lastEvent?.type).toBe("done");
+
+ const doneEvent = doneEvents[doneEvents.length - 1] as DoneEvent;
+ expect(doneEvent.chatId).toBeDefined();
+ if (doneEvent.chatId) {
+ createdChatIds.push(doneEvent.chatId);
+ }
});
it("should stream chat with stream() context manager", async () => {
@@ -463,6 +780,34 @@ describe.skipIf(SKIP_TESTS)("OpenRAG TypeScript SDK Integration", () => {
}
expect(stream.text.length).toBeGreaterThan(0);
+ expect(stream.chatId).toBeDefined();
+ expect(typeof stream.chatId).toBe("string");
+ if (stream.chatId) {
+ createdChatIds.push(stream.chatId);
+ }
+ } finally {
+ stream.close();
+ }
+ });
+
+ it("consuming a ChatStream twice throws", async () => {
+ const stream = await client.chat.stream({
+ message: "Say 'twice' and nothing else.",
+ });
+
+ try {
+ for await (const _ of stream) {
+ // Consume once, fully.
+ }
+ if (stream.chatId) {
+ createdChatIds.push(stream.chatId);
+ }
+
+ await expect(async () => {
+ for await (const _ of stream) {
+ // no-op; should throw before yielding anything
+ }
+ }).rejects.toThrow();
} finally {
stream.close();
}
@@ -481,6 +826,9 @@ describe.skipIf(SKIP_TESTS)("OpenRAG TypeScript SDK Integration", () => {
}
expect(collected.length).toBeGreaterThan(0);
+ if (stream.chatId) {
+ createdChatIds.push(stream.chatId);
+ }
} finally {
stream.close();
}
@@ -494,6 +842,9 @@ describe.skipIf(SKIP_TESTS)("OpenRAG TypeScript SDK Integration", () => {
try {
const text = await stream.finalText();
expect(text.length).toBeGreaterThan(0);
+ if (stream.chatId) {
+ createdChatIds.push(stream.chatId);
+ }
} finally {
stream.close();
}
@@ -512,17 +863,36 @@ describe.skipIf(SKIP_TESTS)("OpenRAG TypeScript SDK Integration", () => {
chatId: response1.chatId!,
});
expect(response2.response).toBeDefined();
+ expect(response2.chatId).toBe(response1.chatId);
+
+ if (response1.chatId) {
+ createdChatIds.push(response1.chatId);
+ }
});
it("should list conversations", async () => {
// Create a conversation first
- await client.chat.create({ message: "Test message for listing." });
+ const created = await client.chat.create({ message: "Test message for listing." });
+ if (created.chatId) {
+ createdChatIds.push(created.chatId);
+ }
// List conversations
const result = await client.chat.list();
expect(result.conversations).toBeDefined();
expect(Array.isArray(result.conversations)).toBe(true);
+ expect(result.conversations.length).toBeGreaterThan(0);
+
+ for (const conversation of result.conversations) {
+ expect(typeof conversation.chatId).toBe("string");
+ expect(typeof conversation.createdAt).toBe("string");
+ expect(conversation.createdAt!.length).toBeGreaterThan(0);
+ expect(typeof conversation.lastActivity).toBe("string");
+ expect(conversation.lastActivity!.length).toBeGreaterThan(0);
+ expect(typeof conversation.messageCount).toBe("number");
+ expect(conversation.messageCount).toBeGreaterThanOrEqual(0);
+ }
});
it("should get a specific conversation", async () => {
@@ -531,6 +901,9 @@ describe.skipIf(SKIP_TESTS)("OpenRAG TypeScript SDK Integration", () => {
message: "Test message for get.",
});
expect(response.chatId).toBeDefined();
+ if (response.chatId) {
+ createdChatIds.push(response.chatId);
+ }
// Get the conversation
const conversation = await client.chat.get(response.chatId!);
@@ -539,6 +912,11 @@ describe.skipIf(SKIP_TESTS)("OpenRAG TypeScript SDK Integration", () => {
expect(conversation.messages).toBeDefined();
expect(Array.isArray(conversation.messages)).toBe(true);
expect(conversation.messages.length).toBeGreaterThanOrEqual(1);
+
+ for (const message of conversation.messages) {
+ expect(["user", "assistant"]).toContain(message.role);
+ expect(typeof message.content).toBe("string");
+ }
});
it("should delete a conversation", async () => {
@@ -553,5 +931,16 @@ describe.skipIf(SKIP_TESTS)("OpenRAG TypeScript SDK Integration", () => {
expect(result).toBe(true);
});
+
+ it("deleting a nonexistent conversation throws NotFoundError", async () => {
+ // Unlike Python's chat.delete() (chat.py), which explicitly catches
+ // NotFoundError and returns False for idempotency, chat.ts's delete()
+ // has no such catch and lets the 404 propagate. This asserts TS's
+ // actual current behavior; the cross-SDK inconsistency (Python is
+ // idempotent here, TS is not) is worth aligning in a follow-up.
+ await expect(
+ client.chat.delete(`nonexistent-chat-${Date.now()}`)
+ ).rejects.toThrow(NotFoundError);
+ });
});
});
diff --git a/tests/integration/sdk/README.md b/tests/integration/sdk/README.md
index 2c40f2703..d7efd95d2 100644
--- a/tests/integration/sdk/README.md
+++ b/tests/integration/sdk/README.md
@@ -14,8 +14,11 @@ make test-sdk
| # | Test | Expected |
|---|------|----------|
| 1 | Construct client with no API key | Raises `AuthenticationError` immediately |
-| 2 | Send request with invalid API key | Raises `AuthenticationError` with status 401 or 403 |
-| 3 | Send request with well-formed but non-existent key | Raises `AuthenticationError` |
+| 2 | Construct client with `extra_headers` only (IBM auth mode) and make a real API call | Headers reach the server end-to-end; succeeds, or raises a clean `AuthenticationError` (401) if IBM auth mode isn't enabled on the target instance |
+| 3 | Construct client with only `OPENRAG_API_KEY` env var set | Client constructs without error |
+| 4 | Explicit `api_key` argument with `OPENRAG_API_KEY` env var also set | Explicit value takes precedence over the env var |
+| 5 | Send request with invalid API key | Raises `AuthenticationError` with status 401 or 403 |
+| 6 | Send request with well-formed but non-existent key | Raises `AuthenticationError` |
---
@@ -23,20 +26,21 @@ make test-sdk
| # | Test | Expected |
|---|------|----------|
-| 4 | Non-streaming chat | Returns non-empty response string |
-| 5 | Streaming chat (`create(stream=True)`) | Yields content events with text deltas |
-| 6 | Streaming via context manager (`stream()`) | Accumulated `stream.text` is non-empty |
-| 7 | `text_stream` async iterator | Yields plain text chunks |
-| 8 | `final_text()` | Returns full accumulated response |
-| 9 | Conversation continuation (pass `chat_id`) | Second reply uses same conversation |
-| 10 | List conversations | Returns list of conversations |
-| 11 | Get conversation by ID | Returns conversation with message history |
-| 12 | Delete existing conversation | Returns `True` |
-| 13 | Chat with ingested document (RAG) | Response sources include the ingested file |
-| 14 | Stream continuation with `chat_id` | Follow-up stream uses existing conversation |
-| 15 | Every response includes `chat_id` | `chat_id` is a non-empty string |
-| 16 | `chat_id` available after stream consumed | `stream.chat_id` is populated |
-| 17 | `sources` field on response | Always a list (may be empty) |
+| 7 | Non-streaming chat | Returns non-empty response string; any `sources` are valid `Source`s with `score` in `[0, 1]` |
+| 8 | Chat with `filters`, `limit`, and `score_threshold` together | `len(response.sources) <= limit` |
+| 9 | Streaming chat (`create(stream=True)`) | Yields content events with text deltas; the last event is `done` with a non-null `chat_id` |
+| 10 | Streaming via context manager (`stream()`) | Accumulated `stream.text` is non-empty |
+| 11 | `text_stream` async iterator | Yields plain text chunks |
+| 12 | `final_text()` | Returns full accumulated response |
+| 13 | Conversation continuation (pass `chat_id`) | Second reply uses same conversation; `followup.chat_id == chat_id` |
+| 14 | List conversations | Returns list of conversations; `title`, `created_at`, `last_activity`, `message_count` are correctly typed |
+| 15 | Get conversation by ID | Returns conversation with message history; message `role`/`content` are correctly typed |
+| 16 | Delete existing conversation | Returns `True` |
+| 17 | Chat with ingested document (RAG) | Response sources include the ingested file |
+| 18 | Stream continuation with `chat_id` | Follow-up stream uses existing conversation |
+| 19 | Every response includes `chat_id` | `chat_id` is a non-empty string |
+| 20 | `chat_id` available after stream consumed | `stream.chat_id` is populated |
+| 21 | `sources` field on response | Always a list (may be empty) |
---
@@ -44,14 +48,21 @@ make test-sdk
| # | Test | Expected |
|---|------|----------|
-| 18 | Ingest file (async, `wait=False`) | Returns `task_id`; polling reaches terminal state |
-| 19 | Ingest file (blocking, `wait=True`) | Returns terminal status with `successful_files >= 0` |
-| 20 | Delete ingested document | `success=True`, `deleted_chunks > 0` |
-| 21 | Delete never-ingested filename | `success=False`, `deleted_chunks=0`, error message present |
-| 22 | Ingest via file object (`io.BytesIO`) | Accepted and processed without error |
-| 23 | Re-ingest same filename twice | Does not raise; second call returns a status |
-| 24 | Ingest `.md` file | Accepted and processed without error |
-| 25 | Poll task status manually | `get_task_status()` returns a status; `wait_for_task()` returns `completed` or `failed` |
+| 22 | Ingest file (async, `wait=False`) | Returns a non-empty `task_id`; polling reaches terminal state |
+| 23 | Ingest file (blocking, `wait=True`) | Returns terminal status with `total_files`, `processed_files`, `successful_files`, `failed_files` all populated |
+| 24 | Delete ingested document (deterministic ingest) | `success=True`, `deleted_chunks > 0` |
+| 25 | Delete never-ingested filename | `success=False`, `deleted_chunks=0`, error message present |
+| 26 | `get_task_status()` with a nonexistent task id | Raises `NotFoundError` |
+| 27 | Delete by a genuinely nonexistent `filter_id` | Raises `NotFoundError` (distinct from the wildcard-`data_sources` rejection case, which raises a generic `OpenRAGError`) |
+| 28 | `wait_for_task()` with a very small `timeout` | Raises `TimeoutError` before the task completes |
+| 29 | Ingest via file object (`io.BytesIO`) | Accepted and processed without error |
+| 30 | Re-ingest same filename twice | Does not raise; second call returns a status |
+| 31 | Ingest `.md` file | Accepted and processed without error |
+| 32 | Poll task status manually | `get_task_status()` returns a status; `wait_for_task()` returns `completed` or `failed` |
+| 33 | Delete documents by `filter_id` | Removes only the filenames in the filter's `data_sources` |
+| 34 | Delete by `filter_id` with wildcard `data_sources` | Rejected with `OpenRAGError` |
+| 35 | Delete with both `filename` and `filter_id` | Rejected with `ValueError` |
+| 36 | Delete with neither `filename` nor `filter_id` | Rejected with `ValueError` |
---
@@ -59,12 +70,14 @@ make test-sdk
| # | Test | Expected |
|---|------|----------|
-| 26 | Basic search query | Returns a results list |
-| 27 | Search with `limit=1` | Returns at most 1 result |
-| 28 | Search with `score_threshold=0.99` | Returns a list (may be empty) without error |
-| 29 | Nonsense/obscure query | Returns empty list, no error |
-| 30 | Unicode and emoji in query | Returns list, no error |
-| 31 | Result fields | Each result has `text` (non-empty string) |
+| 37 | Basic search query | Returns a results list; each result's `score` is in `[0, 1]` |
+| 38 | Search with `limit=1` | Returns at most 1 result |
+| 39 | Search with `score_threshold=0.99` | Returns a list (may be empty) without error |
+| 40 | Search with `score_threshold=0.5` | All returned results have `score >= 0.5` |
+| 41 | Nonsense/obscure query | Returns empty list, no error |
+| 42 | Unicode and emoji in query | Returns list, no error |
+| 43 | Result fields (`limit=5`) | At most 5 results; `text` is a non-empty string; `page`/`mimetype` are `None` or correctly typed |
+| 44 | Whitespace-only query (`" "`) | Raises `ValidationError` |
---
@@ -72,8 +85,8 @@ make test-sdk
| # | Test | Expected |
|---|------|----------|
-| 32 | Get settings | Response includes `agent` and `knowledge` sections |
-| 33 | Update `chunk_size` setting | Update succeeds; value readable back unchanged |
+| 45 | Get settings | Response includes `agent` and `knowledge` sections, including `chunk_overlap`, `table_structure`, `ocr`, `picture_descriptions`, and `agent.system_prompt` (each `None` or correctly typed) |
+| 46 | Update `chunk_size` setting | Update succeeds; value readable back unchanged |
---
@@ -81,7 +94,9 @@ make test-sdk
| # | Test | Expected |
|---|------|----------|
-| 34 | List models for a provider (`openai`) | Returns `language_models` and `embedding_models` as lists |
+| 47 | List models for a provider (`openai`) | Returns `language_models` and `embedding_models` as lists |
+| 48 | List models, parametrized per provider (`openai`, `anthropic`, `ollama`, `watsonx`) | `openai` is required to return typed `ModelOption` entries (`value: str`) with at most one `default`; other providers are marked `SKIPPED` (not failed) if unconfigured (`ValidationError`) |
+| 49 | List models for an invalid provider | Raises `ValidationError` |
---
@@ -89,14 +104,14 @@ make test-sdk
| # | Test | Expected |
|---|------|----------|
-| 35 | Create filter | `success=True`, `id` returned |
-| 36 | Search filters by name | Returns list containing the created filter |
-| 37 | Get filter by ID | Returns filter with correct `id` and `name` |
-| 38 | Update filter description | Update returns `True`; description readable back |
-| 39 | Delete filter | Returns `True` |
-| 40 | Get deleted filter | Returns `None` |
-| 41 | Pass `filter_id` to `chat.create()` | No error; response returned |
-| 42 | Pass `filter_id` to `search.query()` | No error; results returned |
+| 50 | Create filter | `success=True`, `id` returned, `error is None` |
+| 51 | Search filters by name | Returns list containing the created filter; a guaranteed-no-match query returns `[]`; `limit=1` returns at most 1 |
+| 52 | Get filter by ID | Returns filter with correct `id`, `name`, `query_data`, `owner`, `created_at`, `updated_at` |
+| 53 | Update filter description | Update returns `True`; description readable back |
+| 54 | Delete filter (wrapped in `try`/`finally` so it can't leak) | Returns `True` |
+| 55 | Get deleted filter | Returns `None` |
+| 56 | Pass `filter_id` to `chat.create()` | No error; response returned |
+| 57 | Pass `filter_id` to `search.query()` | No error; results returned |
---
@@ -104,12 +119,15 @@ make test-sdk
| # | Test | Expected |
|---|------|----------|
-| 43 | Connect to dead port | Raises a network exception within timeout |
-| 44 | Get conversation with random UUID | Raises `NotFoundError` |
-| 45 | Delete conversation with random UUID | Returns `False` |
-| 46 | Update settings with invalid value (`chunk_size=-999999`) | Raises `OpenRAGError` subclass |
-| 47 | Call `ingest()` with no arguments | Raises `ValueError` |
-| 48 | Call `ingest()` with `BytesIO` but no filename | Raises `ValueError` |
+| 58 | Connect to dead port | Raises a network exception within timeout |
+| 59 | Get conversation with random UUID | Raises `NotFoundError` with `status_code == 404` |
+| 60 | Delete conversation with random UUID | Returns `False` |
+| 61 | Update settings with invalid value (`chunk_size=-999999`) | Raises `OpenRAGError` subclass |
+| 62 | Call `ingest()` with no arguments | Raises `ValueError` |
+| 63 | Call `ingest()` with `BytesIO` but no filename | Raises `ValueError` |
+| 64 | Iterate a fully-consumed `ChatStream` a second time | Raises `RuntimeError` |
+| 65 | `client.close()` with a caller-supplied `http_client` | Does not close the external `httpx.AsyncClient` |
+| 66 | API call after an `async with OpenRAGClient(...)` block exits | Raises (the SDK-owned client was closed on `__aexit__`) |
---
@@ -117,10 +135,10 @@ make test-sdk
| # | Test | Expected |
|---|------|----------|
-| 49 | Full RAG pipeline: ingest → search → chat | Chat sources include the ingested document |
-| 50 | Multi-turn conversation with RAG | Second turn uses same `chat_id`; context carried over |
-| 51 | Knowledge filter scopes search and chat | Search and chat succeed with `filter_id`; filter cleaned up |
+| 67 | Full RAG pipeline: ingest → search → chat | Chat sources include the ingested document |
+| 68 | Multi-turn conversation with RAG | Second turn uses same `chat_id`; context carried over |
+| 69 | Knowledge filter scopes search and chat | Search and chat succeed with `filter_id`; filter cleaned up |
---
-**Total: 51 tests across 8 domains.**
+**Total: 69 tests across 9 domains.**
diff --git a/tests/integration/sdk/test_auth.py b/tests/integration/sdk/test_auth.py
index 851209f76..5ccd328b5 100644
--- a/tests/integration/sdk/test_auth.py
+++ b/tests/integration/sdk/test_auth.py
@@ -2,6 +2,7 @@
import os
+import httpx
import pytest
from .conftest import _base_url
@@ -30,25 +31,56 @@ class TestAuth:
if env_backup is not None:
os.environ["OPENRAG_API_KEY"] = env_backup
- def test_extra_headers_alone_satisfies_auth_check(self):
- """Client must not raise when extra_headers are provided without an api_key.
+ @pytest.mark.asyncio
+ async def test_extra_headers_alone_satisfies_auth_check(self):
+ """Client must not raise when extra_headers are provided without an api_key,
+ and must actually attach those headers to real outgoing requests.
This covers the IBM auth mode where X-Username / X-Api-Key are passed via
extra_headers instead of the OPENRAG_API_KEY env var or api_key argument.
+ A real request is made here to prove the headers are sent end-to-end
+ rather than merely accepted at construction time. Whether the server
+ authenticates the request successfully depends on IBM_AUTH_ENABLED
+ being set on the target instance (off by default for a standard
+ `make test-sdk` run), so both a successful response and a clean 401
+ (a server-side rejection, which still proves the request — and its
+ headers — reached the server) are accepted outcomes.
"""
from openrag_sdk import OpenRAGClient
+ from openrag_sdk.exceptions import AuthenticationError
env_backup = os.environ.pop("OPENRAG_API_KEY", None)
+ captured_headers: dict[str, str] = {}
+
+ async def capture_request(request: httpx.Request) -> None:
+ captured_headers.update(request.headers)
+
try:
- client = OpenRAGClient(
- extra_headers={"X-Username": "testuser", "X-Api-Key": "ibm-key"},
- base_url=_base_url,
- )
- assert client is not None
+ async with httpx.AsyncClient(
+ timeout=30.0, event_hooks={"request": [capture_request]}
+ ) as http_client:
+ ibm_client = OpenRAGClient(
+ extra_headers={"X-Username": "testuser", "X-Api-Key": "ibm-key"},
+ base_url=_base_url,
+ http_client=http_client,
+ )
+ assert ibm_client is not None
+
+ try:
+ settings = await ibm_client.settings.get()
+ assert settings.agent is not None
+ except AuthenticationError as e:
+ assert e.status_code == 401
finally:
if env_backup is not None:
os.environ["OPENRAG_API_KEY"] = env_backup
+ # httpx.Headers normalizes header names to lowercase internally, so
+ # captured_headers (a plain dict built from iterating request.headers)
+ # holds lowercase keys regardless of the casing extra_headers used.
+ assert captured_headers.get("x-username") == "testuser"
+ assert captured_headers.get("x-api-key") == "ibm-key"
+
def test_env_var_api_key_satisfies_auth_check(self):
"""Client must not raise when only the OPENRAG_API_KEY env var is set."""
from openrag_sdk import OpenRAGClient
diff --git a/tests/integration/sdk/test_chat.py b/tests/integration/sdk/test_chat.py
index 8955f8a4b..8543394c6 100644
--- a/tests/integration/sdk/test_chat.py
+++ b/tests/integration/sdk/test_chat.py
@@ -4,6 +4,7 @@ import os
from pathlib import Path
import pytest
+from openrag_sdk import Source
pytestmark = pytest.mark.skipif(
os.environ.get("SKIP_SDK_INTEGRATION_TESTS") == "true",
@@ -18,26 +19,63 @@ class TestChat:
async def test_chat_non_streaming(self, client):
"""Non-streaming chat returns a non-empty response string."""
response = await client.chat.create(message="Say hello in exactly 3 words.")
+ try:
+ assert response.response is not None
+ assert isinstance(response.response, str)
+ assert len(response.response) > 0
+ assert response.sources is not None
+ assert isinstance(response.sources, list)
+ for s in response.sources:
+ assert isinstance(s, Source)
+ # score is a raw OpenSearch relevance score (boosted BM25/KNN
+ # hybrid), not normalized to [0, 1] -- it can exceed 1.
+ assert s.score >= 0
+ finally:
+ if response.chat_id:
+ await client.chat.delete(response.chat_id)
- assert response.response is not None
- assert isinstance(response.response, str)
- assert len(response.response) > 0
- assert response.sources is not None
- assert isinstance(response.sources, list)
+ @pytest.mark.asyncio
+ async def test_chat_with_filters_limit_and_score_threshold(self, client, test_file: Path):
+ """filters, limit, and score_threshold together must be accepted and honoured."""
+ await client.documents.ingest(file_path=str(test_file))
+ response = None
+ try:
+ response = await client.chat.create(
+ message="What does the document say about elephants?",
+ filters={"data_sources": [test_file.name]},
+ limit=3,
+ score_threshold=0.0,
+ )
+ assert response.sources is not None
+ assert len(response.sources) <= 3
+ finally:
+ if response is not None and response.chat_id:
+ await client.chat.delete(response.chat_id)
+ await client.documents.delete(test_file.name)
@pytest.mark.asyncio
async def test_chat_streaming_create(self, client):
- """create(stream=True) yields content events with text deltas."""
+ """create(stream=True) yields content events with text deltas, ending in done."""
collected_text = ""
+ events = []
async for event in await client.chat.create(
message="Say 'test' and nothing else.",
stream=True,
):
+ events.append(event)
if event.type == "content":
collected_text += event.delta
- assert len(collected_text) > 0
+ try:
+ assert len(collected_text) > 0
+ assert len(events) > 0
+ assert events[-1].type == "done"
+ assert events[-1].chat_id is not None
+ finally:
+ chat_id = events[-1].chat_id if events else None
+ if chat_id:
+ await client.chat.delete(chat_id)
@pytest.mark.asyncio
async def test_chat_streaming_context_manager(self, client):
@@ -47,6 +85,9 @@ class TestChat:
pass
assert len(stream.text) > 0
+ if stream.chat_id:
+ await client.chat.delete(stream.chat_id)
+
@pytest.mark.asyncio
async def test_chat_text_stream(self, client):
"""text_stream yields plain text deltas."""
@@ -56,7 +97,11 @@ class TestChat:
async for text in stream.text_stream:
collected += text
- assert len(collected) > 0
+ try:
+ assert len(collected) > 0
+ finally:
+ if stream.chat_id:
+ await client.chat.delete(stream.chat_id)
@pytest.mark.asyncio
async def test_chat_final_text(self, client):
@@ -64,29 +109,50 @@ class TestChat:
async with client.chat.stream(message="Say 'done' and nothing else.") as stream:
text = await stream.final_text()
- assert len(text) > 0
+ try:
+ assert len(text) > 0
+ finally:
+ if stream.chat_id:
+ await client.chat.delete(stream.chat_id)
@pytest.mark.asyncio
async def test_chat_conversation_continuation(self, client):
"""A second message with chat_id continues the same conversation."""
response1 = await client.chat.create(message="Remember the number 42.")
assert response1.chat_id is not None
-
- response2 = await client.chat.create(
- message="What number did I ask you to remember?",
- chat_id=response1.chat_id,
- )
- assert response2.response is not None
+ chat_id = response1.chat_id
+
+ try:
+ response2 = await client.chat.create(
+ message="What number did I ask you to remember?",
+ chat_id=chat_id,
+ )
+ assert response2.response is not None
+ followup = response2
+ assert followup.chat_id == chat_id
+ finally:
+ await client.chat.delete(chat_id)
@pytest.mark.asyncio
async def test_list_conversations(self, client):
"""list() returns a ConversationListResponse with a list of conversations."""
- await client.chat.create(message="Test message for listing.")
+ created = await client.chat.create(message="Test message for listing.")
+ assert created.chat_id is not None
- result = await client.chat.list()
+ try:
+ result = await client.chat.list()
- assert result.conversations is not None
- assert isinstance(result.conversations, list)
+ assert result.conversations is not None
+ assert isinstance(result.conversations, list)
+ assert len(result.conversations) >= 1
+
+ conv = result.conversations[0]
+ assert isinstance(conv.title, str)
+ assert isinstance(conv.created_at, str) and len(conv.created_at) > 0
+ assert isinstance(conv.last_activity, str) and len(conv.last_activity) > 0
+ assert conv.message_count >= 0
+ finally:
+ await client.chat.delete(created.chat_id)
@pytest.mark.asyncio
async def test_get_conversation(self, client):
@@ -94,12 +160,19 @@ class TestChat:
response = await client.chat.create(message="Test message for get.")
assert response.chat_id is not None
- conversation = await client.chat.get(response.chat_id)
+ try:
+ conversation = await client.chat.get(response.chat_id)
+
+ assert conversation.chat_id == response.chat_id
+ assert conversation.messages is not None
+ assert isinstance(conversation.messages, list)
+ assert len(conversation.messages) >= 1
- assert conversation.chat_id == response.chat_id
- assert conversation.messages is not None
- assert isinstance(conversation.messages, list)
- assert len(conversation.messages) >= 1
+ msg = conversation.messages[0]
+ assert msg.role in ("user", "assistant")
+ assert isinstance(msg.content, str)
+ finally:
+ await client.chat.delete(response.chat_id)
@pytest.mark.asyncio
async def test_delete_conversation(self, client):
@@ -138,25 +211,29 @@ class TestChatExtended:
r1 = await client.chat.create(message="Remember the colour blue.")
assert r1.chat_id is not None
- collected = ""
- async with client.chat.stream(
- message="What colour did I ask you to remember?",
- chat_id=r1.chat_id,
- ) as stream:
- async for text in stream.text_stream:
- collected += text
+ try:
+ collected = ""
+ async with client.chat.stream(
+ message="What colour did I ask you to remember?",
+ chat_id=r1.chat_id,
+ ) as stream:
+ async for text in stream.text_stream:
+ collected += text
- assert len(collected) > 0
- await client.chat.delete(r1.chat_id)
+ assert len(collected) > 0
+ finally:
+ await client.chat.delete(r1.chat_id)
@pytest.mark.asyncio
async def test_chat_response_has_chat_id(self, client):
"""Every non-streaming response must include a chat_id for continuation."""
response = await client.chat.create(message="Hello.")
- assert response.chat_id is not None
- assert isinstance(response.chat_id, str)
- assert len(response.chat_id) > 0
- await client.chat.delete(response.chat_id)
+ try:
+ assert response.chat_id is not None
+ assert isinstance(response.chat_id, str)
+ assert len(response.chat_id) > 0
+ finally:
+ await client.chat.delete(response.chat_id)
@pytest.mark.asyncio
async def test_stream_chat_id_available_after_iteration(self, client):
@@ -165,6 +242,9 @@ class TestChatExtended:
await stream.final_text()
assert stream.chat_id is not None
+ if stream.chat_id:
+ await client.chat.delete(stream.chat_id)
+
@pytest.mark.asyncio
async def test_list_conversations_returns_list(self, client):
"""list() always returns a ConversationListResponse with a list."""
diff --git a/tests/integration/sdk/test_documents.py b/tests/integration/sdk/test_documents.py
index f46ad25fd..a0453d82b 100644
--- a/tests/integration/sdk/test_documents.py
+++ b/tests/integration/sdk/test_documents.py
@@ -6,7 +6,7 @@ import uuid
from pathlib import Path
import pytest
-from openrag_sdk.exceptions import OpenRAGError
+from openrag_sdk.exceptions import NotFoundError, OpenRAGError
pytestmark = pytest.mark.skipif(
os.environ.get("SKIP_SDK_INTEGRATION_TESTS") == "true",
@@ -22,31 +22,41 @@ class TestDocuments:
"""wait=False returns a task_id immediately; polling reaches a terminal state."""
result = await client.documents.ingest(file_path=str(test_file), wait=False)
assert result.task_id is not None
+ assert isinstance(result.task_id, str) and result.task_id
- final_status = await client.documents.wait_for_task(result.task_id)
- assert final_status.status is not None
- assert final_status.successful_files >= 0
+ try:
+ final_status = await client.documents.wait_for_task(result.task_id)
+ assert final_status.status is not None
+ assert final_status.successful_files >= 0
+ finally:
+ await client.documents.delete(test_file.name)
@pytest.mark.asyncio
async def test_ingest_document(self, client, test_file: Path):
"""wait=True polls until completion and returns a terminal status."""
- result = await client.documents.ingest(file_path=str(test_file))
- assert result.status is not None
- assert result.successful_files >= 0
+ try:
+ result = await client.documents.ingest(file_path=str(test_file))
+ assert result.status is not None
+ assert result.successful_files >= 0
+ assert result.total_files >= 1
+ assert result.processed_files >= 0
+ assert result.failed_files >= 0
+ finally:
+ await client.documents.delete(test_file.name)
@pytest.mark.asyncio
async def test_delete_document(self, client, test_file: Path):
"""Deleting an ingested document succeeds when chunks were indexed."""
ingest_result = await client.documents.ingest(file_path=str(test_file))
+ assert ingest_result.successful_files > 0, (
+ "Ingestion did not succeed for any file "
+ f"(status={ingest_result.status}, failed_files={ingest_result.failed_files}); "
+ "cannot deterministically test deletion of an indexed document"
+ )
result = await client.documents.delete(test_file.name)
-
- if ingest_result.successful_files > 0:
- assert result.success is True
- assert result.deleted_chunks > 0
- else:
- assert result.success is False
- assert result.deleted_chunks == 0
+ assert result.success is True
+ assert result.deleted_chunks > 0
@pytest.mark.asyncio
async def test_delete_missing_document_is_idempotent(self, client):
@@ -59,6 +69,38 @@ class TestDocuments:
assert result.filename == missing_filename
assert result.error is not None
+ @pytest.mark.asyncio
+ async def test_get_task_status_nonexistent_raises_not_found(self, client):
+ """Fetching status for a task id that doesn't exist must raise NotFoundError."""
+ with pytest.raises(NotFoundError):
+ await client.documents.get_task_status(f"nonexistent-{uuid.uuid4().hex}")
+
+ @pytest.mark.asyncio
+ async def test_delete_by_nonexistent_filter_id_raises_not_found(self, client):
+ """Deleting by a genuinely nonexistent filter_id must raise NotFoundError.
+
+ Distinct from the wildcard-data_sources rejection test in
+ TestDeleteByFilterId, which raises a generic OpenRAGError (validation),
+ not NotFoundError.
+ """
+ with pytest.raises(NotFoundError):
+ await client.documents.delete(filter_id=f"nonexistent-uuid-{uuid.uuid4().hex}")
+
+ @pytest.mark.asyncio
+ async def test_wait_for_task_timeout_raises_timeout_error(self, client):
+ """wait_for_task() with timeout=0 must raise TimeoutError without polling.
+
+ A timeout racing real task duration (e.g. a tiny nonzero value) is
+ inherently flaky: if the task happens to reach a terminal status on
+ the very first poll -- which can happen near-instantly, including on
+ failure -- wait_for_task returns normally instead of raising,
+ regardless of how small the timeout was. timeout=0 makes the
+ elapsed<timeout loop guard false before the first poll, so this is
+ deterministic and doesn't require a real ingest at all.
+ """
+ with pytest.raises(TimeoutError):
+ await client.documents.wait_for_task(f"nonexistent-{uuid.uuid4().hex}", timeout=0)
+
class TestDocumentsExtended:
"""Additional document ingestion scenarios."""
@@ -74,11 +116,12 @@ class TestDocumentsExtended:
).encode()
filename = f"file_obj_{unique_token[:8]}.md"
- result = await client.documents.ingest(file=io.BytesIO(content), filename=filename)
- assert result.status is not None
- assert result.successful_files >= 0
-
- await client.documents.delete(filename)
+ try:
+ result = await client.documents.ingest(file=io.BytesIO(content), filename=filename)
+ assert result.status is not None
+ assert result.successful_files >= 0
+ finally:
+ await client.documents.delete(filename)
@pytest.mark.asyncio
async def test_reingest_same_filename_does_not_raise(self, client, tmp_path):
@@ -87,22 +130,25 @@ class TestDocumentsExtended:
file_path = tmp_path / f"reingest_{unique_token[:8]}.md"
file_path.write_text(f"# Reingest Test\n\nToken: {unique_token}\n")
- result1 = await client.documents.ingest(file_path=str(file_path))
- assert result1.status is not None
-
- result2 = await client.documents.ingest(file_path=str(file_path))
- assert result2.status is not None
+ try:
+ result1 = await client.documents.ingest(file_path=str(file_path))
+ assert result1.status is not None
- await client.documents.delete(file_path.name)
+ result2 = await client.documents.ingest(file_path=str(file_path))
+ assert result2.status is not None
+ finally:
+ await client.documents.delete(file_path.name)
@pytest.mark.asyncio
async def test_ingest_markdown_format(self, client, tmp_path):
"""Verify .md files are accepted and processed without error."""
file_path = tmp_path / f"format_md_{uuid.uuid4().hex[:8]}.md"
file_path.write_text("# Markdown Format\n\n## Section\n\nContent here.\n")
- result = await client.documents.ingest(file_path=str(file_path))
- assert result.status is not None
- await client.documents.delete(file_path.name)
+ try:
+ result = await client.documents.ingest(file_path=str(file_path))
+ assert result.status is not None
+ finally:
+ await client.documents.delete(file_path.name)
@pytest.mark.asyncio
async def test_task_status_polling(self, client, tmp_path):
@@ -110,16 +156,17 @@ class TestDocumentsExtended:
file_path = tmp_path / f"poll_{uuid.uuid4().hex[:8]}.md"
file_path.write_text("# Polling Test\n\nContent for polling test.\n")
- task_response = await client.documents.ingest(file_path=str(file_path), wait=False)
- assert task_response.task_id is not None
-
- status = await client.documents.get_task_status(task_response.task_id)
- assert status.status is not None
+ try:
+ task_response = await client.documents.ingest(file_path=str(file_path), wait=False)
+ assert task_response.task_id is not None
- final = await client.documents.wait_for_task(task_response.task_id)
- assert final.status in ("completed", "failed")
+ status = await client.documents.get_task_status(task_response.task_id)
+ assert status.status is not None
- await client.documents.delete(file_path.name)
+ final = await client.documents.wait_for_task(task_response.task_id)
+ assert final.status in ("completed", "failed")
+ finally:
+ await client.documents.delete(file_path.name)
class TestDeleteByFilterId:
diff --git a/tests/integration/sdk/test_errors.py b/tests/integration/sdk/test_errors.py
index 511cc7f95..fb228bf3f 100644
--- a/tests/integration/sdk/test_errors.py
+++ b/tests/integration/sdk/test_errors.py
@@ -4,6 +4,7 @@ import io
import os
import uuid
+import httpx
import pytest
from .conftest import _base_url
@@ -28,7 +29,7 @@ class TestErrorHandling:
timeout=3.0,
)
try:
- with pytest.raises(Exception):
+ with pytest.raises(Exception): # noqa: B017 - any network exception is acceptable here
await dead_client.settings.get()
finally:
await dead_client.close()
@@ -38,8 +39,9 @@ class TestErrorHandling:
"""Fetching a conversation with a random UUID must raise NotFoundError."""
from openrag_sdk.exceptions import NotFoundError
- with pytest.raises(NotFoundError):
+ with pytest.raises(NotFoundError) as exc_info:
await client.chat.get(str(uuid.uuid4()))
+ assert exc_info.value.status_code == 404
@pytest.mark.asyncio
async def test_delete_nonexistent_conversation_returns_false(self, client):
@@ -66,3 +68,46 @@ class TestErrorHandling:
"""Providing a file object without a filename must raise ValueError."""
with pytest.raises(ValueError):
await client.documents.ingest(file=io.BytesIO(b"content"))
+
+ @pytest.mark.asyncio
+ async def test_reiterating_consumed_stream_raises_runtime_error(self, client):
+ """Iterating a ChatStream a second time after full consumption must raise RuntimeError."""
+ chat_id = None
+ async with client.chat.stream(message="Say one word.") as stream:
+ async for _ in stream:
+ pass
+ chat_id = stream.chat_id
+
+ try:
+ with pytest.raises(RuntimeError):
+ async for _ in stream:
+ pass
+ finally:
+ if chat_id:
+ await client.chat.delete(chat_id)
+
+ @pytest.mark.asyncio
+ async def test_close_does_not_close_external_http_client(self):
+ """The SDK must not close a caller-supplied httpx.AsyncClient on client.close()."""
+ from openrag_sdk import OpenRAGClient
+
+ ext_client = httpx.AsyncClient()
+ try:
+ sdk_client = OpenRAGClient(
+ api_key="orag_test", base_url=_base_url, http_client=ext_client
+ )
+ await sdk_client.close()
+ assert not ext_client.is_closed
+ finally:
+ await ext_client.aclose()
+
+ @pytest.mark.asyncio
+ async def test_client_closed_after_context_manager_exit(self):
+ """An SDK-owned http client is closed on __aexit__; later calls must fail."""
+ from openrag_sdk import OpenRAGClient
+
+ async with OpenRAGClient(api_key="orag_test", base_url=_base_url) as c:
+ pass
+
+ with pytest.raises(RuntimeError):
+ await c.settings.get()
diff --git a/tests/integration/sdk/test_filters.py b/tests/integration/sdk/test_filters.py
index b11fc7426..8939e66bb 100644
--- a/tests/integration/sdk/test_filters.py
+++ b/tests/integration/sdk/test_filters.py
@@ -84,32 +84,48 @@ class TestKnowledgeFilters:
)
assert create_result.success is True
assert create_result.id is not None
+ assert create_result.error is None
filter_id = create_result.id
- # Search
- filters = await client.knowledge_filters.search("Python SDK Test")
- assert isinstance(filters, list)
- assert any(f.name == "Python SDK Test Filter" for f in filters)
-
- # Get
- filter_obj = await client.knowledge_filters.get(filter_id)
- assert filter_obj is not None
- assert filter_obj.id == filter_id
- assert filter_obj.name == "Python SDK Test Filter"
-
- # Update
- update_success = await client.knowledge_filters.update(
- filter_id,
- {"description": "Updated description from Python SDK test"},
- )
- assert update_success is True
-
- updated_filter = await client.knowledge_filters.get(filter_id)
- assert updated_filter.description == "Updated description from Python SDK test"
+ try:
+ # Search
+ filters = await client.knowledge_filters.search("Python SDK Test")
+ assert isinstance(filters, list)
+ assert any(f.name == "Python SDK Test Filter" for f in filters)
+
+ # Search: a query guaranteed not to match anything returns [].
+ no_match = await client.knowledge_filters.search(
+ f"zzz_no_such_filter_{uuid.uuid4().hex}"
+ )
+ assert no_match == []
+
+ # Search: limit constrains the number of returned filters.
+ limited = await client.knowledge_filters.search("Python SDK Test", limit=1)
+ assert len(limited) <= 1
+
+ # Get
+ filter_obj = await client.knowledge_filters.get(filter_id)
+ assert filter_obj is not None
+ assert filter_obj.id == filter_id
+ assert filter_obj.name == "Python SDK Test Filter"
+ assert filter_obj.query_data is not None
+ assert filter_obj.owner is not None
+ assert filter_obj.created_at is not None
+ assert filter_obj.updated_at is not None
+
+ # Update
+ update_success = await client.knowledge_filters.update(
+ filter_id,
+ {"description": "Updated description from Python SDK test"},
+ )
+ assert update_success is True
- # Delete
- delete_success = await client.knowledge_filters.delete(filter_id)
- assert delete_success is True
+ updated_filter = await client.knowledge_filters.get(filter_id)
+ assert updated_filter.description == "Updated description from Python SDK test"
+ finally:
+ # Delete
+ delete_success = await client.knowledge_filters.delete(filter_id)
+ assert delete_success is True
deleted_filter = await client.knowledge_filters.get(filter_id)
assert deleted_filter is None
diff --git a/tests/integration/sdk/test_models.py b/tests/integration/sdk/test_models.py
index c86366116..349d53fb2 100644
--- a/tests/integration/sdk/test_models.py
+++ b/tests/integration/sdk/test_models.py
@@ -3,22 +3,72 @@
import os
import pytest
+from openrag_sdk.exceptions import ValidationError
pytestmark = pytest.mark.skipif(
os.environ.get("SKIP_SDK_INTEGRATION_TESTS") == "true",
reason="SDK integration tests skipped",
)
+ALL_PROVIDERS = ("openai", "anthropic", "ollama", "watsonx")
+
class TestModels:
"""Test model listing per provider."""
@pytest.mark.asyncio
async def test_list_models(self, client):
- """Listing models for a provider must return language and embedding model lists."""
+ """Listing models for a provider must return language and embedding model lists.
+
+ Only `openai` is guaranteed to be configured by the test onboarding
+ fixture (conftest.ensure_onboarding). The other providers are covered
+ defensively in test_list_models_all_providers below, since a typical
+ local dev instance may not have anthropic/ollama/watsonx configured —
+ the server responds with a 400 (ValidationError) in that case, not an
+ empty list.
+ """
models = await client.models.list("openai")
assert models.language_models is not None
assert isinstance(models.language_models, list)
assert models.embedding_models is not None
assert isinstance(models.embedding_models, list)
+
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize("provider", ALL_PROVIDERS)
+ async def test_list_models_per_provider(self, client, provider):
+ """List models for each supported provider.
+
+ Only `openai` is required to be configured (it's the provider set up
+ by the test onboarding fixture, conftest.ensure_onboarding) — its
+ assertions must pass. Any other provider that isn't configured on
+ this instance raises ValidationError (HTTP 400); those are marked as
+ explicitly skipped (visible per-provider in the pytest report) rather
+ than silently passed over or required to succeed.
+ """
+ try:
+ models = await client.models.list(provider)
+ except ValidationError:
+ if provider == "openai":
+ raise
+ pytest.skip(f"provider '{provider}' is not configured on this instance")
+
+ assert isinstance(models.language_models, list)
+ assert isinstance(models.embedding_models, list)
+
+ if models.language_models:
+ first = models.language_models[0]
+ assert isinstance(first.value, str)
+ defaults = [m for m in models.language_models if m.default is True]
+ assert len(defaults) <= 1, (
+ f"Provider {provider} returned more than one default language model"
+ )
+ assert len(defaults) >= 1, (
+ f"Provider {provider} has language models but none marked default"
+ )
+
+ @pytest.mark.asyncio
+ async def test_list_models_invalid_provider_raises_validation_error(self, client):
+ """Listing models for an unrecognized provider must raise ValidationError."""
+ with pytest.raises(ValidationError):
+ await client.models.list("invalid_provider")
diff --git a/tests/integration/sdk/test_search.py b/tests/integration/sdk/test_search.py
index c917f335f..caec7e554 100644
--- a/tests/integration/sdk/test_search.py
+++ b/tests/integration/sdk/test_search.py
@@ -4,6 +4,7 @@ import os
from pathlib import Path
import pytest
+from openrag_sdk.exceptions import ValidationError
pytestmark = pytest.mark.skipif(
os.environ.get("SKIP_SDK_INTEGRATION_TESTS") == "true",
@@ -19,8 +20,15 @@ class TestSearch:
"""A basic search query returns a results list."""
await client.documents.ingest(file_path=str(test_file))
- results = await client.search.query("purple elephants dancing")
- assert results.results is not None
+ try:
+ results = await client.search.query("purple elephants dancing")
+ assert results.results is not None
+ for r in results.results:
+ # score is a raw OpenSearch relevance score (boosted BM25/KNN
+ # hybrid), not normalized to [0, 1] -- it can exceed 1.
+ assert r.score >= 0
+ finally:
+ await client.documents.delete(test_file.name)
class TestSearchExtended:
@@ -31,25 +39,41 @@ class TestSearchExtended:
"""limit parameter caps the number of results returned."""
await client.documents.ingest(file_path=str(test_file))
- results = await client.search.query("test", limit=1)
- assert results.results is not None
- assert len(results.results) <= 1
+ try:
+ results = await client.search.query("test", limit=1)
+ assert results.results is not None
+ assert len(results.results) <= 1
+ finally:
+ await client.documents.delete(test_file.name)
@pytest.mark.asyncio
async def test_search_with_high_score_threshold_returns_empty(self, client, test_file: Path):
"""A score_threshold of 0.99 should filter out most or all results."""
await client.documents.ingest(file_path=str(test_file))
- results = await client.search.query("test", score_threshold=0.99)
- assert results.results is not None
- assert isinstance(results.results, list)
+ try:
+ results = await client.search.query("test", score_threshold=0.99)
+ assert results.results is not None
+ assert isinstance(results.results, list)
+ finally:
+ await client.documents.delete(test_file.name)
+
+ @pytest.mark.asyncio
+ async def test_search_with_score_threshold_filters_low_scores(self, client, test_file: Path):
+ """A score_threshold of 0.5 must only return results scoring at or above it."""
+ await client.documents.ingest(file_path=str(test_file))
+
+ try:
+ results = await client.search.query("test", score_threshold=0.5)
+ assert results.results is not None
+ assert all(r.score >= 0.5 for r in results.results)
+ finally:
+ await client.documents.delete(test_file.name)
@pytest.mark.asyncio
async def test_search_no_results_for_obscure_query(self, client):
"""A nonsense query must return an empty list, not raise an error."""
- results = await client.search.query(
- "zzz_xyzzy_nonexistent_content_abc123_qwerty_999"
- )
+ results = await client.search.query("zzz_xyzzy_nonexistent_content_abc123_qwerty_999")
assert results.results is not None
assert isinstance(results.results, list)
@@ -62,10 +86,22 @@ class TestSearchExtended:
@pytest.mark.asyncio
async def test_search_returns_result_fields(self, client, test_file: Path):
- """Each search result must have text populated as a string."""
+ """Each search result must have text populated as a string, and limit is respected."""
await client.documents.ingest(file_path=str(test_file))
- results = await client.search.query("purple elephants dancing", limit=5)
- for result in results.results:
- assert result.text is not None
- assert isinstance(result.text, str)
+ try:
+ results = await client.search.query("purple elephants dancing", limit=5)
+ assert len(results.results) <= 5
+ for result in results.results:
+ assert result.text is not None
+ assert isinstance(result.text, str)
+ assert result.page is None or isinstance(result.page, int)
+ assert result.mimetype is None or isinstance(result.mimetype, str)
+ finally:
+ await client.documents.delete(test_file.name)
+
+ @pytest.mark.asyncio
+ async def test_search_whitespace_query_raises_validation_error(self, client):
+ """A whitespace-only query must raise ValidationError, not be treated as valid."""
+ with pytest.raises(ValidationError):
+ await client.search.query(" ")
diff --git a/tests/integration/sdk/test_settings.py b/tests/integration/sdk/test_settings.py
index 0f28b12d0..be3735885 100644
--- a/tests/integration/sdk/test_settings.py
+++ b/tests/integration/sdk/test_settings.py
@@ -21,6 +21,15 @@ class TestSettings:
assert settings.agent is not None
assert settings.knowledge is not None
+ knowledge = settings.knowledge
+ assert knowledge.chunk_overlap is None or isinstance(knowledge.chunk_overlap, int)
+ assert knowledge.table_structure is None or isinstance(knowledge.table_structure, bool)
+ assert knowledge.ocr is None or isinstance(knowledge.ocr, bool)
+ assert knowledge.picture_descriptions is None or isinstance(
+ knowledge.picture_descriptions, bool
+ )
+ assert settings.agent.system_prompt is None or isinstance(settings.agent.system_prompt, str)
+
@pytest.mark.asyncio
async def test_update_settings(self, client):
"""Updating a setting must persist and be readable back."""