OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
import path from "path";
import { ACTIVE_PROVIDER_CONFIG } from "../config/provider";
import { expect, test } from "../utils/fixtures";
import logger from "../utils/logger";
import { navigateToHome } from "../utils/navigation";
const TEST_DOCUMENT = "test_chunk_search.pdf";
const TEST_DOCUMENT_PATH = path.join(__dirname, "../test-data", TEST_DOCUMENT);
// Test tokens from the document
const TEST_TOKENS = [
"RAG-TEST-ALPHA-001",
"RAG-TEST-BETA-002",
"RAG-TEST-GAMMA-003",
"RAG-TEST-DELTA-004",
"RAG-TEST-EPSILON-005",
"RAG-TEST-ZETA-006",
"RAG-TEST-ETA-007",
"RAG-TEST-THETA-008",
"RAG-TEST-IOTA-009",
"RAG-TEST-KAPPA-010",
];
test(`Chunk Search & Ranking - ${ACTIVE_PROVIDER_CONFIG.provider} @33219236`, async ({
page,
settings,
knowledge,
cleanupDocuments,
}) => {
test.setTimeout(320000);
// Navigate to the application
await navigateToHome(page);
logger.info(
`\nπ§ͺ Testing Chunk Search & Ranking with ${ACTIVE_PROVIDER_CONFIG.provider}`,
);
// Step 1: Cleanup test document if it exists
logger.info(` π§Ή Cleaning up existing test document...`);
try {
await knowledge.deleteDocument(TEST_DOCUMENT);
logger.info(` β Test document cleaned up`);
} catch (_error) {
logger.info(` βΉοΈ No existing test document to clean up`);
}
// Step 2: Set embedding model for active provider
logger.info(
` βοΈ Setting embedding model for ${ACTIVE_PROVIDER_CONFIG.provider}...`,
);
await settings.clickTab("Ingestion");
await settings.selectModel(
"Embedding model",
ACTIVE_PROVIDER_CONFIG.embedding,
);
logger.info(
` β Embedding model set to: ${ACTIVE_PROVIDER_CONFIG.embedding}`,
);
// Step 3: Ingest the test document
logger.info(` π Ingesting test document...`);
const ingestedFileName = await knowledge.ingestFile(TEST_DOCUMENT_PATH);
logger.info(` β Document ingested: ${ingestedFileName}`);
// Register for cleanup
await cleanupDocuments([TEST_DOCUMENT]);
// Step 4: Wait for document to be indexed (Active status)
logger.info(` β³ Waiting for document to be indexed...`);
await knowledge.verifyDocumentActive(TEST_DOCUMENT);
logger.info(` β Document is indexed and active`);
// Step 5: Open the chunk viewer
logger.info(` π Opening chunk viewer...`);
await knowledge.openDocument(TEST_DOCUMENT);
logger.info(` β Chunk viewer opened`);
// Step 6: Test chunk search for multiple tokens
logger.info(
`\n π Testing chunk search for ${TEST_TOKENS.length} unique tokens...`,
);
let successCount = 0;
let failureCount = 0;
const failures: string[] = [];
for (const token of TEST_TOKENS) {
logger.info(`\n π Searching for token: ${token}`);
// Search for the token and get top 2 chunks
const top2Chunks = await knowledge.searchChunks(token);
// Log the results
logger.info(` π Top 2 chunks retrieved:`);
top2Chunks.forEach((chunk, index) => {
const preview = chunk.substring(0, 100).replace(/\n/g, " ");
const hasToken = chunk.includes(token);
const indicator = hasToken ? "β" : "β";
logger.info(` ${indicator} ${index + 1}. ${preview}...`);
});
// Validation 1: Check if token appears in any of the top 2 chunks
const tokenFoundInTop2 = top2Chunks.some((chunk) => chunk.includes(token));
if (tokenFoundInTop2) {
logger.info(` β
Token found in top 2 chunks`);
successCount++;
} else {
logger.info(` β Token NOT found in top 2 chunks`);
failureCount++;
failures.push(token);
}
}
// Step 7: Report results
logger.info(`\n π Test Results Summary:`);
logger.info(
` β
Successful searches: ${successCount}/${TEST_TOKENS.length}`,
);
logger.info(` β Failed searches: ${failureCount}/${TEST_TOKENS.length}`);
if (failures.length > 0) {
logger.info(` Failed tokens: ${failures.join(", ")}`);
}
// Step 8: Assertions
// We expect at least 80% success rate (8 out of 10 tokens)
// This accounts for potential ranking variations across embedding models
const successRate = successCount / TEST_TOKENS.length;
const minSuccessRate = 0.8;
logger.info(`\n π Success rate: ${(successRate * 100).toFixed(1)}%`);
logger.info(
` π― Required success rate: ${(minSuccessRate * 100).toFixed(1)}%`,
);
if (successRate >= minSuccessRate) {
logger.info(
` β
SUCCESS: Chunk search ranking is working correctly for Azure OpenAI\n`,
);
} else {
throw new Error(
`β FAILED: Chunk search ranking below acceptable threshold\n` +
` Success rate: ${(successRate * 100).toFixed(1)}% (required: ${(minSuccessRate * 100).toFixed(1)}%)\n` +
` Failed tokens: ${failures.join(", ")}\n` +
` This indicates the embedding model is not ranking relevant chunks correctly.`,
);
}
// Playwright assertions for reporting
expect(successRate).toBeGreaterThanOrEqual(minSuccessRate);
expect(successCount).toBeGreaterThanOrEqual(
Math.ceil(TEST_TOKENS.length * minSuccessRate),
);
});