Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python
4.7 KB
Raw Blame
import * as fs from "fs";
import path from "path";
import { TEST_CONFIG } from "../config/test.config";
import { expect, test } from "../utils/fixtures";
import logger from "../utils/logger";
import { navigateToApp, navigateToSettings } from "../utils/navigation";

interface ChunkSizeMetrics {
  chunkSize: number;
  indexingTime: number;
  responseTime: number;
  responseLength: number;
  responseText: string;
}

test.describe("Chunk Size Impact Analysis @33219206 , @34581149 , @3481151", () => {
  const metricsResults: ChunkSizeMetrics[] = [];
  const testQuestion = TEST_CONFIG.questions.docling.pipeline;
  const chunkSizeScenarios = [
    { size: 200, label: "Small Chunks (200)" },
    { size: 500, label: "Medium Chunks (500)" },
    { size: 1000, label: "Large Chunks (1000)" },
  ];

  test.beforeEach(async ({ page }) => {
    // Navigate to app (handles login and onboarding)
    await navigateToApp(page);
    // Verify test document exists
    if (!fs.existsSync(TEST_CONFIG.documents.docling.path)) {
      throw new Error(
        `Test document not found at: ${TEST_CONFIG.documents.docling.path}`,
      );
    }
  });

  for (const scenario of chunkSizeScenarios) {
    test(`Chunk Size = ${scenario.size} (${scenario.label})`, async ({
      page,
      settings,
      knowledge,
      chat,
    }) => {
      test.setTimeout(300000); // 5 minutes timeout

      // Configure chunk settings
      await settings.clickTab("Ingestion");
      await settings.updateChunkSettings(
        scenario.size.toString(),
        TEST_CONFIG.chunkSettings.defaultOverlap,
      );

      // Upload document and measure indexing time
      const indexingStartTime = Date.now();
      await knowledge.ingestFile(TEST_CONFIG.documents.docling.path);
      const indexingTime = Date.now() - indexingStartTime;

      // Navigate to chat and ask question
      await chat.open();
      await chat.askQuestion(testQuestion);

      const queryStartTime = Date.now();
      const responseText = await chat.getLastResponse(
        TEST_CONFIG.timeouts.default,
      );
      const responseTime = Date.now() - queryStartTime;

      // Store metrics
      const metrics: ChunkSizeMetrics = {
        chunkSize: scenario.size,
        indexingTime,
        responseTime,
        responseLength: responseText.length,
        responseText,
      };

      metricsResults.push(metrics);

      // Assertions
      expect(indexingTime).toBeGreaterThan(0);
      expect(responseTime).toBeGreaterThan(0);
      expect(responseText.length).toBeGreaterThan(0);
    });
  }
  test.afterAll(async () => {
    if (metricsResults.length > 1) {
      const summary = metricsResults
        .map(
          (m) =>
            `Size ${m.chunkSize}: ${(m.responseTime / 1000).toFixed(1)}s response`,
        )
        .join(", ");
      logger.info(`Chunk Size Analysis Complete - ${summary}`);
    }
  });
});

test.describe("Large Chunk Size - Wrong Section Retrieval Test", () => {
  const testDocument = path.join(
    __dirname,
    "../test-data/Customer_analysis_small.csv",
  );
  const documentName = "Customer_analysis_small.csv";
  const testQuestion =
    "Which customer from Arizona has the highest Customer Lifetime Value?";
  const expectedCustomerId = "CP85232";
  const expectedValue = "44795";

  test.beforeEach(async ({ page }) => {
    // Navigate to app (handles login and onboarding)
    await navigateToApp(page);
  });

  test.beforeAll(async () => {
    // Verify test document exists (no page fixture needed)
    if (!fs.existsSync(testDocument)) {
      throw new Error(`Test document not found at: ${testDocument}`);
    }
  });

  test("Set chunk size to 1000 and verify correct answer retrieval", async ({
    page,
    settings,
    knowledge,
    chat,
  }) => {
    test.setTimeout(300000);

    await settings.clickTab("Ingestion");
    await settings.updateChunkSettings(
      "1000",
      TEST_CONFIG.chunkSettings.defaultOverlap,
    );
    await knowledge.deleteDocument(documentName);
    await knowledge.ingestFile(testDocument, true);
    await knowledge.verifyDocumentActive(documentName);

    await chat.openNewChat();
    const response = await chat.askQuestion(
      testQuestion,
      TEST_CONFIG.timeouts.default,
    );

    const hasCorrectCustomer = response.includes(expectedCustomerId);
    const hasCorrectValue =
      response.includes(expectedValue) ||
      response.includes("44,795") ||
      response.includes("$44,795");

    logger.info("\n" + "=".repeat(80));
    logger.info(
      `CHUNK SIZE 1000: ${hasCorrectCustomer && hasCorrectValue ? "✅ Correct" : "❌ Wrong"} - Expected: ${expectedCustomerId} ($${expectedValue})`,
    );
    logger.info(`Response: ${response}`);
    logger.info("=".repeat(80) + "\n");

    expect(response.length).toBeGreaterThan(0);
  });
});