Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python
5.7 KB
Raw Blame
import * as fs from "fs";
import path from "path";
import { TEST_CONFIG } from "../config/test.config";
import { expect, test } from "../utils/fixtures";
import logger from "../utils/logger";
import { navigateToApp } from "../utils/navigation";

interface ChunkOverlapMetrics {
  overlap: number;
  indexingTime: number;
  responseTime: number;
  responseLength: number;
  responseText: string;
}

test.describe("Chunk Overlap Impact Analysis @33219205 , @34581144 , @34581148", () => {
  const metricsResults: ChunkOverlapMetrics[] = [];
  const testQuestion = TEST_CONFIG.questions.docling.models;
  const overlapScenarios = [
    { overlap: 50, label: "Low Overlap (10%)" },
    { overlap: 100, label: "Medium Overlap (20%)" },
    { overlap: 150, label: "High Overlap (30%)" },
    { overlap: 200, label: "Very High Overlap (40%)" },
  ];

  test.beforeEach(async ({ page }) => {
    // Navigate to app (handles login and onboarding)
    await navigateToApp(page);

    // Verify test document exists
    if (!fs.existsSync(TEST_CONFIG.documents.docling.path)) {
      throw new Error(
        `Test document not found at: ${TEST_CONFIG.documents.docling.path}`,
      );
    }
  });

  for (const scenario of overlapScenarios) {
    test(`Chunk Overlap = ${scenario.overlap} (${scenario.label})`, async ({
      page,
      settings,
      knowledge,
      chat,
    }) => {
      test.setTimeout(300000); // 5 minutes timeout

      // Configure chunk settings (fixed size, variable overlap)
      await settings.clickTab("Ingestion");
      await settings.updateChunkSettings(
        TEST_CONFIG.chunkSettings.defaultSize,
        scenario.overlap.toString(),
      );

      // Upload document and measure indexing time
      const indexingStartTime = Date.now();
      await knowledge.ingestFile(TEST_CONFIG.documents.docling.path);
      const indexingTime = Date.now() - indexingStartTime;

      // Navigate to chat and ask question
      await chat.open();

      const queryStartTime = Date.now();
      const responseText = await chat.askQuestion(
        testQuestion,
        TEST_CONFIG.timeouts.default,
      );
      const responseTime = Date.now() - queryStartTime;

      // Store metrics
      const metrics: ChunkOverlapMetrics = {
        overlap: scenario.overlap,
        indexingTime,
        responseTime,
        responseLength: responseText.length,
        responseText,
      };

      metricsResults.push(metrics);

      // Assertions
      expect(indexingTime).toBeGreaterThan(0);
      expect(responseTime).toBeGreaterThan(0);
      expect(responseText.length).toBeGreaterThan(0);
    });
  }

  test.afterAll(async () => {
    if (metricsResults.length > 1) {
      const baseline = metricsResults[0];
      const final = metricsResults[metricsResults.length - 1];
      const timeChange = (
        (final.responseTime - baseline.responseTime) /
        1000
      ).toFixed(2);
      logger.info(
        `Chunk Overlap Analysis Complete - Response time change: ${timeChange}s (overlap ${baseline.overlap} → ${final.overlap})`,
      );
    }
  });
});

test.describe("Chunk Overlap Edge Case - Very Low Overlap with CSV Data", () => {
  const testDocument = path.join(
    __dirname,
    "../test-data/Customer_analysis_small.csv",
  );
  const documentName = "Customer_analysis_small.csv";
  const testQuestion =
    "Which customer has a higher Income: BU79786 or QZ44356?";

  test.beforeAll(async () => {
    if (!fs.existsSync(testDocument)) {
      throw new Error(`Test document not found at: ${testDocument}`);
    }
  });

  test.beforeEach(async ({ page }) => {
    // Navigate to app (handles login and onboarding)
    await navigateToApp(page);
  });

  test("Compare overlap=1 vs overlap=50 for the same question", async ({
    page,
    settings,
    knowledge,
    chat,
  }) => {
    test.setTimeout(600000);

    const results: Array<{
      overlap: string;
      response: string;
      quality: string;
    }> = [];

    // Test with both overlap values
    for (const overlap of ["1", "50"]) {
      await settings.clickTab("Ingestion");
      await settings.updateChunkSettings(
        TEST_CONFIG.chunkSettings.defaultSize,
        overlap,
      );

      // Upload document (will overwrite if exists)
      await knowledge.ingestFile(testDocument, true);
      await knowledge.verifyDocumentActive(documentName);

      await chat.openNewChat();
      const response = await chat.askQuestion(
        testQuestion,
        TEST_CONFIG.timeouts.default,
      );

      // Analyze response
      const responseLower = response.toLowerCase();
      const mentionsBU79786 = responseLower.includes("bu79786");
      const mentionsQZ44356 = responseLower.includes("qz44356");
      const mentions56274 =
        response.includes("56274") || response.includes("56,274");
      const mentions2322 =
        response.includes("2322") || response.includes("2,322");

      let quality = "COMPLETE";
      if (!mentionsBU79786 || !mentionsQZ44356) {
        quality = "PARTIAL";
      } else if (!mentions56274 || !mentions2322) {
        quality = "INCOMPLETE";
      }

      results.push({ overlap, response, quality });
    }

    // Final comparison log
    if (results.length === 2) {
      const [result1, result50] = results;
      logger.info("\n" + "=".repeat(80));
      logger.info(
        `OVERLAP COMPARISON: ${result1.quality} (overlap=1) → ${result50.quality} (overlap=50)`,
      );
      logger.info("=".repeat(80));
      logger.info(`Overlap=1: ${result1.response}`);
      logger.info(`Overlap=50: ${result50.response}`);
      logger.info("=".repeat(80) + "\n");
    }

    // Assertions
    expect(results.length).toBeGreaterThan(0);
    results.forEach((result) => {
      expect(result.response.length).toBeGreaterThan(0);
    });
  });
});