Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python
22.8 KB
Raw Blame
import { expect, Locator, Page } from "@playwright/test";
import path from "path";
import logger from "../utils/logger";
import { getBaseUrl, navigateToChat } from "../utils/navigation";

export class Chat {
  // Locators - defined at class level for better maintainability
  private readonly chatLink = () =>
    this.page.getByRole("link", { name: "Chat" });
  private readonly plusButton = () =>
    this.page.locator('div[role="presentation"] button').filter({
      has: this.page.locator("svg.lucide-plus"),
    });
  private readonly questionInput = () =>
    this.page.getByRole("textbox", { name: "Ask a question..." });
  private readonly lastMarkdownResponse = () =>
    this.page.locator(".markdown.prose").last();
  private readonly lastMarkdownDiv = () =>
    this.page.locator("div.markdown").last();
  private readonly filterButton = () =>
    this.page.locator('button[data-filter-button="true"]');
  private readonly noFilterOption = () =>
    this.page.getByText("No knowledge filter");
  private readonly htmlElement = () => this.page.locator("html");
  private readonly mainContent = () =>
    this.page.locator('main, [role="main"], body').first();
  private readonly deleteConversationMenuItem = () =>
    this.page.getByRole("menuitem", { name: /delete conversation/i });
  private readonly deleteButton = () =>
    this.page.getByRole("button", { name: /^delete$/i });
  private readonly conversationDeletedToast = () =>
    this.page.getByText(/conversation deleted successfully/i);
  private readonly selectToggleButton = () =>
    this.page.getByTestId("chat-select-toggle");
  private readonly selectAllCheckBox = () =>
    this.page.getByTestId("chat-select-all");

  /**
   * Get locator for a filter option by name
   * @param filterName - Name of the filter
   * @returns Locator for the filter option
   */
  private getFilterOption(filterName: string): Locator {
    return this.page.getByText(filterName, { exact: true });
  }

  /**
   * Get locator for tool call by pattern
   * @returns Locator for the tool call element
   */
  private getToolCallLocator(): Locator {
    return this.page
      .getByText(/Function Call:.*opensearch_url_ingestion_flow/i)
      .last();
  }

  /**
   * Get locator for chat row by title
   * @param chatTitle - The title of the chat
   * @returns Locator for the chat row
   */
  private getChatRow(chatTitle: string): Locator {
    const cleanTitle =
      chatTitle.length > 35 ? chatTitle.substring(0, 35) : chatTitle;
    return this.page
      .locator("button")
      .filter({
        has: this.page.getByText(cleanTitle, { exact: false }),
      })
      .first();
  }

  constructor(private page: Page) {}

  async open() {
    await this.chatLink().click();
  }

  /**
   * Open a fresh chat by navigating directly to the chat URL
   * This automatically opens a new chat session
   */
  async openNewChat() {
    const baseUrl = getBaseUrl().replace(/\/$/, "");
    await this.page.goto(`${baseUrl}/chat`);
    // Wait for the question input to be ready and for any background init calls to settle
    await this.page
      .waitForLoadState("networkidle", { timeout: 15000 })
      .catch(() => {});
    await this.questionInput().waitFor({ state: "visible", timeout: 15000 });
  }

  /**
   * Ask a question in the chat and wait for the complete response
   * Uses API interception with UI fallback
   * @param question - The question to ask
   * @param timeout - Maximum time to wait for response (default: 60000ms)
   * @returns The complete response text from the assistant
   */
  async askQuestion(
    question: string,
    timeout: number = 120000,
  ): Promise<string> {
    const input = this.questionInput();
    let fullResponse = "";
    // Fill the input first, then set up the intercept just before submitting.
    // This prevents catching background /api/langflow calls (e.g. suggestions/init)
    // that fire when the chat page loads.
    await input.fill(question);
    // Set up response intercept — matches only POST requests whose body contains
    // the question text, so background/suggestion calls are ignored.
    const responsePromise = this.page.waitForResponse(
      async (response) => {
        if (
          !response.url().includes("/api/langflow") ||
          response.request().method() !== "POST"
        ) {
          return false;
        }
        try {
          const body = response.request().postData() || "";
          return body.includes(question.substring(0, 30));
        } catch {
          return true; // body not readable — accept it as a fallback
        }
      },
      { timeout },
    );
    await this.page.keyboard.press("Enter");
    const response = await responsePromise;
    try {
      const raw = await response.text();
      const lines = raw.split("\n").filter((line) => line.trim());
      for (const line of lines) {
        try {
          const chunk = JSON.parse(line);
          if (chunk.delta?.content) {
            fullResponse += chunk.delta.content;
          }
          if (chunk.response?.text) {
            fullResponse = chunk.response.text; // final override
          }
          if (chunk.error) {
            fullResponse =
              typeof chunk.error === "string"
                ? chunk.error
                : (chunk.error.message as string) ||
                  JSON.stringify(chunk.error);
          }
          if (chunk.detail) {
            fullResponse =
              typeof chunk.detail === "string"
                ? chunk.detail
                : (chunk.detail.message as string) ||
                  JSON.stringify(chunk.detail);
          }
          if (chunk.item?.results?.length > 0) {
            for (const result of chunk.item.results) {
              if (result.filename && !fullResponse.includes(result.filename)) {
                fullResponse += `Source: ${result.filename}`;
              }
            }
          }
        } catch {
          // ignore malformed chunks
        }
      }
    } catch {
      // fallback to UI
    }

    if (!fullResponse.trim()) {
      const errorLocator = this.page
        .locator(".border-destructive\\/50, [class*='text-destructive']")
        .last();
      if (await errorLocator.isVisible({ timeout: 2000 }).catch(() => false)) {
        fullResponse = (await errorLocator.textContent()) || "";
      }
    }

    return fullResponse.trim();
  }

  /**
   * Ingest a URL via chat and capture tool call data
   * @param url - The URL to ingest
   * @returns Object containing the tool call locator and captured tool data
   */
  async ingestUrl(url: string): Promise<{
    toolCall: Locator;
    toolData: any;
    fullResponse: string;
  }> {
    const input = this.questionInput();
    let capturedToolData: any = null;
    let fullResponseText = "";

    // Collect ALL /api/langflow POST responses until we find tool call data or the
    // streaming response is complete. A single waitForResponse resolves on the first
    // matching call (often a background/init call), so we use a route listener instead.
    const collectedPromises: Promise<string>[] = [];
    const responseHandler = (response: any) => {
      if (
        !response.url().includes("/api/langflow") ||
        response.request().method() !== "POST"
      )
        return;
      const promise = response
        .body()
        .then((buf: any) => buf.toString("utf-8"))
        .catch(() => response.text())
        .catch(() => "");
      collectedPromises.push(promise);
    };
    this.page.on("response", responseHandler);

    try {
      await input.fill(`Please ingest this URL: ${url}`);
      await this.page.keyboard.press("Enter");

      // Wait for the full streaming response to stabilise in the UI
      await this.waitForStreamingResponse(120000);
    } finally {
      // Stop collecting responses
      this.page.off("response", responseHandler);
    }

    const collectedResponses = await Promise.all(collectedPromises);

    // Parse all collected API responses for tool call data and response text
    for (const raw of collectedResponses) {
      const lines = raw.split("\n").filter((line) => line.trim());
      for (const line of lines) {
        try {
          const chunk = JSON.parse(line);
          // Capture tool call data (prefer complete done events, accumulate delta arguments)
          if (
            (chunk.type === "response.output_item.done" ||
              chunk.type === "response.output_item.added") &&
            (chunk.item?.type === "tool_call" ||
              chunk.item?.type === "function_call")
          ) {
            if (
              !capturedToolData ||
              chunk.item.arguments ||
              chunk.item.inputs ||
              chunk.type === "response.output_item.done"
            ) {
              capturedToolData = { ...chunk.item };
            }
          } else if (chunk.delta?.tool_calls) {
            for (const tc of chunk.delta.tool_calls) {
              if (tc.function?.name) {
                if (!capturedToolData) {
                  capturedToolData = {
                    type: tc.type || "function_call",
                    id: tc.id,
                    name: tc.function.name,
                    arguments: tc.function.arguments || "",
                  };
                } else if (tc.function.arguments) {
                  capturedToolData.arguments =
                    (capturedToolData.arguments || "") + tc.function.arguments;
                }
              }
            }
          }
          // Build full response text
          if (chunk.delta?.content) {
            fullResponseText += chunk.delta.content;
          }
          if (chunk.response?.text) {
            fullResponseText = chunk.response.text;
          }
        } catch {
          // ignore malformed chunks
        }
      }
    }

    if (capturedToolData) {
      const name = capturedToolData.tool_name || capturedToolData.name || "";
      let inputs = capturedToolData.inputs;
      if (!inputs && capturedToolData.arguments) {
        try {
          inputs =
            typeof capturedToolData.arguments === "string"
              ? JSON.parse(capturedToolData.arguments)
              : capturedToolData.arguments;
        } catch {
          inputs = {};
        }
      }
      capturedToolData = {
        ...capturedToolData,
        tool_name: name,
        inputs: inputs || {},
      };
    }

    // Fallback: read response text from the UI if API parsing yielded nothing
    if (!fullResponseText) {
      fullResponseText =
        (await this.lastMarkdownResponse()
          .textContent()
          .catch(() => "")) || "";
    }

    // The "Function Call:" label in the UI is optional — the app may not render it
    // for all tool invocations. Use it when present; fall back to the locator object.
    const toolCall = this.getToolCallLocator();
    const toolCallVisible = await toolCall
      .isVisible({ timeout: 3000 })
      .catch(() => false);
    if (!toolCallVisible) {
      logger.info(
        `  ℹ️  "Function Call:" UI label not visible — relying on captured API tool data`,
      );
    }

    return {
      toolCall,
      toolData: capturedToolData,
      fullResponse: fullResponseText.trim(),
    };
  }

  async isToolFailed(toolCall: Locator): Promise<boolean> {
    const container = toolCall.locator(
      'xpath=ancestor::div[contains(@class,"border")]',
    );
    const text = await container.textContent();
    return /error|failed|timeout/i.test(text || "");
  }

  /**
   * Apply a knowledge filter in the chat
   * @param filterName - Name of the filter to apply
   */
  async applyKnowledgeFilter(filterName: string) {
    const filterBtn = this.filterButton();
    await filterBtn.click();
    await this.page.waitForTimeout(500);
    const filterOption = this.getFilterOption(filterName);
    await expect(filterOption).toBeVisible({ timeout: 5000 });
    await filterOption.click();
    await this.page.waitForTimeout(500);
  }

  /**
   * Remove the currently applied knowledge filter
   */
  async removeKnowledgeFilter() {
    const filterBtn = this.filterButton();
    await filterBtn.click();
    await this.page.waitForTimeout(500);
    const noFilter = this.noFilterOption();
    await expect(noFilter).toBeVisible({ timeout: 5000 });
    await noFilter.click();
    await this.page.waitForTimeout(500);
  }

  /**
   * Check if a knowledge filter exists in the filter list
   * @param filterName - Name of the filter to check
   * @returns true if filter exists, false otherwise
   */
  async isFilterAvailable(filterName: string): Promise<boolean> {
    const filterBtn = this.filterButton();
    await filterBtn.click();
    await this.page.waitForTimeout(500);
    try {
      const filterOption = this.getFilterOption(filterName);
      const isVisible = await filterOption.isVisible({ timeout: 2000 });
      // Close the filter dropdown
      await filterBtn.click();
      await this.page.waitForTimeout(500);
      return isVisible;
    } catch {
      // Close the filter dropdown
      await filterBtn.click();
      await this.page.waitForTimeout(500);
      return false;
    }
  }

  /**
   * Upload a file in the chat section
   * @param filePath - Path to the file to upload
   * @returns The filename that was uploaded
   */
  async ingestFileInChat(filePath: string) {
    logger.info(`Uploading file from chat: ${filePath}`);
    const fileName = path.basename(filePath);
    const [fileChooser] = await Promise.all([
      this.page.waitForEvent("filechooser"),
      this.plusButton().click(),
    ]);
    await fileChooser.setFiles(filePath);
    logger.info(`File uploaded and ready for querying: ${fileName}`);
    return fileName;
  }

  /**
   * Verify uploaded file in the chat section
   * @param fileName - File name
   */
  async verifyFileInChat(fileName: string) {
    const uploadedFile = this.page.locator("p.text-muted-foreground").filter({
      hasText: fileName,
    });
    await expect(uploadedFile).toBeVisible({ timeout: 10000 });
    logger.info(`File visible in chat: ${fileName}`);
  }

  /**
   * Wait for a response containing specific text
   * @param text - Text to wait for in the response
   * @param timeout - Maximum time to wait
   */
  async waitForResponseContaining(
    text: string | RegExp,
    timeout: number = 60000,
  ) {
    const response = this.lastMarkdownDiv();
    await expect(response).toBeVisible({ timeout });
    await expect(response).toContainText(text, { timeout });
  }

  /**
   * Get the last response from the chat
   * @param timeout - Maximum time to wait for response
   * @returns The response text
   */
  async getLastResponse(timeout: number = 60000): Promise<string> {
    const response = this.lastMarkdownDiv();
    await expect(response).toBeVisible({ timeout });
    return await response.innerText();
  }

  /**
   * Get the current theme applied to the page
   * @returns 'light' or 'dark' based on the HTML class or data attribute
   */
  async getCurrentTheme(): Promise<"light" | "dark"> {
    const html = this.htmlElement();
    const classList = (await html.getAttribute("class")) || "";
    const dataTheme = (await html.getAttribute("data-theme")) || "";
    if (classList.includes("dark") || dataTheme.includes("dark")) {
      return "dark";
    }
    return "light";
  }

  /**
   * Detect if the UI is SaaS (has profile menu) or OSS (direct theme button)
   * @returns true if SaaS, false if OSS
   */
  private async isSaaSUI(): Promise<boolean> {
    // Check for SaaS profile button with aria-haspopup="menu"
    const switchThemeButton = this.page
      .locator("button")
      .filter({
        has: this.page.locator("svg.lucide-sun, svg.lucide-moon"),
      })
      .first();
    try {
      await switchThemeButton.waitFor({ state: "visible", timeout: 2000 });
      return false;
    } catch {
      return true;
    }
  }

  /**
   * Switch to a specific theme mode
   * Handles both SaaS (profile menu → theme button) and OSS (direct theme button) UIs
   * @param theme - The theme to switch to: 'light' or 'dark'
   */
  async switchTheme(theme: "light" | "dark" | "system") {
    const currentTheme = await this.getCurrentTheme();
    if (currentTheme === theme) {
      logger.info(`Already in ${theme} theme, skipping switch`);
      return;
    }

    const isSaaS = await this.isSaaSUI();
    logger.info(`Detected UI type: ${isSaaS ? "SaaS" : "OSS"}`);

    if (isSaaS) {
      // SaaS: Click profile button first, then theme button
      const profileButton = this.page
        .locator('button[aria-haspopup="menu"]')
        .first();
      await profileButton.click();
      await this.page.waitForTimeout(500);

      // Click the appropriate theme button based on desired theme
      if (theme === "light") {
        const lightButton = this.page.locator(
          'button[data-testid="menu_light_button"]',
        );
        await lightButton.waitFor({ state: "visible", timeout: 5000 });
        await lightButton.click();
      } else if (theme === "dark") {
        const darkButton = this.page.locator(
          'button[data-testid="menu_dark_button"]',
        );
        await darkButton.waitFor({ state: "visible", timeout: 5000 });
        await darkButton.click();
      } else if (theme === "system") {
        const systemButton = this.page.locator(
          'button[data-testid="menu_system_button"]',
        );
        await systemButton.waitFor({ state: "visible", timeout: 5000 });
        await systemButton.click();
      }
      await this.page.keyboard.press("Escape");
    } else {
      // OSS: Direct theme button click (toggles between light/dark)
      const themeButton = this.page
        .locator("button")
        .filter({
          has: this.page.locator("svg.lucide-sun, svg.lucide-moon"),
        })
        .first();

      await themeButton.waitFor({ state: "visible", timeout: 5000 });
      await themeButton.click();
    }

    await this.page.waitForTimeout(1000);
    logger.info(`Switched to ${theme} theme`);
  }

  /**
   * Verify theme colors are applied correctly
   * @param expectedTheme - The expected theme ('light' or 'dark')
   */
  async verifyThemeColors(expectedTheme: "light" | "dark") {
    const main = this.mainContent();
    const backgroundColor = await main.evaluate((el) => {
      return window.getComputedStyle(el).backgroundColor;
    });
    const rgbMatch = backgroundColor.match(/rgb\((\d+),\s*(\d+),\s*(\d+)\)/);
    if (rgbMatch) {
      const [, r, g, b] = rgbMatch.map(Number);
      const brightness = (r + g + b) / 3;
      if (expectedTheme === "dark") {
        expect(brightness).toBeLessThan(128);
      } else {
        expect(brightness).toBeGreaterThan(128);
      }
    }
  }

  /**
   * Delete a chat conversation by its title (first prompt)
   * Finds the latest chat with the given title, hovers to reveal menu, and deletes it
   * @param chatTitle - The title of the chat (first prompt given to the chat)
   * @returns Promise that resolves when deletion is complete
   */
  async deleteChatByTitle(chatTitle: string) {
    logger.info(`Deleting chat: ${chatTitle}`);
    await this.openNewChat();
    const chatRow = this.getChatRow(chatTitle);
    await expect(chatRow).toBeVisible({ timeout: 30000 });
    await chatRow.hover();
    const moreOptionsButton = chatRow.locator('[aria-haspopup="menu"]');
    await expect(moreOptionsButton).toBeVisible({ timeout: 5000 });
    await moreOptionsButton.click();
    await this.deleteConversationMenuItem().click();
    await this.deleteButton().click();
    await expect(this.conversationDeletedToast()).toBeVisible();
    logger.info(`Deleted chat successfully`);
  }

  // ============================================================================
  // METHODS ADDED FOR AUTOMATED TESTING SUITE
  // ============================================================================

  /**
   * Skip the onboarding tour if it appears
   */
  async skipOnboardingTour() {
    const skipButton = this.page.getByRole("button", {
      name: /skip overview/i,
    });
    try {
      await skipButton.waitFor({ timeout: 3000 });
      await skipButton.click();
      await this.page.waitForTimeout(500);
    } catch (_error) {
      // No onboarding tour, continue
    }
  }

  /**
   * Get the number of available chat suggestion buttons (buttons with >= 15 characters text)
   */
  async getSuggestionCount(): Promise<number> {
    const mainChatArea = this.page.locator('main, [role="main"]').first();
    const suggestionButtons = mainChatArea.getByRole("button").filter({
      hasText: /.{15,}/,
    });
    return await suggestionButtons.count();
  }

  /**
   * Click the first available chat suggestion button and return its text
   */
  async clickFirstSuggestion(): Promise<string> {
    const mainChatArea = this.page.locator('main, [role="main"]').first();
    const suggestionButtons = mainChatArea.getByRole("button").filter({
      hasText: /.{15,}/,
    });
    const firstSuggestion = suggestionButtons.first();
    const suggestionText = (await firstSuggestion.textContent()) || "";
    await firstSuggestion.click();
    return suggestionText;
  }

  /**
   * Wait for the streaming markdown response to complete
   */
  async waitForStreamingResponse(maxWaitTimeMs: number = 120000) {
    const responseLocator = this.page.locator("div.markdown").last();
    await expect(responseLocator).toBeVisible({ timeout: 30000 });
    await this.page.waitForTimeout(2000);

    let previousText = "";
    let stableCount = 0;
    const startTime = Date.now();

    while (stableCount < 3 && Date.now() - startTime < maxWaitTimeMs) {
      await this.page.waitForTimeout(1000);
      const currentText = (await responseLocator.textContent()) || "";

      if (currentText.trim() === "Thinking...") {
        continue;
      }

      if (currentText === previousText && currentText.trim().length > 0) {
        stableCount++;
      } else {
        stableCount = 0;
        previousText = currentText;
      }
    }
  }

  /**
   * Check for any function calls and wait for them to complete
   */
  async waitForFunctionCallsComplete(timeout: number = 30000) {
    const functionCalls = this.page.getByText(/Function Call:/i);
    const functionCallCount = await functionCalls.count();
    if (functionCallCount > 0) {
      const completedBadge = this.page.getByText("completed");
      if ((await completedBadge.count()) > 0) {
        await expect(completedBadge.first()).toBeVisible({ timeout });
      }
    }
  }

  async getConversationCount(): Promise<number> {
    return this.page.locator('[data-testid^="conversation-button-"]').count();
  }

  async enterSelectionMode() {
    await this.selectToggleButton().click();
    await expect(this.selectAllCheckBox()).toBeVisible({ timeout: 5000 });
  }

  async exitSelectionMode() {
    await this.selectToggleButton().click();
  }

  async selectAllConversations() {
    await this.selectAllCheckBox().check();
  }

  async clickBulkDelete() {
    await this.page.getByTestId("bulk-delete-confirm").click();
  }

  async selectConversationByTitle(title: string) {
    const row = this.page.getByTestId(`conversation-button-${title}`);
    const checkbox = row.locator('input[type="checkbox"]');
    await checkbox.check();
  }
}