Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python
11.4 KB
Raw Blame
import { expect, type Page } from "@playwright/test";
import path from "path";
import { ACTIVE_PROVIDER_CONFIG } from "../config/provider";

export type LLMProvider =
  | "azure"
  | "openai"
  | "anthropic"
  | "watsonx"
  | "ollama";
export type EmbeddingProvider = "azure" | "openai" | "watsonx" | "ollama";

/**
 * E2E credentials use Azure OpenAI, whose onboarding tab is `azure`.
 * `azure_ai` is Azure AI Foundry and has separate credentials, but older CI
 * configuration used that provider name for the shared Azure OpenAI secret.
 */
function normalizeProviderKey(provider: string): string {
  return provider === "azure_ai" ? "azure" : provider;
}

function getProviderKey(providerName: string): LLMProvider {
  switch (providerName) {
    case "Azure OpenAI":
      return "azure";
    case "IBM watsonx.ai":
      return "watsonx";
    case "Ollama":
      return "ollama";
    case "Anthropic":
      return "anthropic";
    default:
      return "openai";
  }
}

const activeProviderKey = getProviderKey(ACTIVE_PROVIDER_CONFIG.provider);

const defaultLlmProvider: LLMProvider =
  (normalizeProviderKey(
    (process.env.LLM_PROVIDER || "").trim().toLowerCase(),
  ) as LLMProvider) || activeProviderKey;

const defaultEmbeddingProvider: EmbeddingProvider =
  (normalizeProviderKey(
    (process.env.EMBEDDING_PROVIDER || "").trim().toLowerCase(),
  ) as EmbeddingProvider) ||
  (activeProviderKey === "anthropic"
    ? "openai"
    : (activeProviderKey as EmbeddingProvider));

export async function completeOnboarding(
  page: Page,
  {
    llmProvider = defaultLlmProvider,
    embeddingProvider = defaultEmbeddingProvider,
    reset = false,
  }: {
    llmProvider?: LLMProvider;
    embeddingProvider?: EmbeddingProvider;
    reset?: boolean;
  } = {},
) {
  // Fast path checks for environment variables
  const checkCredentials = (provider: string) => {
    if (provider === "ollama") return;
    if (provider === "azure") {
      const key = process.env.AZURE_OPENAI_API_KEY || process.env.AZURE_API_KEY;
      const endpoint =
        process.env.AZURE_OPENAI_ENDPOINT ||
        process.env.AZURE_OPENAI_API_BASE ||
        process.env.AZURE_API_BASE;
      if (!key) throw new Error("AZURE_OPENAI_API_KEY is not set");
      if (!endpoint) throw new Error("AZURE_OPENAI_ENDPOINT is not set");
      return;
    }
    const envVarName = `${provider.toUpperCase()}_API_KEY`;
    if (!process.env[envVarName]) {
      throw new Error(`${envVarName} is not set`);
    }
    if (provider === "watsonx" && !process.env.WATSONX_PROJECT_ID) {
      throw new Error("WATSONX_PROJECT_ID is not set");
    }
  };

  checkCredentials(llmProvider);
  if ((embeddingProvider as string) === "anthropic") {
    throw new Error("Anthropic is not a valid embedding provider");
  }
  checkCredentials(embeddingProvider);

  // Go to the base URL (frontend)
  await page.goto("/");

  if (reset) {
    const response = await page.request.post("/api/onboarding/rollback");
    if (!response.ok() && response.status() !== 400) {
      const text = await response.text();
      throw new Error(`Failed to rollback onboarding: ${text}`);
    }
    await page.reload();
  }

  // Wait for either onboarding to be complete or onboarding content to be visible
  const completedLocator = page.getByTestId("onboarding-completed");
  const contentLocator = page.getByTestId("onboarding-content");

  try {
    await expect(completedLocator.or(contentLocator)).toBeVisible({
      timeout: 15000,
    });
  } catch {
    await page.reload();
    await expect(completedLocator.or(contentLocator)).toBeVisible({
      timeout: 15000,
    });
  }

  const isCompleted = await completedLocator.isVisible();
  const isFirstStep =
    (await page
      .getByTestId("openai-llm-tab")
      .isVisible()
      .catch(() => false)) ||
    (await page
      .getByTestId("azure-llm-tab")
      .isVisible()
      .catch(() => false));

  if (isCompleted && !reset) {
    return;
  }

  const needsRollback = (reset && isCompleted) || !isFirstStep;

  if (needsRollback) {
    const response = await page.request.post("/api/onboarding/rollback");
    if (!response.ok()) {
      const text = await response.text();
      console.error(
        `Rollback failed with status ${response.status()}: ${text}`,
      );
      if (response.status() !== 400) {
        throw new Error(`Failed to rollback onboarding: ${text}`);
      }
    }
    await page.reload();
    // After rollback and reload, we must see the onboarding content
    await expect(contentLocator).toBeVisible({ timeout: 15000 });
  }

  const setupProvider = async (provider: string, isEmbedding: boolean) => {
    const tabId = `${provider}-${isEmbedding ? "embedding" : "llm"}-tab`;
    await page.getByTestId(tabId).scrollIntoViewIfNeeded();
    await page.waitForTimeout(500);
    await page.getByTestId(tabId).click();

    if (provider === "azure") {
      const endpoint =
        process.env.AZURE_OPENAI_ENDPOINT ||
        process.env.AZURE_OPENAI_API_BASE ||
        process.env.AZURE_API_BASE ||
        "";
      const key =
        process.env.AZURE_OPENAI_API_KEY || process.env.AZURE_API_KEY || "";
      const baseInput = page.locator(
        "#onboarding-azure-api_base, [data-testid='endpoint']",
      );
      if ((await baseInput.isVisible()) && (await baseInput.isEnabled())) {
        const val = await baseInput.inputValue();
        if (!val && endpoint) {
          await baseInput.fill(endpoint);
        }
      }
      const keyInput = page.locator(
        "#onboarding-azure-api_key, [data-testid='api-key']",
      );
      if ((await keyInput.isVisible()) && (await keyInput.isEnabled())) {
        const val = await keyInput.inputValue();
        if (!val && key) {
          await keyInput.fill(key);
        }
      }
    } else if (provider !== "ollama") {
      const getFromEnvSwitch = page.getByTestId("get-from-env-switch");

      // Check if switch is visible and toggle off to enter explicit key if needed
      if (await getFromEnvSwitch.isVisible()) {
        if (await getFromEnvSwitch.isChecked()) {
          await getFromEnvSwitch.click();
        }
        await expect(getFromEnvSwitch).not.toBeChecked();
      }

      const apiKeyField = page.getByTestId("api-key");
      // For embedding, the key might be automatically populated if it's the same provider
      // but let's ensure it's filled.
      if ((await apiKeyField.isVisible()) && (await apiKeyField.isEnabled())) {
        const apiKey = process.env[`${provider.toUpperCase()}_API_KEY`];
        await apiKeyField.fill(apiKey!);
      }

      if (provider === "watsonx") {
        const projectIdField = page.getByTestId("project-id");
        if (
          (await projectIdField.isVisible()) &&
          (await projectIdField.isEnabled())
        ) {
          const projectId = process.env.WATSONX_PROJECT_ID;
          await projectIdField.fill(projectId!);
        }
      }
    }

    // Model selection
    // Some providers might have models loaded immediately, some might take time
    const advancedBtn = page.getByTestId("advanced-settings-button");
    if (await advancedBtn.isVisible()) {
      await advancedBtn.click();
    }

    const modelSelectorId = isEmbedding
      ? "embedding-model-selector"
      : "language-model-selector";
    const selector = page.getByTestId(modelSelectorId);

    // Wait for the selector to be enabled (models loaded)
    await expect(selector).toBeEnabled({ timeout: 30000 });
    await selector.click();

    // Select preferred model if available, else first option
    const preferredModel =
      provider === activeProviderKey
        ? isEmbedding
          ? ACTIVE_PROVIDER_CONFIG.embedding
          : ACTIVE_PROVIDER_CONFIG.language
        : isEmbedding
          ? process.env.EMBEDDING_MODEL ||
            (provider === "azure" ? "text-embedding-3-small" : undefined)
          : process.env.LLM_MODEL ||
            (provider === "azure" ? "gpt-4.1" : undefined);

    await page.waitForTimeout(2000);
    let selected = false;
    if (preferredModel) {
      const searchInput = page.getByTestId("model-search-input");
      await searchInput.fill(preferredModel, { timeout: 2000 });
      await page.waitForTimeout(2000);
      const preferredOption = page.getByTestId(
        `model-option-${preferredModel}`,
      );
      if (
        await preferredOption.isVisible({ timeout: 2000 }).catch(() => false)
      ) {
        await preferredOption.click();
        selected = true;
      } else {
        await searchInput.clear();
      }
    }
    if (!selected) {
      await expect(page.getByTestId(/^model-option-/).first()).toBeVisible();
      await page
        .getByTestId(/^model-option-/)
        .first()
        .click();
    }

    // Complete this step
    await page.getByTestId("onboarding-complete-button").click();

    const doneLocator = page.getByText("Done");
    const errorLocator = page.getByTestId("onboarding-error");

    await expect(
      page.getByText("Thinking").or(doneLocator).or(errorLocator),
    ).toBeVisible();

    await expect(doneLocator.or(errorLocator)).toBeVisible({
      timeout: isEmbedding ? 120000 : 60000,
    });

    if (await errorLocator.isVisible()) {
      const errorText = await errorLocator.innerText();
      throw new Error(`Onboarding step failed: ${errorText}`);
    }
  };

  // 1. LLM configuration
  await setupProvider(llmProvider, false);

  // 2. Embeddings configuration
  await setupProvider(embeddingProvider, true);

  // 3. What is OpenRAG (Tutorial)
  await expect(
    page.getByText("Excellent, let's move on to learning the basics."),
  ).toBeVisible();

  await page.waitForTimeout(2000);

  await expect(page.getByTestId("suggestion-0")).toBeVisible();
  await page.getByTestId("suggestion-0").click();

  await expect(page.getByTestId("user-message").first()).toHaveText(
    "What is OpenRAG?",
  );
  const openRagAnswer = page.getByText("is an open-source package");
  await expect(page.getByText("Thinking").or(openRagAnswer)).toBeVisible({
    timeout: 60000,
  });
  await expect(openRagAnswer).toBeVisible({
    timeout: 60000,
  });

  // 4. Add your document
  await expect(page.getByText("Lastly, let's add your data.")).toBeVisible({
    timeout: 30000,
  });
  await page.waitForTimeout(2000);
  await expect(page.getByTestId("upload-button")).toBeVisible();

  const fileChooserPromise = page.waitForEvent("filechooser");
  await page.getByTestId("upload-button").click();
  const fileChooser = await fileChooserPromise;
  await fileChooser.setFiles(
    path.join(__dirname, "../assets", "test-document.md"),
  );

  const uploadDoneLocator = page.getByText("Done");
  const uploadErrorLocator = page.getByTestId("onboarding-upload-error");

  await expect(uploadDoneLocator.or(uploadErrorLocator)).toBeVisible({
    timeout: 120000,
  });

  if (await uploadErrorLocator.isVisible()) {
    const errorText = await uploadErrorLocator.innerText();
    throw new Error(`Onboarding document upload failed: ${errorText}`);
  }

  // If the ingest review dialog opened, close it so onboarding can advance to chat
  const reviewDialog = page.getByRole("dialog", { name: /ingestion review/i });
  const closeReviewButton = reviewDialog.getByRole("button", {
    name: /close/i,
  });
  const reviewOpened = await closeReviewButton
    .waitFor({ state: "visible", timeout: 3000 })
    .then(() => true)
    .catch(() => false);
  if (reviewOpened) {
    await closeReviewButton.click();
    await expect(reviewDialog).toBeHidden();
  }

  await expect(page.getByTestId("onboarding-content")).toBeHidden({
    timeout: 15000,
  });
}