Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

fix: reduce flakiness of tests and fix failing e2e tests (#2481)

Lucas Oliveiracb7213d249cd
diff --git a/frontend/app/knowledge/page.tsx b/frontend/app/knowledge/page.tsx
index 0c51c8045..4c214abb8 100644
--- a/frontend/app/knowledge/page.tsx
+++ b/frontend/app/knowledge/page.tsx
@@ -816,6 +816,7 @@ function SearchPage() {
setSelectedRows((current) =>
sameFileSelection(current, nextSelected) ? current : nextSelected,
);
+ api.refreshCells({ force: true });
}, [gridRowsSelectionKey, isDeletableKnowledgeRow, getGridApi]);
// ── Shared column fragments ──────────────────────────────────────────────
@@ -825,6 +826,11 @@ function SearchPage() {
headerName: "Source",
sortable: true,
comparator: serverSideComparator,
+ valueGetter: (params: ValueGetterParams<File>) =>
+ `${params.data?.filename ?? ""}::${params.data?.status ?? "active"}`,
+ valueFormatter: (params: ValueFormatterParams<File>) =>
+ params.data?.filename ??
+ (typeof params.value === "string" ? params.value.split("::")[0] : "-"),
checkboxSelection: (params: CheckboxSelectionCallbackParams<File>) =>
isDeletableKnowledgeRow(params?.data),
headerCheckboxSelection: true,
@@ -832,6 +838,9 @@ function SearchPage() {
? { flex: 2.2, minWidth: 260 }
: { initialFlex: 2, minWidth: 220 }),
cellRenderer: ({ data, value }: CustomCellRendererProps<File>) => {
+ const filename =
+ data?.filename ??
+ (typeof value === "string" ? value.split("::")[0] : "");
const status = data?.status || "active";
const isActive = status === "active";
const showOpenragSourceAnimation =
@@ -855,9 +864,7 @@ function SearchPage() {
)}
onClick={() => {
if (!isActive) return;
- router.push(
- buildChunksUrl(data?.filename ?? "", effectiveSearchText),
- );
+ router.push(buildChunksUrl(filename, effectiveSearchText));
}}
>
{getSourceIcon(data?.connector_type)}
@@ -871,11 +878,11 @@ function SearchPage() {
: "text-foreground",
)}
>
- {value}
+ {filename}
</span>
</TooltipTrigger>
<TooltipContent side="top" align="start">
- {value}
+ {filename}
</TooltipContent>
</Tooltip>
</button>
diff --git a/frontend/tests/core/url_ingestion.spec.ts b/frontend/tests/core/url_ingestion.spec.ts
index 728c4ec57..139e29d58 100644
--- a/frontend/tests/core/url_ingestion.spec.ts
+++ b/frontend/tests/core/url_ingestion.spec.ts
@@ -101,7 +101,7 @@ test("URL connector ingestion - Invalid URL handling @34581217", async ({
);
// Attempt to ingest invalid URL
- const invalidUrl = "http://www.invalid-url.com";
+ const invalidUrl = "http://www.nonexistent-domain.invalid";
logger.info(` 🌐 Ingesting invalid URL: ${invalidUrl}`);
await chat.open();
const { toolData, fullResponse } = await chat.ingestUrl(invalidUrl);
diff --git a/frontend/tests/pages/Knowledge.ts b/frontend/tests/pages/Knowledge.ts
index dab72b881..ddd4a5b66 100644
--- a/frontend/tests/pages/Knowledge.ts
+++ b/frontend/tests/pages/Knowledge.ts
@@ -668,11 +668,17 @@ export class Knowledge {
// Find the document row using reliable method
const row = await this.findRowAcrossPages(fileName);
- // Click on the document link
- const fileLink = row.locator("span").filter({ hasText: fileName }).first();
+ // Click on the document link button
+ const fileButton = row
+ .getByRole("button", { name: fileName })
+ .or(row.locator("button").filter({ hasText: fileName }));
- await expect(fileLink).toBeVisible({ timeout: 10000 });
- await fileLink.click();
+ await expect(fileButton).toBeVisible({ timeout: 10000 });
+ await expect(fileButton).not.toHaveClass(/cursor-default/, {
+ timeout: 10000,
+ });
+ await fileButton.click();
+ await this.page.waitForURL(/\/knowledge\/chunks/, { timeout: 15000 });
}
async getFirstChunkText(): Promise<string> {
@@ -1093,7 +1099,7 @@ export class Knowledge {
}
return false;
},
- { timeout: 10000 },
+ { timeout: 15000 },
)
.toBe(true);
// Get all chunks after search
diff --git a/frontend/tests/pages/Settings.ts b/frontend/tests/pages/Settings.ts
index d2ac88af1..bdef24ed8 100644
--- a/frontend/tests/pages/Settings.ts
+++ b/frontend/tests/pages/Settings.ts
@@ -47,7 +47,7 @@ export class Settings {
private readonly removeModelProviderButton = () =>
this.page.getByRole("button", { name: "Remove" });
private readonly removeAnywayButton = () =>
- this.page.getByRole("button", { name: "Remove Anyway" });
+ this.page.getByRole("button", { name: /remove anyway/i });
// The dialog's Remove button is also styled `.text-destructive`, so the old
// selector matched it and made `successToast.or(errorMsg)` ambiguous under
// Playwright strict mode. Target the error element itself.
@@ -118,7 +118,7 @@ export class Settings {
*/
private getEditSetupButton(providerKey: ModelProvider) {
return this.getProviderCard(providerKey).getByRole("button", {
- name: "Edit Setup",
+ name: /edit setup|fix setup/i,
});
}
@@ -385,6 +385,8 @@ export class Settings {
*/
async configureWatsonxai() {
logger.info("Configuring watsonx.ai settings");
+ const card = this.getProviderCard("watsonx");
+ await expect(card).toBeVisible({ timeout: 15000 });
const configureBtn = this.getConfigureButton("watsonx");
const editBtn = this.getEditSetupButton("watsonx");
// If Configure button is visible -> do setup
@@ -411,10 +413,35 @@ export class Settings {
logger.info("Watsonx.ai configuration completed");
await expect(editBtn).toBeEnabled();
}
- // Else if already configured -> skip setup
+ // Else if already configured -> skip setup (or reconfigure if in Fix Setup state)
else if (await editBtn.isVisible()) {
- logger.info("Watsonx.ai already configured. Skipping setup.");
- await expect(editBtn).toBeEnabled();
+ const btnText = await editBtn.innerText();
+ if (/fix setup/i.test(btnText)) {
+ logger.info("Watsonx.ai is in Fix Setup state. Reconfiguring...");
+ await editBtn.click();
+ await expect(this.getSetupHeading("IBM watsonx.ai")).toBeVisible();
+ const { url, projectId, apiKey } = config.watsonx;
+ await this.watsonxEndPointCombobox().click();
+ await this.page.waitForSelector('[role="option"]', {
+ state: "visible",
+ timeout: 15000,
+ });
+ const option = this.getWatsonxOption(url);
+ await expect(option).toBeVisible({ timeout: 10000 });
+ await option.click();
+ await this.watsonxProjectIDInput().fill(projectId);
+ await this.apiKeyInput().fill(apiKey);
+ await this.saveModelProviderButton().click();
+ await this.awaitProviderConfigResult(
+ "Watsonx.ai",
+ "IBM watsonx.ai successfully configured",
+ );
+ logger.info("Watsonx.ai configuration completed");
+ await expect(editBtn).toBeEnabled();
+ } else {
+ logger.info("Watsonx.ai already configured. Skipping setup.");
+ await expect(editBtn).toBeEnabled();
+ }
}
// Neither found
else {
@@ -461,19 +488,46 @@ export class Settings {
providerKey: ModelProvider,
providerName: string,
) {
+ const card = this.getProviderCard(providerKey);
+ await expect(card).toBeVisible({ timeout: 15000 });
const editButton = this.getEditSetupButton(providerKey);
const configureButton = this.getConfigureButton(providerKey);
- // If already configured (Edit Setup visible)
+ // If already configured (Edit Setup / Fix Setup visible)
if (await editButton.isVisible()) {
logger.info(`${providerName} is configured. Removing setup...`);
await editButton.click();
await this.removeModelProviderButton().click();
await this.getRemoveConfigButton().click();
- await this.clickRemoveAnywayIfDisplayed();
- await expect(
- this.getToastByText(`${providerName} configuration removed`),
- ).toBeVisible({ timeout: 15000 });
- await this.page.waitForTimeout(10000);
+
+ const successToast = this.getToastByText(
+ `${providerName} configuration removed`,
+ );
+ const removeAnywayBtn = this.removeAnywayButton();
+
+ let outcome = "pending";
+ await expect
+ .poll(
+ async () => {
+ if (await removeAnywayBtn.isVisible().catch(() => false)) {
+ outcome = "remove_anyway";
+ } else if (await successToast.isVisible().catch(() => false)) {
+ outcome = "success";
+ } else {
+ outcome = "pending";
+ }
+ return outcome;
+ },
+ { timeout: 15000 },
+ )
+ .not.toBe("pending");
+
+ if (outcome === "remove_anyway") {
+ logger.info("Remove Anyway button is displayed. Clicking it.");
+ await removeAnywayBtn.click();
+ }
+ await expect(successToast).toBeVisible({ timeout: 15000 });
+
+ await this.page.waitForTimeout(1000);
}
// If not configured
else if (await configureButton.isVisible()) {
@@ -511,6 +565,8 @@ export class Settings {
*/
async configureOpenAPI() {
logger.info("Configuring Openai settings");
+ const card = this.getProviderCard("openai");
+ await expect(card).toBeVisible({ timeout: 15000 });
const configureBtn = this.getConfigureButton("openai");
const editBtn = this.getEditSetupButton("openai");
@@ -528,14 +584,32 @@ export class Settings {
logger.info("OpenAI configuration completed");
await expect(editBtn).toBeEnabled();
}
- // Else if already configured -> skip setup
+ // Else if already configured -> skip setup (or reconfigure if in Fix Setup state)
else if (await editBtn.isVisible()) {
- logger.info("OpenAI already configured. Skipping setup.");
- await expect(editBtn).toBeEnabled();
+ const btnText = await editBtn.innerText();
+ if (/fix setup/i.test(btnText)) {
+ logger.info("OpenAI is in Fix Setup state. Reconfiguring...");
+ await editBtn.click();
+ await expect(this.getSetupHeading("OpenAI")).toBeVisible();
+ const apiKey = config.openaiApiKey;
+ await this.apiKeyInput().fill(apiKey);
+ await this.saveModelProviderButton().click();
+ await this.awaitProviderConfigResult(
+ "OpenAI",
+ "OpenAI successfully configured",
+ );
+ logger.info("OpenAI configuration completed");
+ await expect(editBtn).toBeEnabled();
+ } else {
+ logger.info("OpenAI already configured. Skipping setup.");
+ await expect(editBtn).toBeEnabled();
+ }
}
// Neither found
else {
- throw new Error("Neither Configure nor Edit Setup button is visible");
+ throw new Error(
+ "Neither Configure nor Edit Setup button is visible for OpenAI",
+ );
}
}
@@ -544,9 +618,10 @@ export class Settings {
*/
async configureAzureOpenAI() {
logger.info("Configuring Azure OpenAI settings");
+ const card = this.getProviderCard("azure");
+ await expect(card).toBeVisible({ timeout: 15000 });
const configureBtn = this.getConfigureButton("azure");
const editBtn = this.getEditSetupButton("azure");
- await this.page.waitForTimeout(500);
if (await configureBtn.isVisible()) {
await configureBtn.click();
@@ -574,8 +649,37 @@ export class Settings {
logger.info("Azure OpenAI configuration completed");
await expect(editBtn).toBeEnabled();
} else if (await editBtn.isVisible()) {
- logger.info("Azure OpenAI already configured. Skipping setup.");
- await expect(editBtn).toBeEnabled();
+ const btnText = await editBtn.innerText();
+ if (/fix setup/i.test(btnText)) {
+ logger.info("Azure OpenAI is in Fix Setup state. Reconfiguring...");
+ await editBtn.click();
+ await expect(this.getSetupHeading("Azure OpenAI")).toBeVisible();
+ const apiKey = config.azure.apiKey;
+ const endpoint = config.azure.endpoint;
+ if (endpoint) {
+ const endpointInput = this.page
+ .locator('input[id*="endpoint"], input[id*="api_base"]')
+ .first();
+ if (await endpointInput.isVisible()) {
+ await endpointInput.fill(endpoint);
+ }
+ }
+ if (apiKey) {
+ const apiKeyInput = this.providerApiKeyInput();
+ await expect(apiKeyInput).toBeVisible();
+ await apiKeyInput.fill(apiKey);
+ }
+ await this.saveModelProviderButton().click();
+ await this.awaitProviderConfigResult(
+ "Azure OpenAI",
+ "Azure OpenAI successfully configured",
+ );
+ logger.info("Azure OpenAI configuration completed");
+ await expect(editBtn).toBeEnabled();
+ } else {
+ logger.info("Azure OpenAI already configured. Skipping setup.");
+ await expect(editBtn).toBeEnabled();
+ }
} else {
throw new Error(
"Neither Configure nor Edit Setup button is visible for Azure OpenAI",
@@ -592,29 +696,39 @@ export class Settings {
apiKey: string,
) {
logger.info("Configuring watsonx.ai settings with invalid credentials");
+ const card = this.getProviderCard("watsonx");
+ await expect(card).toBeVisible({ timeout: 15000 });
const configureBtn = this.getConfigureButton("watsonx");
- // If Configure button is visible -> do setup
+ const editBtn = this.getEditSetupButton("watsonx");
+
if (await configureBtn.isVisible()) {
await configureBtn.click();
- await expect(this.getSetupHeading("IBM watsonx.ai")).toBeVisible();
- await this.watsonxEndPointCombobox().click();
- // Wait for the dropdown to open by polling until any option appears
- await this.page.waitForSelector('[role="option"]', {
- state: "visible",
- timeout: 15000,
- });
- const option = this.getWatsonxOption(url);
- await expect(option).toBeVisible({ timeout: 10000 });
- await option.click();
- await this.watsonxProjectIDInput().fill(projectId);
- await this.apiKeyInput().fill(apiKey);
- await this.saveModelProviderButton().click();
- logger.info(
- "Verify that watsonx.ai configuration failed due to invalid credentials",
+ } else if (await editBtn.isVisible()) {
+ await editBtn.click();
+ } else {
+ throw new Error(
+ "Neither Configure nor Edit Setup button is visible for watsonx",
);
- await expect(this.providerConnectionErrorMessage()).toBeVisible({
- timeout: 30000,
- });
}
+
+ await expect(this.getSetupHeading("IBM watsonx.ai")).toBeVisible();
+ await this.watsonxEndPointCombobox().click();
+ // Wait for the dropdown to open by polling until any option appears
+ await this.page.waitForSelector('[role="option"]', {
+ state: "visible",
+ timeout: 15000,
+ });
+ const option = this.getWatsonxOption(url);
+ await expect(option).toBeVisible({ timeout: 10000 });
+ await option.click();
+ await this.watsonxProjectIDInput().fill(projectId);
+ await this.apiKeyInput().fill(apiKey);
+ await this.saveModelProviderButton().click();
+ logger.info(
+ "Verify that watsonx.ai configuration failed due to invalid credentials",
+ );
+ await expect(this.providerConnectionErrorMessage()).toBeVisible({
+ timeout: 30000,
+ });
}
}
diff --git a/frontend/tests/utils/onboarding.ts b/frontend/tests/utils/onboarding.ts
index 4a94f6f02..f8b2afc0f 100644
--- a/frontend/tests/utils/onboarding.ts
+++ b/frontend/tests/utils/onboarding.ts
@@ -339,5 +339,21 @@ export async function completeOnboarding(
throw new Error(`Onboarding document upload failed: ${errorText}`);
}
- await expect(page.getByTestId("onboarding-content")).toBeHidden();
+ // If the ingest review dialog opened, close it so onboarding can advance to chat
+ const reviewDialog = page.getByRole("dialog", { name: /ingestion review/i });
+ const closeReviewButton = reviewDialog.getByRole("button", {
+ name: /close/i,
+ });
+ const reviewOpened = await closeReviewButton
+ .waitFor({ state: "visible", timeout: 3000 })
+ .then(() => true)
+ .catch(() => false);
+ if (reviewOpened) {
+ await closeReviewButton.click();
+ await expect(reviewDialog).toBeHidden();
+ }
+
+ await expect(page.getByTestId("onboarding-content")).toBeHidden({
+ timeout: 15000,
+ });
}