OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
import { describe, expect, it } from "vitest";
import {
createUrlSourcePayload,
INITIAL_URL_SOURCE_FORM,
isUrlSourceFormValid,
isWebsiteSourceNameAvailable,
lines,
} from "./form";
describe("URL source form helpers", () => {
it("normalizes multiline settings and applies page-only limits", () => {
const form = {
...INITIAL_URL_SOURCE_FORM,
name: "Docs",
starting_url: "https://docs.example.com/guides",
scope: "page" as const,
additional_hosts: " assets.example.com\n\ncdn.example.com ",
include_paths: " /guides\n/reference ",
exclude_paths: " /archive ",
max_pages: 250,
max_depth: 4,
};
expect(lines(form.additional_hosts)).toEqual([
"assets.example.com",
"cdn.example.com",
]);
expect(isUrlSourceFormValid(form)).toBe(true);
expect(createUrlSourcePayload(form)).toMatchObject({
additional_hosts: ["assets.example.com", "cdn.example.com"],
include_paths: ["/guides", "/reference"],
exclude_paths: ["/archive"],
max_pages: 1,
max_depth: 0,
change_detection: "normalized_content_hash",
});
});
it("rejects invalid URLs, host exceptions, and paths", () => {
const valid = {
...INITIAL_URL_SOURCE_FORM,
name: "Docs",
starting_url: "https://docs.example.com",
};
expect(
isUrlSourceFormValid({ ...valid, starting_url: "ftp://example.com" }),
).toBe(false);
expect(
isUrlSourceFormValid({ ...valid, additional_hosts: "192.168.1.10" }),
).toBe(false);
expect(isUrlSourceFormValid({ ...valid, include_paths: "docs" })).toBe(
false,
);
expect(isUrlSourceFormValid({ ...valid, max_pages: 0 })).toBe(false);
expect(isUrlSourceFormValid({ ...valid, max_depth: 21 })).toBe(false);
expect(isUrlSourceFormValid({ ...valid, max_downloaded_mb: 2049 })).toBe(
false,
);
});
it("rejects website connection names that already exist", () => {
const form = {
...INITIAL_URL_SOURCE_FORM,
name: "Docs",
starting_url: "https://docs.example.com",
};
expect(isWebsiteSourceNameAvailable(" docs ", ["Docs"])).toBe(false);
expect(isUrlSourceFormValid(form, ["DOCS"])).toBe(false);
});
});