Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python
3.4 KB
Raw Blame
export type Scope = "path" | "page" | "site";
export type ChangeDetection = "normalized_content_hash" | "always_reingest";

export const CRAWL_LIMITS = {
  max_pages: { min: 1, max: 10_000 },
  max_depth: { min: 0, max: 20 },
  max_downloaded_mb: { min: 1, max: 2048 },
  max_crawl_minutes: { min: 1, max: 60 },
} as const;

export type UrlSourceForm = {
  name: string;
  starting_url: string;
  scope: Scope;
  allow_subdomains: boolean;
  additional_hosts: string;
  include_paths: string;
  exclude_paths: string;
  max_pages: number;
  max_depth: number;
  max_downloaded_mb: number;
  max_crawl_minutes: number;
  change_detection: ChangeDetection;
  resync_behavior: "full" | "root";
  removed_page_behavior: "retain" | "delete";
};

export type UpdateUrlSourceForm = <K extends keyof UrlSourceForm>(
  key: K,
  value: UrlSourceForm[K],
) => void;

export const INITIAL_URL_SOURCE_FORM: UrlSourceForm = {
  name: "",
  starting_url: "",
  scope: "path",
  allow_subdomains: false,
  additional_hosts: "",
  include_paths: "",
  exclude_paths: "",
  max_pages: 1000,
  max_depth: 4,
  max_downloaded_mb: 128,
  max_crawl_minutes: 15,
  change_detection: "normalized_content_hash",
  resync_behavior: "full",
  removed_page_behavior: "retain",
};

export function lines(value: string) {
  return value.split("\n").flatMap((line) => {
    const trimmed = line.trim();
    return trimmed ? [trimmed] : [];
  });
}

export function isWebsiteSourceNameAvailable(
  name: string,
  existingNames: readonly string[],
) {
  const normalized = name.trim().toLocaleLowerCase();
  return (
    !normalized ||
    !existingNames.some(
      (existingName) => existingName.trim().toLocaleLowerCase() === normalized,
    )
  );
}

function validUrl(value: string) {
  try {
    const url = new URL(value);
    return (
      ["http:", "https:"].includes(url.protocol) &&
      !url.username &&
      !url.password &&
      (!url.port || url.port === "80" || url.port === "443") &&
      !/^\d{1,3}(\.\d{1,3}){3}$/.test(url.hostname) &&
      !url.hostname.includes(":")
    );
  } catch {
    return false;
  }
}

export function isUrlSourceFormValid(
  form: UrlSourceForm,
  existingNames: readonly string[] = [],
) {
  const pageScope = form.scope === "page";
  const limitsAreValid = (
    Object.entries(CRAWL_LIMITS) as [
      keyof typeof CRAWL_LIMITS,
      (typeof CRAWL_LIMITS)[keyof typeof CRAWL_LIMITS],
    ][]
  ).every(([key, limits]) => {
    const value =
      pageScope && key === "max_pages"
        ? 1
        : pageScope && key === "max_depth"
          ? 0
          : form[key];
    return (
      Number.isInteger(value) && value >= limits.min && value <= limits.max
    );
  });

  return Boolean(
    form.name.trim() &&
      validUrl(form.starting_url) &&
      lines(form.additional_hosts).every(
        (host) => !host.includes("/") && !/^\d{1,3}(\.\d{1,3}){3}$/.test(host),
      ) &&
      [...lines(form.include_paths), ...lines(form.exclude_paths)].every(
        (path) => path.startsWith("/"),
      ) &&
      limitsAreValid &&
      isWebsiteSourceNameAvailable(form.name, existingNames),
  );
}

export function createUrlSourcePayload(form: UrlSourceForm) {
  const pageScope = form.scope === "page";
  return {
    ...form,
    additional_hosts: lines(form.additional_hosts),
    include_paths: lines(form.include_paths),
    exclude_paths: lines(form.exclude_paths),
    max_pages: pageScope ? 1 : form.max_pages,
    max_depth: pageScope ? 0 : form.max_depth,
  };
}