Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

3.8 KB
Raw Blame
import type { RefreshIntervalValue } from './api'
import type { KnowledgeSourceApi } from './generated/api.schemas'

export type KnowledgeSource = KnowledgeSourceApi
export type CrawlMode = 'single' | 'sitemap' | 'same_origin' | 'github_repo'

export interface TextSourceFormValues {
    name: string
    text: string
    always_include: boolean
}

export interface UrlSourceFormValues {
    name: string
    url: string
    crawl_mode: CrawlMode
    // Comma/newline-separated globs — split server-side. Keeping the form
    // state as a plain string is much easier than a list-of-inputs widget.
    include_globs: string
    exclude_globs: string
    max_pages: number
    max_depth: number
    // Background auto-refresh cadence, sent on both create and edit.
    refresh_interval: RefreshIntervalValue
    always_include: boolean
}

export const MAX_TEXT_BYTES = 1_000_000

// Keep in step with DEFAULT_MAX_PAGES and MAX_URLS_PER_SOURCE.
export const DEFAULT_MAX_PAGES = 200
export const MAX_PAGES_PER_SOURCE = 5000

export const DEFAULT_URL_SOURCE_FORM: UrlSourceFormValues = {
    name: '',
    url: '',
    crawl_mode: 'single',
    include_globs: '',
    exclude_globs: '',
    max_pages: DEFAULT_MAX_PAGES,
    max_depth: 2,
    refresh_interval: 'manual',
    always_include: false,
}

export function validateText({ name, text }: TextSourceFormValues): {
    name: string | undefined
    text: string | undefined
} {
    return {
        name: !name.trim() ? 'Give the source a short name' : undefined,
        text: !text.trim()
            ? 'Paste some content'
            : new Blob([text]).size > MAX_TEXT_BYTES
              ? 'Text exceeds the 1 MB cap. Split it into smaller sources'
              : undefined,
    }
}

export function validateUrl({ name, url, max_pages }: UrlSourceFormValues): {
    name: string | undefined
    url: string | undefined
    max_pages: string | undefined
} {
    let urlError: string | undefined
    if (!url.trim()) {
        urlError = 'Paste a public HTTPS URL'
    } else {
        try {
            const parsed = new URL(url)
            if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
                urlError = 'Only http(s) URLs are allowed'
            }
        } catch {
            urlError = 'Not a valid URL'
        }
    }
    return {
        name: !name.trim() ? 'Give the source a short name' : undefined,
        url: urlError,
        max_pages:
            max_pages < 1 || max_pages > MAX_PAGES_PER_SOURCE
                ? `Max pages must be between 1 and ${MAX_PAGES_PER_SOURCE}`
                : undefined,
    }
}

export function fileNameWithoutExtension(file: File): string {
    return file.name.replace(/\.[^.]+$/, '')
}

export function formatFileSize(bytes: number): string {
    if (bytes < 1024) {
        return `${bytes} B`
    }
    if (bytes < 1024 * 1024) {
        return `${Math.round(bytes / 1024)} KB`
    }
    return `${(bytes / (1024 * 1024)).toFixed(1)} MB`
}

export function splitGlobs(raw: string): string[] {
    return raw
        .split(/[\n,]/)
        .map((s) => s.trim())
        .filter(Boolean)
}

export function editUrlSourceValuesFromSource(source: KnowledgeSource): UrlSourceFormValues {
    const cfg = (source.crawl_config || {}) as {
        include_globs?: string[]
        exclude_globs?: string[]
        max_pages?: number
        max_depth?: number
    }
    return {
        name: source.name,
        url: source.source_url,
        crawl_mode: (source.crawl_mode || 'single') as CrawlMode,
        include_globs: (cfg.include_globs || []).join('\n'),
        exclude_globs: (cfg.exclude_globs || []).join('\n'),
        max_pages: cfg.max_pages ?? DEFAULT_MAX_PAGES,
        max_depth: cfg.max_depth ?? 2,
        refresh_interval: (source.refresh_interval || 'manual') as RefreshIntervalValue,
        always_include: source.always_include ?? false,
    }
}