Skip to content

armgit / dub

Public Imported from GitHub
0

The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.

6.7 KB
Raw Blame
import { vectorIndex } from "../../lib/upstash/vector";
/**
 * Clean raw MDX fetched from Mintlify's .md endpoint.
 * Strips frontmatter, imports, images, JSX components.
 * Returns the cleaned content and the article title extracted from the H1 heading.
 */
async function cleanMdx(
  raw: string,
): Promise<{ content: string; title: string }> {
  let content = raw;

  content = content.replace(/^> ## Documentation Index\n(?:> [^\n]*\n)*/m, "");

  const h1Match = content.match(/^# (.+)$/m);
  const title = h1Match ? h1Match[1].trim() : "";

  content = content.replace(/^---[\s\S]*?---\s*/m, "");
  content = content.replace(/^import\s+.*from\s+['"][^'"]*['"]\s*;?\s*$/gm, "");
  content = content.replace(/!\[.*?\]\(.*?\)/g, "");

  content = content.replace(/<[A-Z][A-Za-z]*\s*\/>/g, "");

  const blockComponents = [
    "CardGroup",
    "Card",
    "Tabs",
    "Tab",
    "Note",
    "Tip",
    "Warning",
    "Info",
    "Check",
    "Steps",
    "Step",
    "Frame",
    "Accordion",
    "AccordionGroup",
    "ResponseField",
    "ParamField",
    "Expandable",
    "Update",
  ];
  for (const tag of blockComponents) {
    content = content.replace(new RegExp(`<${tag}(?:\\s[^>]*)?>`, "g"), "");
    content = content.replace(new RegExp(`</${tag}>`, "g"), "");
  }

  content = content.replace(/^\s*\w+=\{.*?\}\s*$/gm, "");
  content = content.replace(/\{\/\*[\s\S]*?\*\/\}/g, "");
  content = content.replace(/\{`[^`]*`\}/g, "");
  content = content.replace(/<video[\s\S]*?<\/video>/g, "");
  content = content.replace(/<img[^>]*>/g, "");
  content = content.replace(/\n{3,}/g, "\n\n");

  return { content: content.trim(), title };
}

type ArticleChunk = {
  id: string;
  content: string;
  url: string;
  heading: string;
  title: string;
  type: "docs" | "help";
};

/**
 * Split cleaned markdown into chunks at H2/H3 heading boundaries.
 * Each chunk carries the section URL + heading as metadata.
 */
function chunkByHeadings(
  content: string,
  url: string,
  title: string,
): ArticleChunk[] {
  const lines = content.split("\n");
  const chunks: ArticleChunk[] = [];
  let currentHeading = "Introduction";
  let currentLines: string[] = [];
  const seenSlugs = new Map<string, number>();

  const type: "docs" | "help" = url.includes("/help/") ? "help" : "docs";

  const flush = () => {
    const text = currentLines.join("\n").trim();
    if (text.length < 50) {
      currentLines = [];
      return;
    }

    const baseSlug = currentHeading
      .toLowerCase()
      .replace(/[^a-z0-9]+/g, "-")
      .replace(/^-+|-+$/g, "");

    const count = (seenSlugs.get(baseSlug) ?? 0) + 1;
    seenSlugs.set(baseSlug, count);
    const slug = count === 1 ? baseSlug : `${baseSlug}-${count}`;

    const id = `${url}#${slug}`;
    chunks.push({
      id,
      content: `## ${currentHeading}\n\n${text}`,
      url: `${url}#${slug}`,
      heading: currentHeading,
      title,
      type,
    });
    currentLines = [];
  };

  for (const line of lines) {
    const h2Match = line.match(/^## (.+)$/);
    const h3Match = line.match(/^### (.+)$/);

    if (h2Match || h3Match) {
      flush();
      currentHeading = (h2Match || h3Match)![1].trim();
    } else {
      currentLines.push(line);
    }
  }

  flush();

  if (chunks.length === 0 && content.length > 50) {
    const baseSlug = url.split("/").pop() || "article";
    const count = (seenSlugs.get(baseSlug) ?? 0) + 1;
    seenSlugs.set(baseSlug, count);
    const slug = count === 1 ? baseSlug : `${baseSlug}-${count}`;
    chunks.push({
      id: `${url}#${slug}`,
      content,
      url: `${url}#${slug}`,
      heading: "Overview",
      title,
      type,
    });
  }

  return chunks;
}

/**
 * Maps every allowed hostname to its canonical HTTPS origin.
 * The outgoing request origin is always picked from this map — never
 * constructed directly from user-supplied input — to prevent SSRF.
 * This is also the source of truth for the hostname allowlist.
 */
const HOSTNAME_TO_ORIGIN: Record<string, string> = {
  "dub.co": "https://dub.co",
  "www.dub.co": "https://www.dub.co",
};
const ALLOWED_HOSTNAMES = Object.keys(HOSTNAME_TO_ORIGIN);
const ALLOWED_PATH_PREFIXES = ["/docs", "/help"];

/**
 * Sanitize pathname: keep only alphanumeric, hyphens, underscores, slashes,
 * and dots, then collapse any remaining ".." sequences to prevent path
 * traversal even if the caller skips the pre-validation step.
 */
function sanitizePathname(pathname: string): string {
  return pathname.replace(/[^a-zA-Z0-9\-_/.]/g, "").replace(/\.\.+/g, "");
}

/**
 * Fetch, clean, chunk, and upsert a single article URL into Upstash Vector.
 * Uses heading-level chunks directly — no sentence-level fragmentation.
 * Validates URL to restrict fetches to dub.co docs/help (SSRF guard).
 *
 * @param pageviewsMap
 *
 *
 */
export async function upsertDocsEmbeddings(
  url: string,
  pageviewsMap?: Map<string, number>,
): Promise<{ chunks: number; skipped?: boolean }> {
  let parsedUrl: URL;
  try {
    parsedUrl = new URL(url);
  } catch {
    console.warn(`Skipping (invalid URL): ${url}`);
    return { chunks: 0, skipped: true };
  }

  if (
    parsedUrl.protocol !== "https:" ||
    !ALLOWED_HOSTNAMES.includes(parsedUrl.hostname) ||
    parsedUrl.pathname.includes("..") ||
    !ALLOWED_PATH_PREFIXES.some((p) => parsedUrl.pathname.startsWith(p))
  ) {
    console.warn(`Skipping (disallowed URL): ${url}`);
    return { chunks: 0, skipped: true };
  }

  const origin = HOSTNAME_TO_ORIGIN[parsedUrl.hostname] ?? "https://dub.co";
  const sanitizedPath = sanitizePathname(parsedUrl.pathname);
  const pathnameWithMd = sanitizedPath.endsWith(".md")
    ? sanitizedPath
    : `${sanitizedPath}.md`;
  const normalizedUrl = `${origin}${sanitizedPath}`;
  const mdUrl = `${origin}${pathnameWithMd}`;

  const res = await fetch(mdUrl);
  if (!res.ok) {
    console.warn(`Skipping (${res.status}): ${normalizedUrl}`);
    return { chunks: 0, skipped: true };
  }

  const raw = await res.text();
  const { content: cleaned, title } = await cleanMdx(raw);
  const chunks = chunkByHeadings(cleaned, normalizedUrl, title);

  // Look up pageviews for this article (keyed by pathname, not full URL)
  const pathname = new URL(normalizedUrl).pathname;
  const pageviews = pageviewsMap?.get(pathname) ?? 0;

  // Upstash Vector has a 48KB metadata size limit per vector.
  const MAX_METADATA_CONTENT = 4000;

  await Promise.all(
    chunks.map(async (chunk) => {
      await vectorIndex.upsert([
        {
          id: chunk.id,
          data: chunk.content,
          metadata: {
            url: chunk.url,
            heading: chunk.heading,
            title: chunk.title || chunk.heading,
            type: chunk.type,
            pageviews,
            content: chunk.content.slice(0, MAX_METADATA_CONTENT),
          },
        },
      ]);
    }),
  );

  return { chunks: chunks.length };
}