Skip to content

armgit / dub

Public Imported from GitHub
0

The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.

10.5 KB
Raw Blame
import { createId } from "@/lib/api/create-id";
import { addDomainToVercel } from "@/lib/api/domains/add-domain-vercel";
import { isValidDomainFormat } from "@/lib/api/domains/is-valid-domain";
import { handleAndReturnErrorResponse } from "@/lib/api/errors";
import { bulkCreateLinks, createLink, processLink } from "@/lib/api/links";
import { qstash } from "@/lib/cron";
import { verifyQstashSignature } from "@/lib/cron/verify-qstash";
import { prisma } from "@/lib/prisma";
import { ProcessedLinkProps } from "@/lib/types";
import { redis } from "@/lib/upstash";
import { linkMappingSchema } from "@/lib/zod/schemas/import-csv";
import { createLinkBodySchema } from "@/lib/zod/schemas/links";
import { randomBadgeColor } from "@/ui/links/tag-badge";
import {
  APP_DOMAIN_WITH_NGROK,
  DEFAULT_LINK_PROPS,
  DUB_DOMAINS_ARRAY,
  getUrlObjFromString,
  linkConstructorSimple,
  log,
  normalizeString,
  parseDateTime,
} from "@dub/utils";
import { NextResponse } from "next/server";
import * as z from "zod/v4";
import { sendCsvImportEmails } from "./utils";

export const dynamic = "force-dynamic";

const payloadSchema = z.object({
  workspaceId: z.string(),
  userId: z.string(),
  id: z.string(),
  folderId: z.string().nullable(),
  mapping: linkMappingSchema,
});

interface MapperResult {
  success: boolean;
  error?: string;
  data?: {
    domain: string;
    key: string;
    url: string;
    title?: string;
    description?: string;
    tags?: string[];
    createdAt?: Date;
  };
}

interface ErrorLink {
  domain: string;
  key: string;
  error: string;
}

// POST /api/cron/import/csv
export async function POST(req: Request) {
  try {
    const rawBody = await req.text();

    await verifyQstashSignature({
      req,
      rawBody,
    });

    const payload = payloadSchema.parse(JSON.parse(rawBody));
    const { workspaceId, id } = payload;
    const redisKey = `import:csv:${workspaceId}:${id}`;
    const BATCH_SIZE = 100;

    const rows = await redis.lpop<Record<string, string>[]>(
      `${redisKey}:rows`,
      BATCH_SIZE,
    );

    if (rows && rows.length > 0) {
      const mappedLinks: MapperResult[] = rows.map((row) =>
        mapCsvRowToLink(row, payload.mapping),
      );

      await processMappedLinks({
        mappedLinks,
        payload,
      });

      await redis.incrby(`${redisKey}:processed`, rows.length);

      if (rows.length === BATCH_SIZE) {
        const response = await qstash.publishJSON({
          url: `${APP_DOMAIN_WITH_NGROK}/api/cron/import/csv`,
          body: payload,
        });
        return NextResponse.json(response);
      }
    }

    // Finished processing all rows
    const errorLinks = await redis.lrange<ErrorLink>(
      `${redisKey}:failed`,
      0,
      -1,
    );
    const createdCount = parseInt(
      (await redis.get(`${redisKey}:created`)) || "0",
    );
    const domains = await redis.smembers(`${redisKey}:domains`);

    await sendCsvImportEmails({
      workspaceId,
      count: createdCount,
      domains,
      errorLinks,
    });

    await Promise.allSettled([
      redis.del(`${redisKey}:created`),
      redis.del(`${redisKey}:failed`),
      redis.del(`${redisKey}:domains`),
      redis.del(`${redisKey}:rows`),
      redis.del(`${redisKey}:processed`),
    ]);

    return NextResponse.json("OK");
  } catch (error) {
    await log({
      message: `Error importing CSV links: ${error.message}`,
      type: "cron",
    });

    return handleAndReturnErrorResponse(error);
  }
}

// Map a CSV row to a link
const mapCsvRowToLink = (
  row: Record<string, string>,
  mapping: z.infer<typeof linkMappingSchema>,
): MapperResult => {
  try {
    // Helper function to get value from CSV row using case-insensitive matching
    const getValueByKey = (targetKey: string) => {
      const key = Object.keys(row).find(
        (k) => normalizeString(k) === normalizeString(targetKey),
      );

      return key ? row[key].trim() : "";
    };

    const linkValue = getValueByKey(mapping.link);
    const urlValue = getValueByKey(mapping.url);

    if (!linkValue) {
      return {
        success: false,
        error: "Missing required field: link",
      };
    }

    if (!urlValue) {
      return {
        success: false,
        error: "Missing required field: url",
      };
    }

    const linkObj = getUrlObjFromString(linkValue);

    if (!linkObj) {
      return {
        success: false,
        error: `Invalid link format: ${linkValue}`,
      };
    }
    const domain = linkObj.hostname;

    if (!isValidDomainFormat(domain)) {
      return {
        success: false,
        error: `Invalid domain: ${domain}`,
      };
    }

    const key = linkObj.pathname.slice(1) || "_root";

    let urlObj: URL;
    try {
      urlObj = new URL(urlValue);
    } catch {
      return {
        success: false,
        error: `Invalid URL format: ${urlValue}`,
      };
    }

    const link: MapperResult["data"] = {
      domain,
      key,
      url: urlObj.toString(),
    };

    if (mapping.title) {
      const title = getValueByKey(mapping.title);

      if (title) {
        link.title = title;
      }
    }

    if (mapping.description) {
      const description = getValueByKey(mapping.description);

      if (description) {
        link.description = description;
      }
    }

    if (mapping.createdAt) {
      const createdAt = getValueByKey(mapping.createdAt);

      if (createdAt) {
        const date = parseDateTime(createdAt);

        if (date) {
          link.createdAt = date;
        }
      }
    }

    if (mapping.tags) {
      const tags = getValueByKey(mapping.tags);

      if (tags) {
        link.tags = tags
          .split(",")
          .map((tag) => tag.trim())
          .filter(Boolean)
          .map((tag) => normalizeString(tag));
      }
    }

    return {
      success: true,
      data: link,
    };
  } catch (error) {
    return {
      success: false,
      error: error instanceof Error ? error.message : "Unknown error occurred",
    };
  }
};

// Process the mapped links and create the tag/domain/link in the database
const processMappedLinks = async ({
  mappedLinks,
  payload,
}: {
  mappedLinks: MapperResult[];
  payload: z.infer<typeof payloadSchema>;
}) => {
  const { workspaceId, userId, folderId } = payload;
  const redisKey = `import:csv:${workspaceId}:${payload.id}`;

  if (mappedLinks.length === 0) {
    console.log("No links to process.");
    return;
  }

  const successfulMappings = mappedLinks.filter(
    (
      result,
    ): result is { success: true; data: NonNullable<MapperResult["data"]> } =>
      result.success && !!result.data,
  );

  // Process the tags
  let selectedTags = successfulMappings
    .map((result) => result.data.tags || [])
    .flat()
    .filter((tag): tag is string => Boolean(tag));

  selectedTags = [...new Set(selectedTags)];

  const tags = await prisma.tag.findMany({
    where: {
      projectId: workspaceId,
    },
    select: {
      id: true,
      name: true,
    },
  });

  const tagsNotInWorkspace = selectedTags.filter(
    (tag) => !tags.some((t) => t.name.toLowerCase() === tag.toLowerCase()),
  );

  if (tagsNotInWorkspace.length > 0) {
    console.log(`Creating ${tagsNotInWorkspace.length} new tags.`);

    await prisma.tag.createMany({
      data: tagsNotInWorkspace.map((name) => ({
        id: createId({ prefix: "tag_" }),
        projectId: workspaceId,
        name,
        color: randomBadgeColor(),
      })),
      skipDuplicates: true,
    });
  }

  // Process the domains
  let selectedDomains = successfulMappings
    .map((result) => result.data.domain)
    .filter((domain): domain is string => Boolean(domain));

  selectedDomains = [...new Set(selectedDomains)];

  const domains = await prisma.domain.findMany({
    where: {
      projectId: workspaceId,
    },
  });

  const domainsNotInWorkspace = selectedDomains.filter(
    (domain) =>
      !domains.some((d) => d.slug === domain) &&
      !DUB_DOMAINS_ARRAY.includes(domain),
  );

  if (domainsNotInWorkspace.length > 0) {
    console.log(`Creating ${domainsNotInWorkspace.length} new domains.`);

    await Promise.allSettled([
      prisma.domain.createMany({
        data: domainsNotInWorkspace.map((slug) => ({
          id: createId({ prefix: "dom_" }),
          projectId: workspaceId,
          slug,
          primary: false,
        })),
        skipDuplicates: true,
      }),

      domainsNotInWorkspace.map((domain) => addDomainToVercel(domain)),

      domainsNotInWorkspace.map((domain) =>
        createLink({
          ...DEFAULT_LINK_PROPS,
          projectId: workspaceId,
          userId,
          domain,
          key: "_root",
          url: "",
          tags: undefined,
        }),
      ),
    ]);
  }

  if (selectedDomains.length > 0) {
    await redis.sadd(`${redisKey}:domains`, ...(selectedDomains as [string]));
  }

  // Process the links
  let linksToCreate = successfulMappings.map((result) => result.data);

  const existingLinks = await prisma.link.findMany({
    where: {
      projectId: workspaceId,
      shortLink: {
        in: linksToCreate.map((link) => linkConstructorSimple(link)),
      },
    },
    select: {
      shortLink: true,
    },
  });

  console.log(`Skipping ${existingLinks.length} existing links.`);

  linksToCreate = linksToCreate.filter(
    (link) =>
      !existingLinks.some((l) => l.shortLink === linkConstructorSimple(link)),
  );

  const workspace = await prisma.project.findUniqueOrThrow({
    where: {
      id: workspaceId,
    },
    select: {
      id: true,
      plan: true,
      users: {
        where: {
          userId,
        },
      },
    },
  });

  const processedLinks = await Promise.all(
    linksToCreate.map(({ tags, ...link }) =>
      processLink({
        payload: {
          ...createLinkBodySchema.parse({
            ...link,
            tagNames: tags || [],
            folderId,
          }),
        },
        workspace,
        userId,
        bulk: true,
      }),
    ),
  );

  const validLinks = processedLinks
    .filter(({ error }) => error == null)
    .map(({ link }) => link);

  const errorLinks = processedLinks
    .filter(({ error }) => error != null)
    .map(({ link: { domain, key }, error }) => ({
      domain,
      key,
      error,
    }));

  if (validLinks.length > 0) {
    console.log(`Creating ${validLinks.length} new links.`);

    await bulkCreateLinks({
      links: validLinks as ProcessedLinkProps[],
      skipRedisCache: true,
    });

    await redis.incrby(`${redisKey}:created`, validLinks.length);
  }

  if (errorLinks.length > 0) {
    console.log(`${errorLinks.length} failed to create.`);

    await redis.rpush(`${redisKey}:failed`, ...errorLinks);
  }
};