Skip to content

⚡️⚡️⚡️ Self-hosted collaborative bookmark manager to collect, read, annotate, and fully preserve what matters, all in one place.

10.3 KB
Raw Blame
import { prisma } from "@linkwarden/prisma";
import { createFolder } from "@linkwarden/filesystem";
import { JSDOM } from "jsdom";
import { decodeHTML } from "entities";
import { parse, Node, Element, TextNode } from "himalaya";
import { hasPassedLimit } from "@linkwarden/lib/verifyCapacity";

export default async function importFromHTMLFile(
  userId: number,
  rawData: string
) {
  // const importStartMs = Date.now();

  const dom = new JSDOM(rawData);
  const document = dom.window.document;

  // remove bad tags
  document.querySelectorAll("meta").forEach((e) => (e.outerHTML = e.innerHTML));
  document.querySelectorAll("META").forEach((e) => (e.outerHTML = e.innerHTML));
  document.querySelectorAll("P").forEach((e) => (e.outerHTML = e.innerHTML));

  const bookmarks = document.querySelectorAll("A");
  const totalImports = bookmarks.length;

  const hasTooManyLinks = await hasPassedLimit(userId, totalImports);

  if (hasTooManyLinks) {
    return {
      response: `Your subscription has reached the maximum number of links allowed.`,
      status: 400,
    };
  }

  const jsonData = parse(document.documentElement.outerHTML);

  const processedArray = processNodes(jsonData);

  // sortBookmarksTreeByEffectiveDate(processedArray, importStartMs, "asc");

  for (const item of processedArray) {
    await processBookmarks(userId, item as Element);
  }

  return { response: "Success.", status: 200 };
}

async function processBookmarks(
  userId: number,
  data: Node,
  parentCollectionId?: number
) {
  if (data.type === "element") {
    for (const item of data.children) {
      if (item.type === "element" && item.tagName === "dt") {
        // process collection or sub-collection
        let collectionId;
        const collectionName = item.children.find(
          (e) => e.type === "element" && e.tagName === "h3"
        ) as Element;

        if (collectionName) {
          const collectionNameContent = (collectionName.children[0] as TextNode)
            ?.content;
          if (collectionNameContent) {
            collectionId = await createCollection(
              userId,
              collectionNameContent,
              parentCollectionId
            );
          } else {
            // Handle the case when the collection name is empty
            collectionId = await createCollection(
              userId,
              "Untitled Collection",
              parentCollectionId
            );
          }
        }

        await processBookmarks(
          userId,
          item,
          collectionId || parentCollectionId
        );
      } else if (item.type === "element" && item.tagName === "a") {
        // process link
        const rawLinkUrl = item?.attributes.find(
          (e) => e.key.toLowerCase() === "href"
        )?.value;

        const linkUrl = decodeEntities(rawLinkUrl);

        const linkName = (
          item?.children.find((e) => e.type === "text") as TextNode
        )?.content;

        const linkTags = item?.attributes
          .find((e) => e.key === "tags")
          ?.value.split(",")
          .map(decodeEntities);

        // set date if available
        const linkDateValue = item?.attributes.find(
          (e) => e.key.toLowerCase() === "add_date"
        )?.value;

        const linkDate = linkDateValue
          ? new Date(Number(linkDateValue) * 1000)
          : undefined;

        let linkDesc =
          (
            (
              item?.children?.find(
                (e) => e.type === "element" && e.tagName === "dd"
              ) as Element
            )?.children[0] as TextNode
          )?.content || "";

        if (linkUrl && parentCollectionId) {
          await createLink(
            userId,
            linkUrl,
            parentCollectionId,
            linkName,
            linkDesc,
            linkTags,
            linkDate
          );
        } else if (linkUrl) {
          // create a collection named "Imports" and add the link to it
          const collectionId = await createCollection(userId, "Imports");

          await createLink(
            userId,
            linkUrl,
            collectionId,
            linkName,
            linkDesc,
            linkTags,
            linkDate
          );
        }

        await processBookmarks(userId, item, parentCollectionId);
      } else {
        await processBookmarks(userId, item, parentCollectionId);
      }
    }
  }
}

const createCollection = async (
  userId: number,
  collectionName: string,
  parentId?: number
) => {
  collectionName = collectionName.trim().slice(0, 254);

  const findCollection = await prisma.collection.findFirst({
    where: {
      parentId,
      name: collectionName,
      ownerId: userId,
    },
  });

  if (findCollection) {
    return findCollection.id;
  }

  const collectionId = await prisma.collection.create({
    data: {
      name: collectionName,
      parent: parentId
        ? {
            connect: {
              id: parentId,
            },
          }
        : undefined,
      owner: {
        connect: {
          id: userId,
        },
      },
      createdBy: {
        connect: {
          id: userId,
        },
      },
    },
  });

  createFolder({ filePath: `archives/${collectionId.id}` });

  return collectionId.id;
};

const createLink = async (
  userId: number,
  url: string,
  collectionId: number,
  name?: string,
  description?: string,
  tags?: string[],
  importDate?: Date
) => {
  url = url.trim().slice(0, 2047);
  try {
    new URL(url);
  } catch (e) {
    return;
  }

  tags = tags?.map((tag) => tag.trim().slice(0, 49));
  name = name?.trim().slice(0, 254);
  description = description?.trim().slice(0, 254);

  if (importDate) {
    const dateString = importDate.toISOString();
    if (dateString.length > 50) {
      importDate = undefined;
    }
  }

  await prisma.link.create({
    data: {
      name: name || "",
      url,
      description,
      collectionId,
      createdById: userId,
      tags:
        tags && tags[0]
          ? {
              connectOrCreate: tags.map((tag: string) => {
                return {
                  where: {
                    name_ownerId: {
                      name: tag.trim(),
                      ownerId: userId,
                    },
                  },
                  create: {
                    name: tag.trim(),
                    owner: {
                      connect: {
                        id: userId,
                      },
                    },
                  },
                };
              }),
            }
          : undefined,
      importDate: importDate || undefined,
    },
  });
};

function processNodes(nodes: Node[]) {
  const findAndProcessDL = (node: Node) => {
    if (node.type === "element" && node.tagName === "dl") {
      processDLChildren(node);
    } else if (
      node.type === "element" &&
      node.children &&
      node.children.length
    ) {
      node.children.forEach((child) => findAndProcessDL(child));
    }
  };

  const processDLChildren = (dlNode: Element) => {
    dlNode.children.forEach((child, i) => {
      if (child.type === "element" && child.tagName === "dt") {
        const nextSibling = dlNode.children[i + 1];
        if (
          nextSibling &&
          nextSibling.type === "element" &&
          nextSibling.tagName === "dd"
        ) {
          const aElement = child.children.find(
            (el) => el.type === "element" && el.tagName === "a"
          );
          if (aElement && aElement.type === "element") {
            // Add the 'dd' element as a child of the 'a' element
            aElement.children.push(nextSibling);
            // Remove the 'dd' from the parent 'dl' to avoid duplicate processing
            dlNode.children.splice(i + 1, 1);
          }
        }
      }
    });
  };

  nodes.forEach(findAndProcessDL);
  return nodes;
}

function decodeEntities(encoded: string | undefined): string {
  return decodeHTML(encoded ?? "");
}

/**
 * Sort <DT> entries inside each <DL> by "effective date":
 * - If an entry has ADD_DATE (on <A> or <H3>), use that.
 * - Otherwise, use importStartMs as the createdAt fallback for ordering.
 *
 * This ensures auto-increment IDs are created in the same chronological order
 * as (importDate ?? createdAt), satisfying your test.
 */
// function sortBookmarksTreeByEffectiveDate(
//   nodes: Node[],
//   importStartMs: number,
//   direction: "asc" | "desc" = "asc"
// ) {
//   const dir = direction === "asc" ? 1 : -1;

//   const getAttrCaseInsensitive = (el: Element, key: string) =>
//     el.attributes?.find((a) => a.key.toLowerCase() === key.toLowerCase())
//       ?.value;

//   const getAddDateMsFromDT = (dt: Element): number | null => {
//     // Links: <A ADD_DATE="...">
//     const aEl = dt.children.find(
//       (c) => c.type === "element" && c.tagName === "a"
//     ) as Element | undefined;

//     // Folders: <H3 ADD_DATE="...">
//     const h3El = dt.children.find(
//       (c) => c.type === "element" && c.tagName === "h3"
//     ) as Element | undefined;

//     const raw =
//       (aEl && getAttrCaseInsensitive(aEl, "add_date")) ||
//       (h3El && getAttrCaseInsensitive(h3El, "add_date"));

//     if (!raw) return null;

//     const seconds = Number(raw);
//     if (!Number.isFinite(seconds)) return null;

//     return seconds * 1000;
//   };

//   const stableSort = <T>(arr: T[], cmp: (a: T, b: T) => number): T[] =>
//     arr
//       .map((v, i) => ({ v, i }))
//       .sort((a, b) => cmp(a.v, b.v) || a.i - b.i)
//       .map((x) => x.v);

//   const sortDLChildren = (dl: Element) => {
//     const dtChildren: Element[] = [];
//     const otherChildren: Node[] = [];

//     for (const child of dl.children) {
//       if (child.type === "element" && child.tagName === "dt") {
//         dtChildren.push(child as Element);
//       } else {
//         otherChildren.push(child);
//       }
//     }

//     const sortedDTs = stableSort(dtChildren, (a, b) => {
//       const aMs = getAddDateMsFromDT(a) ?? importStartMs;
//       const bMs = getAddDateMsFromDT(b) ?? importStartMs;
//       return (aMs - bMs) * dir;
//     });

//     dl.children = [...sortedDTs, ...otherChildren];
//   };

//   const walk = (node: Node) => {
//     if (node.type !== "element") return;

//     if (node.tagName === "dl") {
//       sortDLChildren(node);
//     }

//     if (node.children?.length) {
//       node.children.forEach(walk);
//     }
//   };

//   nodes.forEach(walk);
// }