⚡️⚡️⚡️ Self-hosted collaborative bookmark manager to collect, read, annotate, and fully preserve what matters, all in one place.
TypeScript bookmarkbookmark-managercollaborationnextjsreact-nativeread-it-laterself-hostedtypescript
finalized archiveHandler/background worker logic
daniel31x13b74ff01ce6fd
diff --git a/.env.sample b/.env.sampleindex 70576b37..c3174e68 100644--- a/.env.sample+++ b/.env.sample@@ -15,7 +15,7 @@ NEXT_PUBLIC_DISABLE_REGISTRATION=NEXT_PUBLIC_CREDENTIALS_ENABLED=DISABLE_NEW_SSO_USERS=RE_ARCHIVE_LIMIT=-NEXT_PUBLIC_MAX_UPLOAD_SIZE=+NEXT_PUBLIC_MAX_FILE_SIZE=# AWS S3 SettingsSPACES_KEY=diff --git a/components/ModalContent/UploadFileModal.tsx b/components/ModalContent/UploadFileModal.tsxindex f6e7af4b..a2a128bd 100644--- a/components/ModalContent/UploadFileModal.tsx+++ b/components/ModalContent/UploadFileModal.tsx@@ -174,7 +174,7 @@ export default function UploadFileModal({ onClose }: Props) {/></label><p className="text-xs font-semibold mt-2">- PDF, PNG, JPG (Up to {process.env.NEXT_PUBLIC_MAX_UPLOAD_SIZE || 30}+ PDF, PNG, JPG (Up to {process.env.NEXT_PUBLIC_MAX_FILE_SIZE || 30}MB)</p></div>diff --git a/lib/api/archiveHandler.ts b/lib/api/archiveHandler.tsnew file mode 100644index 00000000..9ae230fd--- /dev/null+++ b/lib/api/archiveHandler.ts@@ -0,0 +1,272 @@+import { chromium, devices } from "playwright";+import { prisma } from "./db";+import createFile from "./storage/createFile";+import sendToWayback from "./sendToWayback";+import { Readability } from "@mozilla/readability";+import { JSDOM } from "jsdom";+import DOMPurify from "dompurify";+import { Collection, Link, User } from "@prisma/client";+import validateUrlSize from "./validateUrlSize";+import {+ pdfAvailable,+ readabilityAvailable,+ screenshotAvailable,+} from "../shared/getArchiveValidity";++type LinksAndCollectionAndOwner = Link & {+ collection: Collection & {+ owner: User;+ };+};++export default async function archiveHandler(link: LinksAndCollectionAndOwner) {+ const browser = await chromium.launch();+ const context = await browser.newContext(devices["Desktop Chrome"]);+ const page = await context.newPage();++ try {+ const validatedUrl = link.url ? await validateUrlSize(link.url) : undefined;++ if (validatedUrl === null) throw "File is too large to be stored.";++ const contentType = validatedUrl?.get("content-type");+ let linkType = "url";+ let imageExtension = "png";++ if (!link.url) linkType = link.type;+ else if (contentType === "application/pdf") linkType = "pdf";+ else if (contentType?.startsWith("image")) {+ linkType = "image";+ if (contentType === "image/jpeg") imageExtension = "jpeg";+ else if (contentType === "image/png") imageExtension = "png";+ }++ const user = link.collection?.owner;++ // send to archive.org+ if (user.archiveAsWaybackMachine && link.url) sendToWayback(link.url);++ const targetLink = await prisma.link.update({+ where: { id: link.id },+ data: {+ type: linkType,+ screenshotPath: user.archiveAsScreenshot ? "pending" : undefined,+ pdfPath: user.archiveAsPDF ? "pending" : undefined,+ readabilityPath: "pending",+ lastPreserved: new Date().toISOString(),+ },+ });++ if (linkType === "image") {+ await imageHandler(link, imageExtension); // archive image (jpeg/png)+ return;+ } else if (linkType === "pdf") {+ await pdfHandler(link); // archive pdf+ return;+ } else if (user.archiveAsPDF || user.archiveAsScreenshot) {+ // archive url+ link.url &&+ (await page.goto(link.url, { waitUntil: "domcontentloaded" }));++ const content = await page.content();++ // TODO Webarchive+ // const session = await page.context().newCDPSession(page);+ // const doc = await session.send("Page.captureSnapshot", {+ // format: "mhtml",+ // });+ // const saveDocLocally = (doc: any) => {+ // console.log(doc);+ // return createFile({+ // data: doc,+ // filePath: `archives/${targetLink.collectionId}/${link.id}.mhtml`,+ // });+ // };+ // saveDocLocally(doc.data);++ // Readability+ const window = new JSDOM("").window;+ const purify = DOMPurify(window);+ const cleanedUpContent = purify.sanitize(content);+ const dom = new JSDOM(cleanedUpContent, { url: link.url || "" });+ const article = new Readability(dom.window.document).parse();+ const articleText = article?.textContent+ .replace(/ +(?= )/g, "") // strip out multiple spaces+ .replace(/(\r\n|\n|\r)/gm, " "); // strip out line breaks+ if (articleText && articleText !== "") {+ await createFile({+ data: JSON.stringify(article),+ filePath: `archives/${targetLink.collectionId}/${link.id}_readability.json`,+ });++ await prisma.link.update({+ where: { id: link.id },+ data: {+ readabilityPath: `archives/${targetLink.collectionId}/${link.id}_readability.json`,+ textContent: articleText,+ },+ });+ }++ // Screenshot/PDF+ await page.evaluate(+ autoScroll,+ Number(process.env.AUTOSCROLL_TIMEOUT) || 30+ );+ // Check if the user hasn't deleted the link by the time we're done scrolling+ const linkExists = await prisma.link.findUnique({+ where: { id: link.id },+ });+ if (linkExists) {+ const processingPromises = [];++ if (user.archiveAsScreenshot) {+ processingPromises.push(+ page.screenshot({ fullPage: true }).then((screenshot) => {+ return createFile({+ data: screenshot,+ filePath: `archives/${linkExists.collectionId}/${link.id}.png`,+ });+ })+ );+ }+ if (user.archiveAsPDF) {+ processingPromises.push(+ page+ .pdf({+ width: "1366px",+ height: "1931px",+ printBackground: true,+ margin: { top: "15px", bottom: "15px" },+ })+ .then((pdf) => {+ return createFile({+ data: pdf,+ filePath: `archives/${linkExists.collectionId}/${link.id}.pdf`,+ });+ })+ );+ }+ await Promise.allSettled(processingPromises);+ await prisma.link.update({+ where: { id: link.id },+ data: {+ screenshotPath: user.archiveAsScreenshot+ ? `archives/${linkExists.collectionId}/${link.id}.png`+ : undefined,+ pdfPath: user.archiveAsPDF+ ? `archives/${linkExists.collectionId}/${link.id}.pdf`+ : undefined,+ },+ });+ }+ }+ } catch (err) {+ console.log(err);+ console.log("Failed Link details:", link);+ throw err;+ } finally {+ const finalLink = await prisma.link.findUnique({+ where: { id: link.id },+ });++ if (finalLink)+ await prisma.link.update({+ where: { id: link.id },+ data: {+ readabilityPath:+ !finalLink.textContent ||+ finalLink.textContent === "" ||+ !readabilityAvailable(finalLink) ||+ finalLink.type !== "url"+ ? "unavailable"+ : undefined,+ screenshotPath:+ !screenshotAvailable(finalLink) ||+ (finalLink.type !== "url" && finalLink.type !== "pdf")+ ? "unavailable"+ : undefined,+ pdfPath:+ !pdfAvailable(finalLink) ||+ (finalLink.type !== "url" && finalLink.type !== "image")+ ? "unavailable"+ : undefined,+ },+ });++ await browser.close();+ }+}++const autoScroll = async (AUTOSCROLL_TIMEOUT: number) => {+ const timeoutPromise = new Promise<void>((_, reject) => {+ setTimeout(() => {+ reject(new Error(`Webpage was too long to be archived.`));+ }, AUTOSCROLL_TIMEOUT * 1000);+ });++ const scrollingPromise = new Promise<void>((resolve) => {+ let totalHeight = 0;+ let distance = 100;+ let scrollDown = setInterval(() => {+ let scrollHeight = document.body.scrollHeight;+ window.scrollBy(0, distance);+ totalHeight += distance;+ if (totalHeight >= scrollHeight) {+ clearInterval(scrollDown);+ window.scroll(0, 0);+ resolve();+ }+ }, 100);+ });++ await Promise.race([scrollingPromise, timeoutPromise]);+};++const imageHandler = async ({ url, id }: Link, extension: string) => {+ const image = await fetch(url as string).then((res) => res.blob());++ const buffer = Buffer.from(await image.arrayBuffer());++ const linkExists = await prisma.link.findUnique({+ where: { id },+ });++ if (linkExists) {+ await createFile({+ data: buffer,+ filePath: `archives/${linkExists.collectionId}/${id}.${extension}`,+ });++ await prisma.link.update({+ where: { id },+ data: {+ screenshotPath: `archives/${linkExists.collectionId}/${id}.${extension}`,+ },+ });+ }+};++const pdfHandler = async ({ url, id }: Link) => {+ const pdf = await fetch(url as string).then((res) => res.blob());++ const buffer = Buffer.from(await pdf.arrayBuffer());++ const linkExists = await prisma.link.findUnique({+ where: { id },+ });++ if (linkExists) {+ await createFile({+ data: buffer,+ filePath: `archives/${linkExists.collectionId}/${id}.pdf`,+ });++ await prisma.link.update({+ where: { id },+ data: {+ pdfPath: `archives/${linkExists.collectionId}/${id}.pdf`,+ },+ });+ }+};diff --git a/lib/api/controllers/links/postLink.ts b/lib/api/controllers/links/postLink.tsindex eca8a044..912e972f 100644--- a/lib/api/controllers/links/postLink.ts+++ b/lib/api/controllers/links/postLink.ts@@ -1,13 +1,10 @@import { prisma } from "@/lib/api/db";import { LinkIncludingShortenedCollectionAndTags } from "@/types/global";import getTitle from "@/lib/shared/getTitle";-import urlHandler from "@/lib/api/urlHandler";import { UsersAndCollections } from "@prisma/client";import getPermission from "@/lib/api/getPermission";import createFolder from "@/lib/api/storage/createFolder";-import pdfHandler from "../../pdfHandler";import validateUrlSize from "../../validateUrlSize";-import imageHandler from "../../imageHandler";export default async function postLink(link: LinkIncludingShortenedCollectionAndTags,@@ -113,37 +110,5 @@ export default async function postLink(createFolder({ filePath: `archives/${newLink.collectionId}` });- newLink.url && linkType === "url"- ? urlHandler(newLink.id, newLink.url, userId)- : undefined;-- newLink.url && linkType === "pdf"- ? pdfHandler(newLink.id, newLink.url)- : undefined;-- newLink.url && linkType === "image"- ? imageHandler(newLink.id, newLink.url, imageExtension)- : undefined;-- !newLink.url && linkType === "pdf"- ? await prisma.link.update({- where: { id: newLink.id },- data: {- pdfPath: "pending",- lastPreserved: new Date().toISOString(),- },- })- : undefined;-- !newLink.url && linkType === "image"- ? await prisma.link.update({- where: { id: newLink.id },- data: {- screenshotPath: "pending",- lastPreserved: new Date().toISOString(),- },- })- : undefined;-return { response: newLink, status: 200 };}diff --git a/lib/api/imageHandler.ts b/lib/api/imageHandler.tsindex b9437dd1..bff89c3e 100644--- a/lib/api/imageHandler.ts+++ b/lib/api/imageHandler.ts@@ -9,9 +9,9 @@ export default async function imageHandler(extension: string,file?: string) {- const pdf = await fetch(url as string).then((res) => res.blob());+ const image = await fetch(url as string).then((res) => res.blob());- const buffer = Buffer.from(await pdf.arrayBuffer());+ const buffer = Buffer.from(await image.arrayBuffer());const linkExists = await prisma.link.findUnique({where: { id: linkId },diff --git a/lib/api/urlHandler.ts b/lib/api/urlHandler.tsdeleted file mode 100644index 41d7441c..00000000--- a/lib/api/urlHandler.ts+++ /dev/null@@ -1,172 +0,0 @@-import { chromium, devices } from "playwright";-import { prisma } from "@/lib/api/db";-import createFile from "@/lib/api/storage/createFile";-import sendToWayback from "./sendToWayback";-import { Readability } from "@mozilla/readability";-import { JSDOM } from "jsdom";-import DOMPurify from "dompurify";--export default async function urlHandler(- linkId: number,- url: string,- userId: number-) {- const user = await prisma.user.findUnique({ where: { id: userId } });-- const targetLink = await prisma.link.update({- where: { id: linkId },- data: {- screenshotPath: user?.archiveAsScreenshot ? "pending" : null,- pdfPath: user?.archiveAsPDF ? "pending" : null,- readabilityPath: "pending",- lastPreserved: new Date().toISOString(),- },- });-- // archive.org-- if (user?.archiveAsWaybackMachine) sendToWayback(url);-- if (user?.archiveAsPDF || user?.archiveAsScreenshot) {- const browser = await chromium.launch();- const context = await browser.newContext(devices["Desktop Chrome"]);- const page = await context.newPage();-- try {- await page.goto(url, { waitUntil: "domcontentloaded" });-- const content = await page.content();-- // TODO- // const session = await page.context().newCDPSession(page);-- // const doc = await session.send("Page.captureSnapshot", {- // format: "mhtml",- // });-- // const saveDocLocally = (doc: any) => {- // console.log(doc);- // return createFile({- // data: doc,- // filePath: `archives/${targetLink.collectionId}/${linkId}.mhtml`,- // });- // };-- // saveDocLocally(doc.data);-- // Readability-- const window = new JSDOM("").window;- const purify = DOMPurify(window);- const cleanedUpContent = purify.sanitize(content);- const dom = new JSDOM(cleanedUpContent, { url: url });- const article = new Readability(dom.window.document).parse();-- const articleText = article?.textContent- .replace(/ +(?= )/g, "") // strip out multiple spaces- .replace(/(\r\n|\n|\r)/gm, " "); // strip out line breaks-- await createFile({- data: JSON.stringify(article),- filePath: `archives/${targetLink.collectionId}/${linkId}_readability.json`,- });-- await prisma.link.update({- where: { id: linkId },- data: {- readabilityPath: `archives/${targetLink.collectionId}/${linkId}_readability.json`,- textContent: articleText,- },- });-- // Screenshot/PDF-- let faulty = false;-- await page- .evaluate(autoScroll, Number(process.env.AUTOSCROLL_TIMEOUT) || 30)- .catch((err) => {- console.log(err);- faulty = true;- });-- const linkExists = await prisma.link.findUnique({- where: { id: linkId },- });-- if (linkExists && !faulty) {- if (user.archiveAsScreenshot) {- const screenshot = await page.screenshot({ fullPage: true });- await createFile({- data: screenshot,- filePath: `archives/${linkExists.collectionId}/${linkId}.png`,- });- }-- if (user.archiveAsPDF) {- const pdf = await page.pdf({- width: "1366px",- height: "1931px",- printBackground: true,- margin: { top: "15px", bottom: "15px" },- });-- await createFile({- data: pdf,- filePath: `archives/${linkExists.collectionId}/${linkId}.pdf`,- });- }-- await prisma.link.update({- where: { id: linkId },- data: {- screenshotPath: user.archiveAsScreenshot- ? `archives/${linkExists.collectionId}/${linkId}.png`- : null,- pdfPath: user.archiveAsPDF- ? `archives/${linkExists.collectionId}/${linkId}.pdf`- : null,- },- });- } else if (faulty) {- await prisma.link.update({- where: { id: linkId },- data: {- screenshotPath: null,- pdfPath: null,- },- });- }- } catch (err) {- console.log(err);- throw err;- } finally {- await browser.close();- }- }-}--const autoScroll = async (AUTOSCROLL_TIMEOUT: number) => {- const timeoutPromise = new Promise<void>((_, reject) => {- setTimeout(() => {- reject(new Error(`Webpage was too long to be archived.`));- }, AUTOSCROLL_TIMEOUT * 1000);- });-- const scrollingPromise = new Promise<void>((resolve) => {- let totalHeight = 0;- let distance = 100;- let scrollDown = setInterval(() => {- let scrollHeight = document.body.scrollHeight;- window.scrollBy(0, distance);- totalHeight += distance;- if (totalHeight >= scrollHeight) {- clearInterval(scrollDown);- window.scroll(0, 0);- resolve();- }- }, 100);- });-- await Promise.race([scrollingPromise, timeoutPromise]);-};diff --git a/lib/api/validateUrlSize.ts b/lib/api/validateUrlSize.tsindex cde7e338..02b5bcc3 100644--- a/lib/api/validateUrlSize.ts+++ b/lib/api/validateUrlSize.ts@@ -4,7 +4,8 @@ export default async function validateUrlSize(url: string) {const totalSizeMB =Number(response.headers.get("content-length")) / Math.pow(1024, 2);- if (totalSizeMB > 50) return null;+ if (totalSizeMB > (Number(process.env.NEXT_PUBLIC_MAX_FILE_SIZE) || 30))+ return null;else return response.headers;} catch (err) {console.log(err);diff --git a/lib/shared/getArchiveValidity.ts b/lib/shared/getArchiveValidity.tsindex 0650e7c9..d2146d3f 100644--- a/lib/shared/getArchiveValidity.ts+++ b/lib/shared/getArchiveValidity.ts@@ -1,11 +1,9 @@-import { Link } from "@prisma/client";-export function screenshotAvailable(link: any) {return (link &&link.screenshotPath &&link.screenshotPath !== "pending" &&- link.screenshotPath !== "failed"+ link.screenshotPath !== "unavailable");}@@ -14,7 +12,7 @@ export function pdfAvailable(link: any) {link &&link.pdfPath &&link.pdfPath !== "pending" &&- link.pdfPath !== "failed"+ link.pdfPath !== "unavailable");}@@ -23,6 +21,6 @@ export function readabilityAvailable(link: any) {link &&link.readabilityPath &&link.readabilityPath !== "pending" &&- link.readabilityPath !== "failed"+ link.readabilityPath !== "unavailable");}diff --git a/pages/api/v1/archives/[linkId].ts b/pages/api/v1/archives/[linkId].tsindex 8686c154..07760be7 100644--- a/pages/api/v1/archives/[linkId].ts+++ b/pages/api/v1/archives/[linkId].ts@@ -81,7 +81,7 @@ export default async function Index(req: NextApiRequest, res: NextApiResponse) {// // await uploadHandler(linkId, )- // const MAX_UPLOAD_SIZE = Number(process.env.NEXT_PUBLIC_MAX_UPLOAD_SIZE);+ // const MAX_UPLOAD_SIZE = Number(process.env.NEXT_PUBLIC_MAX_FILE_SIZE);// const form = formidable({// maxFields: 1,diff --git a/pages/api/v1/links/[id]/archive/index.ts b/pages/api/v1/links/[id]/archive/index.tsindex 6c82aede..fe8aaffa 100644--- a/pages/api/v1/links/[id]/archive/index.ts+++ b/pages/api/v1/links/[id]/archive/index.ts@@ -1,5 +1,5 @@import type { NextApiRequest, NextApiResponse } from "next";-import urlHandler from "@/lib/api/urlHandler";+import urlHandler from "@/lib/api/archiveHandler";import { prisma } from "@/lib/api/db";import verifyUser from "@/lib/api/verifyUser";import isValidUrl from "@/lib/shared/isValidUrl";diff --git a/scripts/lib/urlHandler.ts b/scripts/lib/urlHandler.tsdeleted file mode 100644index c9bdd854..00000000--- a/scripts/lib/urlHandler.ts+++ /dev/null@@ -1,176 +0,0 @@-import { chromium, devices } from "playwright";-import { prisma } from "../../lib/api/db";-import createFile from "../../lib/api/storage/createFile";-import sendToWayback from "../../lib/api/sendToWayback";-import { Readability } from "@mozilla/readability";-import { JSDOM } from "jsdom";-import DOMPurify from "dompurify";-import { Collection, Link, User } from "@prisma/client";--type LinksAndCollectionAndOwner = Link & {- collection: Collection & {- owner: User;- };-};--export default async function urlHandler(link: LinksAndCollectionAndOwner) {- const user = link.collection?.owner;-- const targetLink = await prisma.link.update({- where: { id: link.id },- data: {- screenshotPath: user.archiveAsScreenshot ? "pending" : null,- pdfPath: user.archiveAsPDF ? "pending" : null,- readabilityPath: "pending",- lastPreserved: new Date().toISOString(),- },- });-- // archive.org-- if (user.archiveAsWaybackMachine && link.url) sendToWayback(link.url);-- if (user.archiveAsPDF || user.archiveAsScreenshot) {- const browser = await chromium.launch();- const context = await browser.newContext(devices["Desktop Chrome"]);- const page = await context.newPage();-- try {- link.url &&- (await page.goto(link.url, { waitUntil: "domcontentloaded" }));-- const content = await page.content();-- // TODO- // const session = await page.context().newCDPSession(page);- // const doc = await session.send("Page.captureSnapshot", {- // format: "mhtml",- // });- // const saveDocLocally = (doc: any) => {- // console.log(doc);- // return createFile({- // data: doc,- // filePath: `archives/${targetLink.collectionId}/${link.id}.mhtml`,- // });- // };- // saveDocLocally(doc.data);-- // Readability-- const window = new JSDOM("").window;- const purify = DOMPurify(window);- const cleanedUpContent = purify.sanitize(content);- const dom = new JSDOM(cleanedUpContent, { url: link.url || "" });- const article = new Readability(dom.window.document).parse();-- const articleText = article?.textContent- .replace(/ +(?= )/g, "") // strip out multiple spaces- .replace(/(\r\n|\n|\r)/gm, " "); // strip out line breaks-- await createFile({- data: JSON.stringify(article),- filePath: `archives/${targetLink.collectionId}/${link.id}_readability.json`,- });-- await prisma.link.update({- where: { id: link.id },- data: {- readabilityPath: `archives/${targetLink.collectionId}/${link.id}_readability.json`,- textContent: articleText,- },- });-- // Screenshot/PDF-- let faulty = false;- await page- .evaluate(autoScroll, Number(process.env.AUTOSCROLL_TIMEOUT) || 30)- .catch((e) => (faulty = true));-- const linkExists = await prisma.link.findUnique({- where: { id: link.id },- });-- if (linkExists && !faulty) {- const processingPromises = [];-- if (user.archiveAsScreenshot) {- const screenshot = await page.screenshot({ fullPage: true });- processingPromises.push(- createFile({- data: screenshot,- filePath: `archives/${linkExists.collectionId}/${link.id}.png`,- })- );- }-- if (user.archiveAsPDF) {- const pdf = await page.pdf({- width: "1366px",- height: "1931px",- printBackground: true,- margin: { top: "15px", bottom: "15px" },- });- processingPromises.push(- createFile({- data: pdf,- filePath: `archives/${linkExists.collectionId}/${link.id}.pdf`,- })- );- }-- await Promise.allSettled(processingPromises);-- await prisma.link.update({- where: { id: link.id },- data: {- screenshotPath: user.archiveAsScreenshot- ? `archives/${linkExists.collectionId}/${link.id}.png`- : null,- pdfPath: user.archiveAsPDF- ? `archives/${linkExists.collectionId}/${link.id}.pdf`- : null,- },- });- } else if (faulty) {- await prisma.link.update({- where: { id: link.id },- data: {- screenshotPath: null,- pdfPath: null,- },- });- }- } catch (err) {- console.log(err);- throw err;- } finally {- await browser.close();- }- }-}--const autoScroll = async (AUTOSCROLL_TIMEOUT: number) => {- const timeoutPromise = new Promise<void>((_, reject) => {- setTimeout(() => {- reject(new Error(`Webpage was too long to be archived.`));- }, AUTOSCROLL_TIMEOUT * 1000);- });-- const scrollingPromise = new Promise<void>((resolve) => {- let totalHeight = 0;- let distance = 100;- let scrollDown = setInterval(() => {- let scrollHeight = document.body.scrollHeight;- window.scrollBy(0, distance);- totalHeight += distance;- if (totalHeight >= scrollHeight) {- clearInterval(scrollDown);- window.scroll(0, 0);- resolve();- }- }, 100);- });-- await Promise.race([scrollingPromise, timeoutPromise]);-};diff --git a/scripts/worker.ts b/scripts/worker.tsindex 213fa7cc..c5c3be81 100644--- a/scripts/worker.ts+++ b/scripts/worker.ts@@ -1,11 +1,9 @@import { Collection, Link, User } from "@prisma/client";import { prisma } from "../lib/api/db";-import urlHandler from "./lib/urlHandler";+import archiveHandler from "../lib/api/archiveHandler";const args = process.argv.slice(2).join(" ");-console.log(process.env.NEXTAUTH_URL);-const archiveTakeCount = Number(process.env.ARCHIVE_TAKE_COUNT || "") || 5;type LinksAndCollectionAndOwner = Link & {@@ -136,7 +134,7 @@ async function processBatch() {`Processing link ${link.url} for user ${link.collection.ownerId}`);- await urlHandler(link);+ await archiveHandler(link);console.log("\x1b[34m%s\x1b[0m",@@ -152,9 +150,12 @@ async function processBatch() {};// Process each link in the batch concurrently- const processingPromises = [...linksOldToNew, ...linksNewToOld].map((e) =>- archiveLink(e)- );+ const processingPromises = [...linksOldToNew, ...linksNewToOld]+ // Make sure we don't process the same link twice+ .filter((value, index, self) => {+ return self.findIndex((item) => item.id === value.id) === index;+ })+ .map((e) => archiveLink(e));await Promise.allSettled(processingPromises);}diff --git a/types/enviornment.d.ts b/types/enviornment.d.tsindex e4d39cf3..b34d57af 100644--- a/types/enviornment.d.ts+++ b/types/enviornment.d.ts@@ -9,7 +9,7 @@ declare global {STORAGE_FOLDER?: string;AUTOSCROLL_TIMEOUT?: string;RE_ARCHIVE_LIMIT?: string;- NEXT_PUBLIC_MAX_UPLOAD_SIZE?: string;+ NEXT_PUBLIC_MAX_FILE_SIZE?: string;SPACES_KEY?: string;SPACES_SECRET?: string;