⚡️⚡️⚡️ Self-hosted collaborative bookmark manager to collect, read, annotate, and fully preserve what matters, all in one place.
TypeScript bookmarkbookmark-managercollaborationnextjsreact-nativeread-it-laterself-hostedtypescript
throw error if a link was blocking the queue for too long
daniel31x13f1dadf1546bc
diff --git a/.env.sample b/.env.sampleindex 0edaa4bb..c93af452 100644--- a/.env.sample+++ b/.env.sample@@ -18,6 +18,7 @@ RE_ARCHIVE_LIMIT=NEXT_PUBLIC_MAX_FILE_SIZE=MAX_LINKS_PER_USER=ARCHIVE_TAKE_COUNT=+BROWSER_TIMEOUT=# AWS S3 SettingsSPACES_KEY=diff --git a/lib/api/archiveHandler.ts b/lib/api/archiveHandler.tsindex aa0eec02..f5be7521 100644--- a/lib/api/archiveHandler.ts+++ b/lib/api/archiveHandler.ts@@ -17,232 +17,260 @@ type LinksAndCollectionAndOwner = Link & {};};+const BROWSER_TIMEOUT = Number(process.env.BROWSER_TIMEOUT) || 5;+export default async function archiveHandler(link: LinksAndCollectionAndOwner) {- const browser = await chromium.launch();+ const browser = await chromium.launch({ headless: false });const context = await browser.newContext(devices["Desktop Chrome"]);const page = await context.newPage();- try {- const validatedUrl = link.url ? await validateUrlSize(link.url) : undefined;-- if (validatedUrl === null) throw "File is too large to be stored.";-- const contentType = validatedUrl?.get("content-type");- let linkType = "url";- let imageExtension = "png";-- if (!link.url) linkType = link.type;- else if (contentType === "application/pdf") linkType = "pdf";- else if (contentType?.startsWith("image")) {- linkType = "image";- if (contentType === "image/jpeg") imageExtension = "jpeg";- else if (contentType === "image/png") imageExtension = "png";- }-- const user = link.collection?.owner;+ const timeoutPromise = new Promise((_, reject) => {+ setTimeout(+ () =>+ reject(+ new Error(+ `Browser has been open for more than ${BROWSER_TIMEOUT} minutes.`+ )+ ),+ BROWSER_TIMEOUT * 60000+ );+ });- // send to archive.org- if (user.archiveAsWaybackMachine && link.url) sendToWayback(link.url);+ try {+ await Promise.race([+ (async () => {+ const validatedUrl = link.url+ ? await validateUrlSize(link.url)+ : undefined;++ if (validatedUrl === null) throw "File is too large to be stored.";++ const contentType = validatedUrl?.get("content-type");+ let linkType = "url";+ let imageExtension = "png";++ if (!link.url) linkType = link.type;+ else if (contentType === "application/pdf") linkType = "pdf";+ else if (contentType?.startsWith("image")) {+ linkType = "image";+ if (contentType === "image/jpeg") imageExtension = "jpeg";+ else if (contentType === "image/png") imageExtension = "png";+ }- const targetLink = await prisma.link.update({- where: { id: link.id },- data: {- type: linkType,- image:- user.archiveAsScreenshot && !link.image?.startsWith("archive")- ? "pending"- : undefined,- pdf:- user.archiveAsPDF && !link.pdf?.startsWith("archive")- ? "pending"- : undefined,- readable: !link.readable?.startsWith("archive") ? "pending" : undefined,- preview: !link.readable?.startsWith("archive") ? "pending" : undefined,- lastPreserved: new Date().toISOString(),- },- });+ const user = link.collection?.owner;- if (linkType === "image" && !link.image?.startsWith("archive")) {- await imageHandler(link, imageExtension); // archive image (jpeg/png)- return;- } else if (linkType === "pdf" && !link.pdf?.startsWith("archive")) {- await pdfHandler(link); // archive pdf- return;- } else if (link.url) {- // archive url-- await page.goto(link.url, { waitUntil: "domcontentloaded" });-- const content = await page.content();-- // TODO single file- // const session = await page.context().newCDPSession(page);- // const doc = await session.send("Page.captureSnapshot", {- // format: "mhtml",- // });- // const saveDocLocally = (doc: any) => {- // console.log(doc);- // return createFile({- // data: doc,- // filePath: `archives/${targetLink.collectionId}/${link.id}.mhtml`,- // });- // };- // saveDocLocally(doc.data);-- // Readability- const window = new JSDOM("").window;- const purify = DOMPurify(window);- const cleanedUpContent = purify.sanitize(content);- const dom = new JSDOM(cleanedUpContent, { url: link.url || "" });- const article = new Readability(dom.window.document).parse();- const articleText = article?.textContent- .replace(/ +(?= )/g, "") // strip out multiple spaces- .replace(/(\r\n|\n|\r)/gm, " "); // strip out line breaks- if (- articleText &&- articleText !== "" &&- !link.readable?.startsWith("archive")- ) {- await createFile({- data: JSON.stringify(article),- filePath: `archives/${targetLink.collectionId}/${link.id}_readability.json`,- });+ // send to archive.org+ if (user.archiveAsWaybackMachine && link.url) sendToWayback(link.url);- await prisma.link.update({+ const targetLink = await prisma.link.update({where: { id: link.id },data: {- readable: `archives/${targetLink.collectionId}/${link.id}_readability.json`,- textContent: articleText,+ type: linkType,+ image:+ user.archiveAsScreenshot && !link.image?.startsWith("archive")+ ? "pending"+ : undefined,+ pdf:+ user.archiveAsPDF && !link.pdf?.startsWith("archive")+ ? "pending"+ : undefined,+ readable: !link.readable?.startsWith("archive")+ ? "pending"+ : undefined,+ preview: !link.readable?.startsWith("archive")+ ? "pending"+ : undefined,+ lastPreserved: new Date().toISOString(),},});- }-- // Preview-- const ogImageUrl = await page.evaluate(() => {- const metaTag = document.querySelector('meta[property="og:image"]');- return metaTag ? (metaTag as any).content : null;- });-- createFolder({- filePath: `archives/preview/${link.collectionId}`,- });-- if (ogImageUrl) {- console.log("Found og:image URL:", ogImageUrl);-- // Download the image- const imageResponse = await page.goto(ogImageUrl);- // Check if imageResponse is not null- if (imageResponse && !link.preview?.startsWith("archive")) {- const buffer = await imageResponse.body();-- // Check if buffer is not null- if (buffer) {- // Load the image using Jimp- Jimp.read(buffer, async (err, image) => {- if (image && !err) {- image?.resize(1280, Jimp.AUTO).quality(20);- const processedBuffer = await image?.getBufferAsync(- Jimp.MIME_JPEG- );-- createFile({- data: processedBuffer,- filePath: `archives/preview/${link.collectionId}/${link.id}.jpeg`,- }).then(() => {- return prisma.link.update({- where: { id: link.id },- data: {- preview: `archives/preview/${link.collectionId}/${link.id}.jpeg`,- },- });- });- }- }).catch((err) => {- console.error("Error processing the image:", err);+ if (linkType === "image" && !link.image?.startsWith("archive")) {+ await imageHandler(link, imageExtension); // archive image (jpeg/png)+ return;+ } else if (linkType === "pdf" && !link.pdf?.startsWith("archive")) {+ await pdfHandler(link); // archive pdf+ return;+ } else if (link.url) {+ // archive url++ await page.goto(link.url, { waitUntil: "domcontentloaded" });++ const content = await page.content();++ // TODO single file+ // const session = await page.context().newCDPSession(page);+ // const doc = await session.send("Page.captureSnapshot", {+ // format: "mhtml",+ // });+ // const saveDocLocally = (doc: any) => {+ // console.log(doc);+ // return createFile({+ // data: doc,+ // filePath: `archives/${targetLink.collectionId}/${link.id}.mhtml`,+ // });+ // };+ // saveDocLocally(doc.data);++ // Readability+ const window = new JSDOM("").window;+ const purify = DOMPurify(window);+ const cleanedUpContent = purify.sanitize(content);+ const dom = new JSDOM(cleanedUpContent, { url: link.url || "" });+ const article = new Readability(dom.window.document).parse();+ const articleText = article?.textContent+ .replace(/ +(?= )/g, "") // strip out multiple spaces+ .replace(/(\r\n|\n|\r)/gm, " "); // strip out line breaks+ if (+ articleText &&+ articleText !== "" &&+ !link.readable?.startsWith("archive")+ ) {+ await createFile({+ data: JSON.stringify(article),+ filePath: `archives/${targetLink.collectionId}/${link.id}_readability.json`,});- } else {- console.log("No image data found.");- }- }- await page.goBack();- } else if (!link.preview?.startsWith("archive")) {- console.log("No og:image found");- await page- .screenshot({ type: "jpeg", quality: 20 })- .then((screenshot) => {- return createFile({- data: screenshot,- filePath: `archives/preview/${link.collectionId}/${link.id}.jpeg`,- });- })- .then(() => {- return prisma.link.update({+ await prisma.link.update({where: { id: link.id },data: {- preview: `archives/preview/${link.collectionId}/${link.id}.jpeg`,+ readable: `archives/${targetLink.collectionId}/${link.id}_readability.json`,+ textContent: articleText,},});+ }++ // Preview++ const ogImageUrl = await page.evaluate(() => {+ const metaTag = document.querySelector('meta[property="og:image"]');+ return metaTag ? (metaTag as any).content : null;});- }- // Screenshot/PDF- await page.evaluate(- autoScroll,- Number(process.env.AUTOSCROLL_TIMEOUT) || 30- );+ createFolder({+ filePath: `archives/preview/${link.collectionId}`,+ });- // Check if the user hasn't deleted the link by the time we're done scrolling- const linkExists = await prisma.link.findUnique({- where: { id: link.id },- });- if (linkExists) {- const processingPromises = [];-- if (user.archiveAsScreenshot && !link.image?.startsWith("archive")) {- processingPromises.push(- page.screenshot({ fullPage: true }).then((screenshot) => {- return createFile({- data: screenshot,- filePath: `archives/${linkExists.collectionId}/${link.id}.png`,- });- })- );- }- if (user.archiveAsPDF && !link.pdf?.startsWith("archive")) {- processingPromises.push(- page- .pdf({- width: "1366px",- height: "1931px",- printBackground: true,- margin: { top: "15px", bottom: "15px" },- })- .then((pdf) => {+ if (ogImageUrl) {+ console.log("Found og:image URL:", ogImageUrl);++ // Download the image+ const imageResponse = await page.goto(ogImageUrl);++ // Check if imageResponse is not null+ if (imageResponse && !link.preview?.startsWith("archive")) {+ const buffer = await imageResponse.body();++ // Check if buffer is not null+ if (buffer) {+ // Load the image using Jimp+ Jimp.read(buffer, async (err, image) => {+ if (image && !err) {+ image?.resize(1280, Jimp.AUTO).quality(20);+ const processedBuffer = await image?.getBufferAsync(+ Jimp.MIME_JPEG+ );++ createFile({+ data: processedBuffer,+ filePath: `archives/preview/${link.collectionId}/${link.id}.jpeg`,+ }).then(() => {+ return prisma.link.update({+ where: { id: link.id },+ data: {+ preview: `archives/preview/${link.collectionId}/${link.id}.jpeg`,+ },+ });+ });+ }+ }).catch((err) => {+ console.error("Error processing the image:", err);+ });+ } else {+ console.log("No image data found.");+ }+ }++ await page.goBack();+ } else if (!link.preview?.startsWith("archive")) {+ console.log("No og:image found");+ await page+ .screenshot({ type: "jpeg", quality: 20 })+ .then((screenshot) => {return createFile({- data: pdf,- filePath: `archives/${linkExists.collectionId}/${link.id}.pdf`,+ data: screenshot,+ filePath: `archives/preview/${link.collectionId}/${link.id}.jpeg`,});})+ .then(() => {+ return prisma.link.update({+ where: { id: link.id },+ data: {+ preview: `archives/preview/${link.collectionId}/${link.id}.jpeg`,+ },+ });+ });+ }++ // Screenshot/PDF+ await page.evaluate(+ autoScroll,+ Number(process.env.AUTOSCROLL_TIMEOUT) || 30);++ // Check if the user hasn't deleted the link by the time we're done scrolling+ const linkExists = await prisma.link.findUnique({+ where: { id: link.id },+ });+ if (linkExists) {+ const processingPromises = [];++ if (+ user.archiveAsScreenshot &&+ !link.image?.startsWith("archive")+ ) {+ processingPromises.push(+ page.screenshot({ fullPage: true }).then((screenshot) => {+ return createFile({+ data: screenshot,+ filePath: `archives/${linkExists.collectionId}/${link.id}.png`,+ });+ })+ );+ }+ if (user.archiveAsPDF && !link.pdf?.startsWith("archive")) {+ processingPromises.push(+ page+ .pdf({+ width: "1366px",+ height: "1931px",+ printBackground: true,+ margin: { top: "15px", bottom: "15px" },+ })+ .then((pdf) => {+ return createFile({+ data: pdf,+ filePath: `archives/${linkExists.collectionId}/${link.id}.pdf`,+ });+ })+ );+ }+ await Promise.allSettled(processingPromises);+ await prisma.link.update({+ where: { id: link.id },+ data: {+ image: user.archiveAsScreenshot+ ? `archives/${linkExists.collectionId}/${link.id}.png`+ : undefined,+ pdf: user.archiveAsPDF+ ? `archives/${linkExists.collectionId}/${link.id}.pdf`+ : undefined,+ },+ });+ }}- await Promise.allSettled(processingPromises);- await prisma.link.update({- where: { id: link.id },- data: {- image: user.archiveAsScreenshot- ? `archives/${linkExists.collectionId}/${link.id}.png`- : undefined,- pdf: user.archiveAsPDF- ? `archives/${linkExists.collectionId}/${link.id}.pdf`- : undefined,- },- });- }- }+ })(),+ timeoutPromise,+ ]);} catch (err) {console.log(err);console.log("Failed Link details:", link);diff --git a/types/enviornment.d.ts b/types/enviornment.d.tsindex 2d81a671..ebb90c7b 100644--- a/types/enviornment.d.ts+++ b/types/enviornment.d.ts@@ -12,6 +12,7 @@ declare global {NEXT_PUBLIC_MAX_FILE_SIZE?: string;MAX_LINKS_PER_USER?: string;ARCHIVE_TAKE_COUNT?: string;+ NEXT_PUBLIC_MAX_FILE_SIZE?: string;SPACES_KEY?: string;SPACES_SECRET?: string;