⚡️⚡️⚡️ Self-hosted collaborative bookmark manager to collect, read, annotate, and fully preserve what matters, all in one place.
TypeScript bookmarkbookmark-managercollaborationnextjsreact-nativeread-it-laterself-hostedtypescript
code improvements
daniel31x132b8f7d4be200
diff --git a/components/ModalContent/PreservedFormatsModal.tsx b/components/ModalContent/PreservedFormatsModal.tsxindex c52c4926..e6906a7f 100644--- a/components/ModalContent/PreservedFormatsModal.tsx+++ b/components/ModalContent/PreservedFormatsModal.tsx@@ -18,6 +18,7 @@ import {import PreservedFormatRow from "@/components/PreserverdFormatRow";import useAccountStore from "@/store/account";import getPublicUserData from "@/lib/client/getPublicUserData";+import { BeatLoader } from "react-spinners";type Props = {onClose: Function;@@ -87,6 +88,15 @@ export default function PreservedFormatsModal({ onClose, activeLink }: Props) {);};+ const atLeastOneFormatAvailable = () => {+ return (+ screenshotAvailable(link) ||+ pdfAvailable(link) ||+ readabilityAvailable(link) ||+ singlefileAvailable(link)+ );+ };+useEffect(() => {(async () => {const data = await getLink(link.id as number, isPublic);@@ -143,11 +153,10 @@ export default function PreservedFormatsModal({ onClose, activeLink }: Props) {<div className="divider mb-2 mt-1"></div>- {isReady() &&- (screenshotAvailable(link) ||- pdfAvailable(link) ||- readabilityAvailable(link) ||- singlefileAvailable(link)) ? (+ {screenshotAvailable(link) ||+ pdfAvailable(link) ||+ readabilityAvailable(link) ||+ singlefileAvailable(link) ? (<p className="mb-3">The following formats are available for this link:</p>@@ -156,56 +165,57 @@ export default function PreservedFormatsModal({ onClose, activeLink }: Props) {)}<div className={`flex flex-col gap-3`}>- {isReady() ? (- <>- {screenshotAvailable(link) ? (- <PreservedFormatRow- name={"Screenshot"}- icon={"bi-file-earmark-image"}- format={- link?.image?.endsWith("png")- ? ArchivedFormat.png- : ArchivedFormat.jpeg- }- activeLink={link}- downloadable={true}- />- ) : undefined}-- {pdfAvailable(link) ? (- <PreservedFormatRow- name={"PDF"}- icon={"bi-file-earmark-pdf"}- format={ArchivedFormat.pdf}- activeLink={link}- downloadable={true}- />- ) : undefined}-- {readabilityAvailable(link) ? (- <PreservedFormatRow- name={"Readable"}- icon={"bi-file-earmark-text"}- format={ArchivedFormat.readability}- activeLink={link}- />- ) : undefined}-- {singlefileAvailable(link) ? (- <PreservedFormatRow- name={"Singlefile"}- icon={"bi-filetype-html"}- format={ArchivedFormat.singlefile}- activeLink={link}- downloadable={true}- />- ) : undefined}- </>- ) : (- <div- className={`w-full h-full flex flex-col justify-center p-10 skeleton bg-base-200`}- >- <i className="bi-stack drop-shadow text-primary text-8xl mx-auto mb-5"></i>+ {screenshotAvailable(link) ? (+ <PreservedFormatRow+ name={"Screenshot"}+ icon={"bi-file-earmark-image"}+ format={+ link?.image?.endsWith("png")+ ? ArchivedFormat.png+ : ArchivedFormat.jpeg+ }+ activeLink={link}+ downloadable={true}+ />+ ) : undefined}++ {pdfAvailable(link) ? (+ <PreservedFormatRow+ name={"PDF"}+ icon={"bi-file-earmark-pdf"}+ format={ArchivedFormat.pdf}+ activeLink={link}+ downloadable={true}+ />+ ) : undefined}++ {singlefileAvailable(link) ? (+ <PreservedFormatRow+ name={"SingleFile (Full Copy)"}+ icon={"bi-filetype-html"}+ format={ArchivedFormat.singlefile}+ activeLink={link}+ downloadable={true}+ />+ ) : undefined}++ {readabilityAvailable(link) ? (+ <PreservedFormatRow+ name={"Readable"}+ icon={"bi-file-earmark-text"}+ format={ArchivedFormat.readability}+ activeLink={link}+ />+ ) : undefined}++ {!isReady() && !atLeastOneFormatAvailable() ? (+ <div className={`w-full h-full flex flex-col justify-center p-10`}>+ <BeatLoader+ color="oklch(var(--p))"+ className="mx-auto mb-3"+ size={30}+ />+<p className="text-center text-2xl">Link preservation is in the queue</p>@@ -213,7 +223,22 @@ export default function PreservedFormatsModal({ onClose, activeLink }: Props) {Please check back later to see the result</p></div>- )}+ ) : !isReady() && atLeastOneFormatAvailable() ? (+ <div className={`w-full h-full flex flex-col justify-center p-5`}>+ <BeatLoader+ color="oklch(var(--p))"+ className="mx-auto mb-3"+ size={20}+ />++ <p className="text-center">+ There are more preserved formats in the queue+ </p>+ <p className="text-center text-sm">+ Please check back later to see the result+ </p>+ </div>+ ) : undefined}<divclassName={`flex flex-col sm:flex-row gap-3 items-center justify-center ${diff --git a/lib/api/archiveHandler.ts b/lib/api/archiveHandler.tsindex 005d36b3..acec0a70 100644--- a/lib/api/archiveHandler.ts+++ b/lib/api/archiveHandler.ts@@ -1,18 +1,14 @@import { LaunchOptions, chromium, devices } from "playwright";import { prisma } from "./db";import createFile from "./storage/createFile";-import sendToWayback from "./sendToWayback";-import { Readability } from "@mozilla/readability";-import { JSDOM } from "jsdom";-import DOMPurify from "dompurify";+import sendToWayback from "./preservationScheme/sendToWayback";import { Collection, Link, User } from "@prisma/client";import validateUrlSize from "./validateUrlSize";import removeFile from "./storage/removeFile";import Jimp from "jimp";-import { execSync } from "child_process";-import axios from "axios";-import { Agent } from "http";import createFolder from "./storage/createFolder";+import archiveAsSinglefile from "./preservationScheme/archiveAsSinglefile";+import archiveAsReadability from "./preservationScheme/archiveAsReadablility";type LinksAndCollectionAndOwner = Link & {collection: Collection & {@@ -23,6 +19,18 @@ type LinksAndCollectionAndOwner = Link & {const BROWSER_TIMEOUT = Number(process.env.BROWSER_TIMEOUT) || 5;export default async function archiveHandler(link: LinksAndCollectionAndOwner) {+ const timeoutPromise = new Promise((_, reject) => {+ setTimeout(+ () =>+ reject(+ new Error(+ `Browser has been open for more than ${BROWSER_TIMEOUT} minutes.`+ )+ ),+ BROWSER_TIMEOUT * 60000+ );+ });+// allow user to configure a proxylet browserOptions: LaunchOptions = {};if (process.env.PROXY) {@@ -35,28 +43,12 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) {}const browser = await chromium.launch(browserOptions);- const context = await browser.newContext({- ...devices["Desktop Chrome"],- ignoreHTTPSErrors: process.env.IGNORE_HTTPS_ERRORS === "true",- });-- const page = await context.newPage();-- const timeoutPromise = new Promise((_, reject) => {- setTimeout(- () =>- reject(- new Error(- `Browser has been open for more than ${BROWSER_TIMEOUT} minutes.`- )- ),- BROWSER_TIMEOUT * 60000- );- });try {await Promise.race([(async () => {+ const user = link.collection?.owner;+const validatedUrl = link.url? await validateUrlSize(link.url): undefined;@@ -76,12 +68,7 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) {else if (contentType.includes("image/png")) imageExtension = "png";}- const user = link.collection?.owner;-- // send to archive.org- if (user.archiveAsWaybackMachine && link.url) sendToWayback(link.url);-- const targetLink = await prisma.link.update({+ await prisma.link.update({where: { id: link.id },data: {type: linkType,@@ -106,6 +93,18 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) {},});+ // SingleFile+ if (+ !link.singlefile?.startsWith("archive") &&+ !link.singlefile?.startsWith("unavailable") &&+ user.archiveAsSinglefile &&+ link.url+ )+ await archiveAsSinglefile(link);++ // send to archive.org+ if (user.archiveAsWaybackMachine && link.url) sendToWayback(link.url);+if (linkType === "image" && !link.image?.startsWith("archive")) {await imageHandler(link, imageExtension); // archive image (jpeg/png)return;@@ -115,230 +114,172 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) {} else if (link.url) {// archive url+ const context = await browser.newContext({+ ...devices["Desktop Chrome"],+ ignoreHTTPSErrors: process.env.IGNORE_HTTPS_ERRORS === "true",+ });++ const page = await context.newPage();+await page.goto(link.url, { waitUntil: "domcontentloaded" });const content = await page.content();- // Singlefile+ // Readabilityif (- user.archiveAsSinglefile &&- !link.singlefile?.startsWith("archive")- ) {- let command = process.env.SINGLEFILE_ARCHIVE_COMMAND;- let httpApi = process.env.SINGLEFILE_ARCHIVE_HTTP_API;- if (command) {- if (command.includes("{{URL}}")) {- try {- let html = execSync(command.replace("{{URL}}", link.url), {- timeout: 60000,- maxBuffer: 1024 * 1024 * 100,- });- await createFile({- data: html,- filePath: `archives/${targetLink.collectionId}/${link.id}.html`,- });- } catch (err) {- console.error(- "Error running SINGLEFILE_ARCHIVE_COMMAND:",- err- );- }- } else {- console.error(- "Invalid SINGLEFILE_ARCHIVE_COMMAND. Missing {{URL}}"- );- }- } else if (httpApi) {- try {- let html = await axios.post(- httpApi,- { url: link.url },- {- headers: {- "Content-Type": "application/x-www-form-urlencoded",- },- httpAgent: new Agent({ keepAlive: false }),- }- );- await createFile({- data: html.data,- filePath: `archives/${targetLink.collectionId}/${link.id}.html`,- });- } catch (err) {- console.error(- "Error fetching Singlefile using SINGLEFILE_ARCHIVE_HTTP_API:",- err- );- }- } else {- console.error(- "No SINGLEFILE_ARCHIVE_COMMAND or SINGLEFILE_ARCHIVE_HTTP_API defined."- );- }- }+ !link.readable?.startsWith("archives") &&+ !link.readable?.startsWith("unavailable")+ )+ await archiveAsReadability(content, link);++ // Preview- // Readability- const window = new JSDOM("").window;- const purify = DOMPurify(window);- const cleanedUpContent = purify.sanitize(content);- const dom = new JSDOM(cleanedUpContent, { url: link.url || "" });- const article = new Readability(dom.window.document).parse();- const articleText = article?.textContent- .replace(/ +(?= )/g, "") // strip out multiple spaces- .replace(/(\r\n|\n|\r)/gm, " "); // strip out line breaksif (- articleText &&- articleText !== "" &&- !link.readable?.startsWith("archive")+ !link.preview?.startsWith("archives") &&+ !link.preview?.startsWith("unavailable")) {- await createFile({- data: JSON.stringify(article),- filePath: `archives/${targetLink.collectionId}/${link.id}_readability.json`,+ const ogImageUrl = await page.evaluate(() => {+ const metaTag = document.querySelector(+ 'meta[property="og:image"]'+ );+ return metaTag ? (metaTag as any).content : null;});- await prisma.link.update({- where: { id: link.id },- data: {- readable: `archives/${targetLink.collectionId}/${link.id}_readability.json`,- textContent: articleText,- },+ createFolder({+ filePath: `archives/preview/${link.collectionId}`,});- }- // Preview-- const ogImageUrl = await page.evaluate(() => {- const metaTag = document.querySelector('meta[property="og:image"]');- return metaTag ? (metaTag as any).content : null;- });-- createFolder({- filePath: `archives/preview/${link.collectionId}`,- });-- if (ogImageUrl) {- console.log("Found og:image URL:", ogImageUrl);-- // Download the image- const imageResponse = await page.goto(ogImageUrl);-- // Check if imageResponse is not null- if (imageResponse && !link.preview?.startsWith("archive")) {- const buffer = await imageResponse.body();-- // Check if buffer is not null- if (buffer) {- // Load the image using Jimp- Jimp.read(buffer, async (err, image) => {- if (image && !err) {- image?.resize(1280, Jimp.AUTO).quality(20);- const processedBuffer = await image?.getBufferAsync(- Jimp.MIME_JPEG- );-- createFile({- data: processedBuffer,- filePath: `archives/preview/${link.collectionId}/${link.id}.jpeg`,- }).then(() => {- return prisma.link.update({- where: { id: link.id },- data: {- preview: `archives/preview/${link.collectionId}/${link.id}.jpeg`,- },+ if (ogImageUrl) {+ console.log("Found og:image URL:", ogImageUrl);++ // Download the image+ const imageResponse = await page.goto(ogImageUrl);++ // Check if imageResponse is not null+ if (imageResponse && !link.preview?.startsWith("archive")) {+ const buffer = await imageResponse.body();++ // Check if buffer is not null+ if (buffer) {+ // Load the image using Jimp+ Jimp.read(buffer, async (err, image) => {+ if (image && !err) {+ image?.resize(1280, Jimp.AUTO).quality(20);+ const processedBuffer = await image?.getBufferAsync(+ Jimp.MIME_JPEG+ );++ createFile({+ data: processedBuffer,+ filePath: `archives/preview/${link.collectionId}/${link.id}.jpeg`,+ }).then(() => {+ return prisma.link.update({+ where: { id: link.id },+ data: {+ preview: `archives/preview/${link.collectionId}/${link.id}.jpeg`,+ },+ });});- });- }- }).catch((err) => {- console.error("Error processing the image:", err);- });- } else {- console.log("No image data found.");+ }+ }).catch((err) => {+ console.error("Error processing the image:", err);+ });+ } else {+ console.log("No image data found.");+ }}- }-- await page.goBack();- } else if (!link.preview?.startsWith("archive")) {- console.log("No og:image found");- await page- .screenshot({ type: "jpeg", quality: 20 })- .then((screenshot) => {- return createFile({- data: screenshot,- filePath: `archives/preview/${link.collectionId}/${link.id}.jpeg`,- });- })- .then(() => {- return prisma.link.update({- where: { id: link.id },- data: {- preview: `archives/preview/${link.collectionId}/${link.id}.jpeg`,- },- });- });- }-- // Screenshot/PDF- await page.evaluate(- autoScroll,- Number(process.env.AUTOSCROLL_TIMEOUT) || 30- );- // Check if the user hasn't deleted the link by the time we're done scrolling- const linkExists = await prisma.link.findUnique({- where: { id: link.id },- });- if (linkExists) {- const processingPromises = [];-- if (- user.archiveAsScreenshot &&- !link.image?.startsWith("archive")- ) {- processingPromises.push(- page.screenshot({ fullPage: true }).then((screenshot) => {+ await page.goBack();+ } else if (!link.preview?.startsWith("archive")) {+ console.log("No og:image found");+ await page+ .screenshot({ type: "jpeg", quality: 20 })+ .then((screenshot) => {return createFile({data: screenshot,- filePath: `archives/${linkExists.collectionId}/${link.id}.png`,+ filePath: `archives/preview/${link.collectionId}/${link.id}.jpeg`,});})- );+ .then(() => {+ return prisma.link.update({+ where: { id: link.id },+ data: {+ preview: `archives/preview/${link.collectionId}/${link.id}.jpeg`,+ },+ });+ });}+ }- // apply administrator's defined pdf margins or default to 15px- const margins = {- top: process.env.PDF_MARGIN_TOP || "15px",- bottom: process.env.PDF_MARGIN_BOTTOM || "15px",- };-- if (user.archiveAsPDF && !link.pdf?.startsWith("archive")) {- processingPromises.push(- page- .pdf({- width: "1366px",- height: "1931px",- printBackground: true,- margin: margins,- })- .then((pdf) => {+ if (+ (!link.image?.startsWith("archives") &&+ !link.image?.startsWith("unavailable")) ||+ (!link.pdf?.startsWith("archives") &&+ !link.pdf?.startsWith("unavailable"))+ ) {+ // Screenshot/PDF+ await page.evaluate(+ autoScroll,+ Number(process.env.AUTOSCROLL_TIMEOUT) || 30+ );++ // Check if the user hasn't deleted the link by the time we're done scrolling+ const linkExists = await prisma.link.findUnique({+ where: { id: link.id },+ });+ if (linkExists) {+ const processingPromises = [];++ if (+ user.archiveAsScreenshot &&+ !link.image?.startsWith("archive")+ ) {+ processingPromises.push(+ page.screenshot({ fullPage: true }).then((screenshot) => {return createFile({- data: pdf,- filePath: `archives/${linkExists.collectionId}/${link.id}.pdf`,+ data: screenshot,+ filePath: `archives/${linkExists.collectionId}/${link.id}.png`,});})- );+ );+ }++ // apply administrator's defined pdf margins or default to 15px+ const margins = {+ top: process.env.PDF_MARGIN_TOP || "15px",+ bottom: process.env.PDF_MARGIN_BOTTOM || "15px",+ };++ if (user.archiveAsPDF && !link.pdf?.startsWith("archive")) {+ processingPromises.push(+ page+ .pdf({+ width: "1366px",+ height: "1931px",+ printBackground: true,+ margin: margins,+ })+ .then((pdf) => {+ return createFile({+ data: pdf,+ filePath: `archives/${linkExists.collectionId}/${link.id}.pdf`,+ });+ })+ );+ }+ await Promise.allSettled(processingPromises);+ await prisma.link.update({+ where: { id: link.id },+ data: {+ image: user.archiveAsScreenshot+ ? `archives/${linkExists.collectionId}/${link.id}.png`+ : undefined,+ pdf: user.archiveAsPDF+ ? `archives/${linkExists.collectionId}/${link.id}.pdf`+ : undefined,+ },+ });}- await Promise.allSettled(processingPromises);- await prisma.link.update({- where: { id: link.id },- data: {- image: user.archiveAsScreenshot- ? `archives/${linkExists.collectionId}/${link.id}.png`- : undefined,- pdf: user.archiveAsPDF- ? `archives/${linkExists.collectionId}/${link.id}.pdf`- : undefined,- },- });}}})(),diff --git a/lib/api/preservationScheme/archiveAsReadablility.ts b/lib/api/preservationScheme/archiveAsReadablility.tsnew file mode 100644index 00000000..dec27dac--- /dev/null+++ b/lib/api/preservationScheme/archiveAsReadablility.ts@@ -0,0 +1,44 @@+import { Readability } from "@mozilla/readability";+import { JSDOM } from "jsdom";+import DOMPurify from "dompurify";+import { prisma } from "../db";+import createFile from "../storage/createFile";+import { Link } from "@prisma/client";++const archiveAsReadablility = async (content: string, link: Link) => {+ const window = new JSDOM("").window;+ const purify = DOMPurify(window);+ const cleanedUpContent = purify.sanitize(content);+ const dom = new JSDOM(cleanedUpContent, { url: link.url || "" });+ const article = new Readability(dom.window.document).parse();+ const articleText = article?.textContent+ .replace(/ +(?= )/g, "") // strip out multiple spaces+ .replace(/(\r\n|\n|\r)/gm, " "); // strip out line breaks+ if (+ articleText &&+ articleText !== "" &&+ !link.readable?.startsWith("archive")+ ) {+ const collectionId = (+ await prisma.link.findUnique({+ where: { id: link.id },+ select: { collectionId: true },+ })+ )?.collectionId;++ await createFile({+ data: JSON.stringify(article),+ filePath: `archives/${collectionId}/${link.id}_readability.json`,+ });++ await prisma.link.update({+ where: { id: link.id },+ data: {+ readable: `archives/${collectionId}/${link.id}_readability.json`,+ textContent: articleText,+ },+ });+ }+};++export default archiveAsReadablility;diff --git a/lib/api/preservationScheme/archiveAsSinglefile.ts b/lib/api/preservationScheme/archiveAsSinglefile.tsnew file mode 100644index 00000000..0c739b8a--- /dev/null+++ b/lib/api/preservationScheme/archiveAsSinglefile.ts@@ -0,0 +1,111 @@+import { execSync } from "child_process";+import createFile from "../storage/createFile";+import axios from "axios";+import { Agent } from "http";+import { prisma } from "../db";+import { Link } from "@prisma/client";++const archiveAsSinglefile = async (link: Link) => {+ if (!link.url) return;++ let command = process.env.SINGLEFILE_ARCHIVE_COMMAND;+ let httpApi = process.env.SINGLEFILE_ARCHIVE_HTTP_API;+ if (command) {+ if (command.includes("{{URL}}")) {+ try {+ let html = execSync(command.replace("{{URL}}", link.url), {+ timeout: 120000,+ maxBuffer: 1024 * 1024 * 30,+ });++ if (!html.length) {+ console.error(+ "Error running SINGLEFILE_ARCHIVE_COMMAND: Empty buffer"+ );+ return;+ }++ const collectionId = (+ await prisma.link.findUnique({+ where: { id: link.id },+ select: { collectionId: true },+ })+ )?.collectionId;++ if (!collectionId) {+ console.error(+ "Error running SINGLEFILE_ARCHIVE_COMMAND: Collection ID not found"+ );+ return;+ }++ await createFile({+ data: html,+ filePath: `archives/${collectionId}/${link.id}.html`,+ }).then(async () => {+ await prisma.link.update({+ where: { id: link.id },+ data: {+ singlefile: `archives/${collectionId}/${link.id}.html`,+ },+ });+ });+ } catch (err) {+ console.error("Error running SINGLEFILE_ARCHIVE_COMMAND:", err);+ }+ } else {+ console.error("Invalid SINGLEFILE_ARCHIVE_COMMAND. Missing {{URL}}");+ }+ } else if (httpApi) {+ try {+ let html = await axios.post(+ httpApi,+ { url: link.url },+ {+ headers: {+ "Content-Type": "application/x-www-form-urlencoded",+ },+ httpAgent: new Agent({ keepAlive: false }),+ }+ );++ if (!html.data.length) {+ console.error("Error running SINGLEFILE_ARCHIVE_COMMAND: Empty buffer");+ return;+ }++ const collectionId = (+ await prisma.link.findUnique({+ where: { id: link.id },+ select: { collectionId: true },+ })+ )?.collectionId;++ if (!collectionId) {+ console.error(+ "Error running SINGLEFILE_ARCHIVE_COMMAND: Collection ID not found"+ );+ return;+ }++ await createFile({+ data: html.data,+ filePath: `archives/${collectionId}/${link.id}.html`,+ }).then(async () => {+ await prisma.link.update({+ where: { id: link.id },+ data: {+ singlefile: `archives/${collectionId}/${link.id}.html`,+ },+ });+ });+ } catch (err) {+ console.error(+ "Error fetching Singlefile using SINGLEFILE_ARCHIVE_HTTP_API:",+ err+ );+ }+ }+};++export default archiveAsSinglefile;diff --git a/lib/api/sendToWayback.ts b/lib/api/preservationScheme/sendToWayback.tssimilarity index 100%rename from lib/api/sendToWayback.tsrename to lib/api/preservationScheme/sendToWayback.ts