⚡️⚡️⚡️ Self-hosted collaborative bookmark manager to collect, read, annotate, and fully preserve what matters, all in one place.
TypeScript bookmarkbookmark-managercollaborationnextjsreact-nativeread-it-laterself-hostedtypescript
code refactoring
daniel31x133618ba907d67
diff --git a/components/ModalContent/PreservedFormatsModal.tsx b/components/ModalContent/PreservedFormatsModal.tsxindex 0030250e..0f2ef369 100644--- a/components/ModalContent/PreservedFormatsModal.tsx+++ b/components/ModalContent/PreservedFormatsModal.tsx@@ -124,7 +124,7 @@ export default function PreservedFormatsModal({ onClose, activeLink }: Props) {clearInterval(interval);}};- }, [link, getLink, link?.singlefile]);+ }, [link?.singlefile]);const updateArchive = async () => {const load = toast.loading(t("sending_request"));@@ -210,10 +210,7 @@ export default function PreservedFormatsModal({ onClose, activeLink }: Props) {className="mx-auto mb-3"size={20}/>-- <p className="text-center">- There are more preserved formats in the queue- </p>+ <p className="text-center">{t("there_are_more_formats")}</p><p className="text-center text-sm">{t("check_back_later")}</p></div>) : undefined}diff --git a/lib/api/archiveHandler.ts b/lib/api/archiveHandler.tsindex 862c6a26..cd95db4d 100644--- a/lib/api/archiveHandler.ts+++ b/lib/api/archiveHandler.ts@@ -1,4 +1,4 @@-import { LaunchOptions, chromium, devices } from "playwright";+import { LaunchOptions, Page, chromium, devices } from "playwright";import { prisma } from "./db";import createFile from "./storage/createFile";import sendToWayback from "./preservationScheme/sendToWayback";@@ -9,6 +9,7 @@ import generatePreview from "./generatePreview";import { removeFiles } from "./manageLinkFiles";import archiveAsSinglefile from "./preservationScheme/archiveAsSinglefile";import archiveAsReadability from "./preservationScheme/archiveAsReadablility";+import shell from "shelljs";type LinksAndCollectionAndOwner = Link & {collection: Collection & {@@ -50,6 +51,26 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) {const page = await context.newPage();+ // await page.goto("https://github.com", {+ // waitUntil: "domcontentloaded",+ // });++ // console.log("Opening page:", link.url);++ // await page.evaluate(autoScroll, Number(process.env.AUTOSCROLL_TIMEOUT) || 30);++ // const dom = await page.content();++ // console.log("The content", dom);++ // shell+ // .echo(dom)+ // .exec(+ // "monolith - -I -b https://marketplace.visualstudio.com/items?itemName=42Crunch.vscode-openapi -j -F -o monolith.html"+ // );++ // console.log("Monolith created!");+createFolder({filePath: `archives/preview/${link.collectionId}`,});@@ -111,13 +132,13 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) {});// SingleFile- if (- !link.singlefile?.startsWith("archive") &&- !link.singlefile?.startsWith("unavailable") &&- user.archiveAsSinglefile &&- link.url- )- await archiveAsSinglefile(link);+ // if (+ // !link.singlefile?.startsWith("archive") &&+ // !link.singlefile?.startsWith("unavailable") &&+ // user.archiveAsSinglefile &&+ // link.url+ // )+ // await archiveAsSinglefile(link);// send to archive.orgif (user.archiveAsWaybackMachine && link.url) sendToWayback(link.url);@@ -131,13 +152,6 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) {} else if (link.url) {// archive url- const context = await browser.newContext({- ...devices["Desktop Chrome"],- ignoreHTTPSErrors: process.env.IGNORE_HTTPS_ERRORS === "true",- });-- const page = await context.newPage();-await page.goto(link.url, { waitUntil: "domcontentloaded" });const content = await page.content();@@ -150,115 +164,20 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) {await archiveAsReadability(content, link);// Preview+ if (+ !link.preview?.startsWith("archives") &&+ !link.preview?.startsWith("unavailable")+ )+ await getArchivePreview(link, page);- const ogImageUrl = await page.evaluate(() => {- const metaTag = document.querySelector('meta[property="og:image"]');- return metaTag ? (metaTag as any).content : null;- });-- if (ogImageUrl) {- console.log("Found og:image URL:", ogImageUrl);-- // Download the image- const imageResponse = await page.goto(ogImageUrl);-- // Check if imageResponse is not null- if (imageResponse && !link.preview?.startsWith("archive")) {- const buffer = await imageResponse.body();- await generatePreview(buffer, link.collectionId, link.id);- }-- await page.goBack();- } else if (!link.preview?.startsWith("archive")) {- console.log("No og:image found");- await page- .screenshot({ type: "jpeg", quality: 20 })- .then((screenshot) => {- return createFile({- data: screenshot,- filePath: `archives/preview/${link.collectionId}/${link.id}.jpeg`,- });- })- .then(() => {- return prisma.link.update({- where: { id: link.id },- data: {- preview: `archives/preview/${link.collectionId}/${link.id}.jpeg`,- },- });- });- }- }-- if (- (!link.image?.startsWith("archives") &&- !link.image?.startsWith("unavailable")) ||- (!link.pdf?.startsWith("archives") &&- !link.pdf?.startsWith("unavailable"))- ) {// Screenshot/PDF- await page.evaluate(- autoScroll,- Number(process.env.AUTOSCROLL_TIMEOUT) || 30- );-- // Check if the user hasn't deleted the link by the time we're done scrolling- const linkExists = await prisma.link.findUnique({- where: { id: link.id },- });- if (linkExists) {- const processingPromises = [];-- if (- user.archiveAsScreenshot &&- !link.image?.startsWith("archive")- ) {- processingPromises.push(- page.screenshot({ fullPage: true }).then((screenshot) => {- return createFile({- data: screenshot,- filePath: `archives/${linkExists.collectionId}/${link.id}.png`,- });- })- );- }-- // apply administrator's defined pdf margins or default to 15px- const margins = {- top: process.env.PDF_MARGIN_TOP || "15px",- bottom: process.env.PDF_MARGIN_BOTTOM || "15px",- };-- if (user.archiveAsPDF && !link.pdf?.startsWith("archive")) {- processingPromises.push(- page- .pdf({- width: "1366px",- height: "1931px",- printBackground: true,- margin: margins,- })- .then((pdf) => {- return createFile({- data: pdf,- filePath: `archives/${linkExists.collectionId}/${link.id}.pdf`,- });- })- );- }- await Promise.allSettled(processingPromises);- await prisma.link.update({- where: { id: link.id },- data: {- image: user.archiveAsScreenshot- ? `archives/${linkExists.collectionId}/${link.id}.png`- : undefined,- pdf: user.archiveAsPDF- ? `archives/${linkExists.collectionId}/${link.id}.pdf`- : undefined,- },- });- }+ if (+ (!link.image?.startsWith("archives") &&+ !link.image?.startsWith("unavailable")) ||+ (!link.pdf?.startsWith("archives") &&+ !link.pdf?.startsWith("unavailable"))+ )+ await captureScreenshotAndPdf(link, page, user);}})(),timeoutPromise,@@ -302,31 +221,6 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) {}}-const autoScroll = async (AUTOSCROLL_TIMEOUT: number) => {- const timeoutPromise = new Promise<void>((_, reject) => {- setTimeout(() => {- reject(new Error(`Webpage was too long to be archived.`));- }, AUTOSCROLL_TIMEOUT * 1000);- });-- const scrollingPromise = new Promise<void>((resolve) => {- let totalHeight = 0;- let distance = 100;- let scrollDown = setInterval(() => {- let scrollHeight = document.body.scrollHeight;- window.scrollBy(0, distance);- totalHeight += distance;- if (totalHeight >= scrollHeight) {- clearInterval(scrollDown);- window.scroll(0, 0);- resolve();- }- }, 100);- });-- await Promise.race([scrollingPromise, timeoutPromise]);-};-const imageHandler = async ({ url, id }: Link, extension: string) => {const image = await fetch(url as string).then((res) => res.blob());@@ -374,3 +268,133 @@ const pdfHandler = async ({ url, id }: Link) => {});}};++const getArchivePreview = async (+ link: LinksAndCollectionAndOwner,+ page: Page+) => {+ const ogImageUrl = await page.evaluate(() => {+ const metaTag = document.querySelector('meta[property="og:image"]');+ return metaTag ? (metaTag as any).content : null;+ });++ if (ogImageUrl) {+ console.log("Found og:image URL:", ogImageUrl);++ // Download the image+ const imageResponse = await page.goto(ogImageUrl);++ // Check if imageResponse is not null+ if (imageResponse && !link.preview?.startsWith("archive")) {+ const buffer = await imageResponse.body();+ generatePreview(buffer, link.collectionId, link.id);+ }++ await page.goBack();+ } else if (!link.preview?.startsWith("archive")) {+ console.log("No og:image found");+ await page+ .screenshot({ type: "jpeg", quality: 20 })+ .then((screenshot) => {+ return createFile({+ data: screenshot,+ filePath: `archives/preview/${link.collectionId}/${link.id}.jpeg`,+ });+ })+ .then(() => {+ return prisma.link.update({+ where: { id: link.id },+ data: {+ preview: `archives/preview/${link.collectionId}/${link.id}.jpeg`,+ },+ });+ });+ }+};++const captureScreenshotAndPdf = async (+ link: LinksAndCollectionAndOwner,+ page: Page,+ user: User+) => {+ await page.evaluate(autoScroll, Number(process.env.AUTOSCROLL_TIMEOUT) || 30);++ // Check if the user hasn't deleted the link by the time we're done scrolling+ const linkExists = await prisma.link.findUnique({+ where: { id: link.id },+ });+ if (linkExists) {+ const processingPromises = [];++ if (user.archiveAsScreenshot && !link.image?.startsWith("archive")) {+ processingPromises.push(+ page.screenshot({ fullPage: true, type: "png" }).then((screenshot) => {+ return createFile({+ data: screenshot,+ filePath: `archives/${linkExists.collectionId}/${link.id}.png`,+ });+ })+ );+ }++ const margins = {+ top: process.env.PDF_MARGIN_TOP || "15px",+ bottom: process.env.PDF_MARGIN_BOTTOM || "15px",+ };++ if (user.archiveAsPDF && !link.pdf?.startsWith("archive")) {+ processingPromises.push(+ page+ .pdf({+ width: "1366px",+ height: "1931px",+ printBackground: true,+ margin: margins,+ })+ .then((pdf) => {+ return createFile({+ data: pdf,+ filePath: `archives/${linkExists.collectionId}/${link.id}.pdf`,+ });+ })+ );+ }+ await Promise.allSettled(processingPromises);+ await prisma.link.update({+ where: { id: link.id },+ data: {+ image: user.archiveAsScreenshot+ ? `archives/${linkExists.collectionId}/${link.id}.png`+ : undefined,+ pdf: user.archiveAsPDF+ ? `archives/${linkExists.collectionId}/${link.id}.pdf`+ : undefined,+ },+ });+ }+};++const autoScroll = async (AUTOSCROLL_TIMEOUT: number) => {+ const timeoutPromise = new Promise<void>((resolve) => {+ setTimeout(() => {+ resolve();+ }, AUTOSCROLL_TIMEOUT * 1000);+ });++ const scrollingPromise = new Promise<void>((resolve) => {+ let totalHeight = 0;+ let distance = 100;+ let scrollDown = setInterval(() => {+ let scrollHeight = document.body.scrollHeight;+ window.scrollBy(0, distance);+ totalHeight += distance;+ if (totalHeight >= scrollHeight) {+ clearInterval(scrollDown);+ window.scroll(0, 0);+ resolve();+ }+ }, 100);+ });++ await Promise.race([scrollingPromise, timeoutPromise]);+};diff --git a/lib/api/generatePreview.ts b/lib/api/generatePreview.tsindex 6e816303..3c2da0f2 100644--- a/lib/api/generatePreview.ts+++ b/lib/api/generatePreview.ts@@ -1,7 +1,6 @@import Jimp from "jimp";import { prisma } from "./db";import createFile from "./storage/createFile";-import createFolder from "./storage/createFolder";const generatePreview = async (buffer: Buffer,diff --git a/lib/api/preservationScheme/archiveAsReadablility.ts b/lib/api/preservationScheme/archiveAsReadablility.tsindex dec27dac..0f9a5c7a 100644--- a/lib/api/preservationScheme/archiveAsReadablility.ts+++ b/lib/api/preservationScheme/archiveAsReadablility.ts@@ -14,11 +14,9 @@ const archiveAsReadablility = async (content: string, link: Link) => {const articleText = article?.textContent.replace(/ +(?= )/g, "") // strip out multiple spaces.replace(/(\r\n|\n|\r)/gm, " "); // strip out line breaks- if (- articleText &&- articleText !== "" &&- !link.readable?.startsWith("archive")- ) {++ console.log(articleText);+ if (articleText && articleText !== "") {const collectionId = (await prisma.link.findUnique({where: { id: link.id },diff --git a/package.json b/package.jsonindex 0045802c..70271221 100644--- a/package.json+++ b/package.json@@ -70,6 +70,7 @@"react-masonry-css": "^1.0.16","react-select": "^5.7.4","react-spinners": "^0.13.8",+ "shelljs": "^0.8.5","socks-proxy-agent": "^8.0.2","stripe": "^12.13.0","tailwind-merge": "^2.3.0",diff --git a/public/locales/en/common.json b/public/locales/en/common.jsonindex 7b6dd576..88f5c331 100644--- a/public/locales/en/common.json+++ b/public/locales/en/common.json@@ -220,8 +220,9 @@"github": "GitHub","twitter": "Twitter","mastodon": "Mastodon",- "link_preservation_in_queue": "LThe Link preservation is currently in the queue",+ "link_preservation_in_queue": "The Link preservation is currently in the queue","check_back_later": "Please check back later to see the result",+ "there_are_more_formats": "There are more preserved formats in the queue","settings": "Settings","switch_to": "Switch to {{theme}}","logout": "Logout",diff --git a/yarn.lock b/yarn.lockindex 55074b61..05b76f35 100644--- a/yarn.lock+++ b/yarn.lock@@ -3514,6 +3514,11 @@ function-bind@^1.1.1:resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.1.tgz#a56899d3ea3c9bab874bb9773b7c5ede92f4895d"integrity sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==+function-bind@^1.1.2:+ version "1.1.2"+ resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.2.tgz#2c02d864d97f3ea6c8830c464cbd11ab6eab7a1c"+ integrity sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==+function.prototype.name@^1.1.5:version "1.1.5"resolved "https://registry.yarnpkg.com/function.prototype.name/-/function.prototype.name-1.1.5.tgz#cce0505fe1ffb80503e6f9e46cc64e46a12a9621"@@ -3651,7 +3656,7 @@ [email protected]:once "^1.3.0"path-is-absolute "^1.0.0"-glob@^7.1.3:+glob@^7.0.0, glob@^7.1.3:version "7.2.3"resolved "https://registry.yarnpkg.com/glob/-/glob-7.2.3.tgz#b8df0fb802bbfa8e89bd1d938b4e16578ed44f2b"integrity sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==@@ -3821,6 +3826,13 @@ has@^1.0.3:dependencies:function-bind "^1.1.1"+hasown@^2.0.0:+ version "2.0.2"+ resolved "https://registry.yarnpkg.com/hasown/-/hasown-2.0.2.tgz#003eaf91be7adc372e84ec59dc37252cedb80003"+ integrity sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==+ dependencies:+ function-bind "^1.1.2"+hexoid@^1.0.0:version "1.0.0"resolved "https://registry.yarnpkg.com/hexoid/-/hexoid-1.0.0.tgz#ad10c6573fb907de23d9ec63a711267d9dc9bc18"@@ -3972,6 +3984,11 @@ internal-slot@^1.0.3, internal-slot@^1.0.4:has "^1.0.3"side-channel "^1.0.4"+interpret@^1.0.0:+ version "1.4.0"+ resolved "https://registry.yarnpkg.com/interpret/-/interpret-1.4.0.tgz#665ab8bc4da27a774a40584e812e3e0fa45b1a1e"+ integrity sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA==+invariant@^2.2.4:version "2.2.4"resolved "https://registry.yarnpkg.com/invariant/-/invariant-2.2.4.tgz#610f3c92c9359ce1db616e538008d23ff35158e6"@@ -4053,6 +4070,13 @@ is-core-module@^2.10.0, is-core-module@^2.11.0, is-core-module@^2.9.0:dependencies:has "^1.0.3"+is-core-module@^2.13.0:+ version "2.13.1"+ resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.13.1.tgz#ad0d7532c6fea9da1ebdc82742d74525c6273384"+ integrity sha512-hHrIjvZsftOsvKSn2TRYl63zvxsgE0K+0mYMoH6gD4omR5IWB2KynivBQczo3+wF1cCkjzvptnI9Q0sPU66ilw==+ dependencies:+ hasown "^2.0.0"+is-date-object@^1.0.1, is-date-object@^1.0.5:version "1.0.5"resolved "https://registry.yarnpkg.com/is-date-object/-/is-date-object-1.0.5.tgz#0841d5536e724c25597bf6ea62e1bd38298df31f"@@ -5372,6 +5396,13 @@ readdirp@~3.6.0:dependencies:picomatch "^2.2.1"+rechoir@^0.6.2:+ version "0.6.2"+ resolved "https://registry.yarnpkg.com/rechoir/-/rechoir-0.6.2.tgz#85204b54dba82d5742e28c96756ef43af50e3384"+ integrity sha512-HFM8rkZ+i3zrV+4LQjwQ0W+ez98pApMGM3HUrN04j3CqzPOzl9nmP15Y8YXNm8QHGv/eacOVEjqhmWpkRV0NAw==+ dependencies:+ resolve "^1.1.6"+redux@^4.0.0, redux@^4.0.1:version "4.2.1"resolved "https://registry.yarnpkg.com/redux/-/redux-4.2.1.tgz#c08f4306826c49b5e9dc901dee0452ea8fce6197"@@ -5439,6 +5470,15 @@ resolve-from@^4.0.0:resolved "https://registry.yarnpkg.com/resolve-from/-/resolve-from-4.0.0.tgz#4abcd852ad32dd7baabfe9b40e00a36db5f392e6"integrity sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==+resolve@^1.1.6:+ version "1.22.8"+ resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.22.8.tgz#b6c87a9f2aa06dfab52e3d70ac8cde321fa5a48d"+ integrity sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw==+ dependencies:+ is-core-module "^2.13.0"+ path-parse "^1.0.7"+ supports-preserve-symlinks-flag "^1.0.0"+resolve@^1.1.7, resolve@^1.19.0, resolve@^1.22.1, resolve@^1.22.2:version "1.22.2"resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.22.2.tgz#0ed0943d4e301867955766c9f3e1ae6d01c6845f"@@ -5565,6 +5605,15 @@ shell-quote@^1.8.1:resolved "https://registry.yarnpkg.com/shell-quote/-/shell-quote-1.8.1.tgz#6dbf4db75515ad5bac63b4f1894c3a154c766680"integrity sha512-6j1W9l1iAs/4xYBI1SYOVZyFcCis9b4KCLQ8fgAGG07QvzaRLVVRQvAy85yNmmZSjYjg4MWh4gNvlPujU/5LpA==+shelljs@^0.8.5:+ version "0.8.5"+ resolved "https://registry.yarnpkg.com/shelljs/-/shelljs-0.8.5.tgz#de055408d8361bed66c669d2f000538ced8ee20c"+ integrity sha512-TiwcRcrkhHvbrZbnRcFYMLl30Dfov3HKqzp5tO5b4pt6G/SezKcYhmDg15zXVBswHmctSAQKznqNW2LO5tTDow==+ dependencies:+ glob "^7.0.0"+ interpret "^1.0.0"+ rechoir "^0.6.2"+side-channel@^1.0.4:version "1.0.4"resolved "https://registry.yarnpkg.com/side-channel/-/side-channel-1.0.4.tgz#efce5c8fdc104ee751b25c58d4290011fa5ea2cf"