⚡️⚡️⚡️ Self-hosted collaborative bookmark manager to collect, read, annotate, and fully preserve what matters, all in one place.
TypeScript bookmarkbookmark-managercollaborationnextjsreact-nativeread-it-laterself-hostedtypescript
improved archive handler
daniel31x13644b827669f3
diff --git a/components/Announcement.tsx b/components/Announcement.tsxindex 2f97297e..5486f8ba 100644--- a/components/Announcement.tsx+++ b/components/Announcement.tsx@@ -10,8 +10,8 @@ export default function Announcement({ toggleAnnouncementBar }: Props) {const announcementId = localStorage.getItem("announcementId");return (- <div className="fixed left-0 right-0 bottom-20 sm:bottom-10 w-full p-5 z-30">- <div className="mx-auto w-full p-2 flex justify-between gap-2 items-center border border-primary shadow-xl rounded-xl bg-base-300 backdrop-blur-sm bg-opacity-80 max-w-md">+ <div className="fixed mx-auto bottom-20 sm:bottom-10 w-full pointer-events-none p-5 z-30">+ <div className="mx-auto pointer-events-auto p-2 flex justify-between gap-2 items-center border border-primary shadow-xl rounded-xl bg-base-300 backdrop-blur-sm bg-opacity-80 max-w-md"><i className="bi-stars text-2xl text-yellow-600 dark:text-yellow-500"></i><p className="w-4/5 text-center text-sm sm:text-base"><Transdiff --git a/lib/api/archiveHandler.ts b/lib/api/archiveHandler.tsindex bbbec787..c5e5f094 100644--- a/lib/api/archiveHandler.ts+++ b/lib/api/archiveHandler.ts@@ -2,7 +2,7 @@ import { LaunchOptions, chromium, devices } from "playwright";import { prisma } from "./db";import sendToWayback from "./preservationScheme/sendToWayback";import { Collection, Link, User } from "@prisma/client";-import validateUrlSize from "./validateUrlSize";+import fetchHeaders from "./fetchHeaders";import createFolder from "./storage/createFolder";import { removeFiles } from "./manageLinkFiles";import handleMonolith from "./preservationScheme/handleMonolith";@@ -65,17 +65,9 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) {(async () => {const user = link.collection?.owner;- const validatedUrl = link.url- ? await validateUrlSize(link.url)- : undefined;+ const header = link.url ? await fetchHeaders(link.url) : undefined;- if (- validatedUrl === null &&- process.env.IGNORE_URL_SIZE_LIMIT !== "true"- )- throw "Something went wrong while retrieving the file size.";-- const contentType = validatedUrl?.get("content-type");+ const contentType = header?.get("content-type");let linkType = "url";let imageExtension = "png";diff --git a/lib/api/controllers/links/postLink.ts b/lib/api/controllers/links/postLink.tsindex 8693f658..8056c111 100644--- a/lib/api/controllers/links/postLink.ts+++ b/lib/api/controllers/links/postLink.ts@@ -1,8 +1,7 @@import { prisma } from "@/lib/api/db";import { LinkIncludingShortenedCollectionAndTags } from "@/types/global";-import getTitle from "@/lib/shared/getTitle";+import fetchTitleAndHeaders from "@/lib/shared/fetchTitleAndHeaders";import createFolder from "@/lib/api/storage/createFolder";-import validateUrlSize from "../../validateUrlSize";import setLinkCollection from "../../setLinkCollection";const MAX_LINKS_PER_USER = Number(process.env.MAX_LINKS_PER_USER) || 30000;@@ -70,17 +69,12 @@ export default async function postLink(status: 400,};- const title =- !(link.name && link.name !== "") && link.url- ? await getTitle(link.url)- : "";+ const { title, headers } = await fetchTitleAndHeaders(link.url || "");const name =link.name && link.name !== "" ? link.name : link.url ? title : "";- const validatedUrl = link.url ? await validateUrlSize(link.url) : undefined;-- const contentType = validatedUrl?.get("content-type");+ const contentType = headers?.get("content-type");let linkType = "url";let imageExtension = "png";diff --git a/lib/api/validateUrlSize.ts b/lib/api/fetchHeaders.tssimilarity index 65%rename from lib/api/validateUrlSize.tsrename to lib/api/fetchHeaders.tsindex 8b418dc6..f423661a 100644--- a/lib/api/validateUrlSize.ts+++ b/lib/api/fetchHeaders.ts@@ -2,7 +2,7 @@ import fetch from "node-fetch";import https from "https";import { SocksProxyAgent } from "socks-proxy-agent";-export default async function validateUrlSize(url: string) {+export default async function fetchHeaders(url: string) {if (process.env.IGNORE_URL_SIZE_LIMIT === "true") return null;try {@@ -29,13 +29,17 @@ export default async function validateUrlSize(url: string) {};}- const response = await fetch(url, fetchOpts);+ const responsePromise = fetch(url, fetchOpts);- const totalSizeMB =- Number(response.headers.get("content-length")) / Math.pow(1024, 2);- if (totalSizeMB > Number(process.env.NEXT_PUBLIC_MAX_FILE_BUFFER || 10))- return null;- else return response.headers;+ const timeoutPromise = new Promise((_, reject) => {+ setTimeout(() => {+ reject(new Error("Fetch header timeout"));+ }, 10 * 1000); // Stop after 10 seconds+ });++ const response = await Promise.race([responsePromise, timeoutPromise]);++ return (response as Response)?.headers || null;} catch (err) {console.log(err);return null;diff --git a/lib/api/preservationScheme/imageHandler.ts b/lib/api/preservationScheme/imageHandler.tsindex 0b2194b8..273c0103 100644--- a/lib/api/preservationScheme/imageHandler.ts+++ b/lib/api/preservationScheme/imageHandler.ts@@ -1,6 +1,7 @@import { Link } from "@prisma/client";import { prisma } from "../db";import createFile from "../storage/createFile";+import generatePreview from "../generatePreview";const imageHandler = async ({ url, id }: Link, extension: string) => {const image = await fetch(url as string).then((res) => res.blob());@@ -18,6 +19,8 @@ const imageHandler = async ({ url, id }: Link, extension: string) => {});if (linkExists) {+ await generatePreview(buffer, linkExists.collectionId, id);+await createFile({data: buffer,filePath: `archives/${linkExists.collectionId}/${id}.${extension}`,diff --git a/lib/shared/getTitle.ts b/lib/shared/fetchTitleAndHeaders.tssimilarity index 83%rename from lib/shared/getTitle.tsrename to lib/shared/fetchTitleAndHeaders.tsindex 01488fdd..a7779646 100644--- a/lib/shared/getTitle.ts+++ b/lib/shared/fetchTitleAndHeaders.ts@@ -2,7 +2,7 @@ import fetch from "node-fetch";import https from "https";import { SocksProxyAgent } from "socks-proxy-agent";-export default async function getTitle(url: string) {+export default async function fetchTitleAndHeaders(url: string) {try {const httpsAgent = new https.Agent({rejectUnauthorized:@@ -41,12 +41,16 @@ export default async function getTitle(url: string) {// regular expression to find the <title> taglet match = text.match(/<title.*>([^<]*)<\/title>/);- if (match) return match[1];- else return "";++ const title = match[1] || "";+ const headers = (response as Response)?.headers || null;++ return { title, headers };} else {- return "";+ return { title: "", headers: null };}} catch (err) {console.log(err);+ return { title: "", headers: null };}}diff --git a/package.json b/package.jsonindex 0045802c..db8cf5d8 100644--- a/package.json+++ b/package.json@@ -60,7 +60,7 @@"next-i18next": "^15.3.0","node-fetch": "^2.7.0","nodemailer": "^6.9.3",- "playwright": "^1.43.1",+ "playwright": "^1.45.0","react": "18.2.0","react-colorful": "^5.6.1","react-dom": "18.2.0",@@ -77,7 +77,7 @@"zustand": "^4.3.8"},"devDependencies": {- "@playwright/test": "^1.43.1",+ "@playwright/test": "^1.45.0","@types/bcrypt": "^5.0.0","@types/dompurify": "^3.0.4","@types/jsdom": "^21.1.3",diff --git a/yarn.lock b/yarn.lockindex 55074b61..943096e4 100644--- a/yarn.lock+++ b/yarn.lock@@ -1293,12 +1293,12 @@tiny-glob "^0.2.9"tslib "^2.4.0"-"@playwright/test@^1.43.1":- version "1.43.1"- resolved "https://registry.yarnpkg.com/@playwright/test/-/test-1.43.1.tgz#16728a59eb8ce0f60472f98d8886d6cab0fa3e42"- integrity sha512-HgtQzFgNEEo4TE22K/X7sYTYNqEMMTZmFS8kTq6m8hXj+m1D8TgwgIbumHddJa9h4yl4GkKb8/bgAl2+g7eDgA==+"@playwright/test@^1.45.0":+ version "1.45.0"+ resolved "https://registry.yarnpkg.com/@playwright/test/-/test-1.45.0.tgz#790a66165a46466c0d7099dd260881802f5aba7e"+ integrity sha512-TVYsfMlGAaxeUllNkywbwek67Ncf8FRGn8ZlRdO291OL3NjG9oMbfVhyP82HQF0CZLMrYsvesqoUekxdWuF9Qw==dependencies:- playwright "1.43.1"+ playwright "1.45.0""@prisma/client@^4.16.2":version "4.16.2"@@ -5005,17 +5005,17 @@ pixelmatch@^4.0.2:dependencies:pngjs "^3.0.0"- version "1.43.1"- resolved "https://registry.yarnpkg.com/playwright-core/-/playwright-core-1.43.1.tgz#0eafef9994c69c02a1a3825a4343e56c99c03b02"- integrity sha512-EI36Mto2Vrx6VF7rm708qSnesVQKbxEWvPrfA1IPY6HgczBplDx7ENtx+K2n4kJ41sLLkuGfmb0ZLSSXlDhqPg==+ version "1.45.0"+ resolved "https://registry.yarnpkg.com/playwright-core/-/playwright-core-1.45.0.tgz#5741a670b7c9060ce06852c0051d84736fb94edc"+ integrity sha512-lZmHlFQ0VYSpAs43dRq1/nJ9G/6SiTI7VPqidld9TDefL9tX87bTKExWZZUF5PeRyqtXqd8fQi2qmfIedkwsNQ==[email protected], playwright@^1.43.1:- version "1.43.1"- resolved "https://registry.yarnpkg.com/playwright/-/playwright-1.43.1.tgz#8ad08984ac66c9ef3d0db035be54dd7ec9f1c7d9"- integrity sha512-V7SoH0ai2kNt1Md9E3Gwas5B9m8KR2GVvwZnAI6Pg0m3sh7UvgiYhRrhsziCmqMJNouPckiOhk8T+9bSAK0VIA==[email protected], playwright@^1.45.0:+ version "1.45.0"+ resolved "https://registry.yarnpkg.com/playwright/-/playwright-1.45.0.tgz#400c709c64438690f13705cb9c88ef93089c5c27"+ integrity sha512-4z3ac3plDfYzGB6r0Q3LF8POPR20Z8D0aXcxbJvmfMgSSq1hkcgvFRXJk9rUq5H/MJ0Ktal869hhOdI/zUTeLA==dependencies:- playwright-core "1.43.1"+ playwright-core "1.45.0"optionalDependencies:fsevents "2.3.2"