⚡️⚡️⚡️ Self-hosted collaborative bookmark manager to collect, read, annotate, and fully preserve what matters, all in one place.
TypeScript bookmarkbookmark-managercollaborationnextjsreact-nativeread-it-laterself-hostedtypescript
minimize the usage of external dependencies
daniel31x134bf220c78662
diff --git a/lib/api/controllers/links/postLink.ts b/lib/api/controllers/links/postLink.tsindex 3159ca64..fab02b38 100644--- a/lib/api/controllers/links/postLink.ts+++ b/lib/api/controllers/links/postLink.ts@@ -66,7 +66,7 @@ export default async function postLink(if (hasTooManyLinks) {return {- response: `Your subscription have reached the maximum number of links allowed.`,+ response: `Your subscription has reached the maximum number of links allowed.`,status: 400,};}diff --git a/lib/api/controllers/migration/importFromHTMLFile.ts b/lib/api/controllers/migration/importFromHTMLFile.tsindex f255bac0..365e74db 100644--- a/lib/api/controllers/migration/importFromHTMLFile.ts+++ b/lib/api/controllers/migration/importFromHTMLFile.ts@@ -3,12 +3,11 @@ import createFolder from "@/lib/api/storage/createFolder";import { JSDOM } from "jsdom";import { parse, Node, Element, TextNode } from "himalaya";import { hasPassedLimit } from "../../verifyCapacity";-import { Readable } from "stream";-import streamToBlob from "stream-to-blob";+import streamToBlob from "@/lib/shared/streamToBlob";export default async function importFromHTMLFile(userId: number,- rawStream: Readable+ rawStream: ReadableStream) {const rawData: Blob = await streamToBlob(rawStream);const dom = new JSDOM(await rawData.arrayBuffer());@@ -26,7 +25,7 @@ export default async function importFromHTMLFile(if (hasTooManyLinks) {return {- response: `Your subscription have reached the maximum number of links allowed.`,+ response: `Your subscription has reached the maximum number of links allowed.`,status: 400,};}diff --git a/lib/api/controllers/migration/importFromLinkwarden.ts b/lib/api/controllers/migration/importFromLinkwarden.tsindex a191d69c..fc521612 100644--- a/lib/api/controllers/migration/importFromLinkwarden.ts+++ b/lib/api/controllers/migration/importFromLinkwarden.ts@@ -2,12 +2,11 @@ import { prisma } from "@/lib/api/db";import { Backup } from "@/types/global";import createFolder from "@/lib/api/storage/createFolder";import { hasPassedLimit } from "../../verifyCapacity";-import { Readable } from "stream";-import streamToBlob from "stream-to-blob";+import streamToBlob from "@/lib/shared/streamToBlob";export default async function importFromLinkwarden(userId: number,- rawStream: Readable+ rawStream: ReadableStream) {const rawData: Blob = await streamToBlob(rawStream);const data: Backup = JSON.parse(await rawData.text());@@ -22,7 +21,7 @@ export default async function importFromLinkwarden(if (hasTooManyLinks) {return {- response: `Your subscription have reached the maximum number of links allowed.`,+ response: `Your subscription has reached the maximum number of links allowed.`,status: 400,};}diff --git a/lib/api/controllers/migration/importFromOmnivore.ts b/lib/api/controllers/migration/importFromOmnivore.tsindex 83ba8357..3214524b 100644--- a/lib/api/controllers/migration/importFromOmnivore.ts+++ b/lib/api/controllers/migration/importFromOmnivore.ts@@ -1,11 +1,8 @@import { prisma } from "@/lib/api/db";import createFolder from "@/lib/api/storage/createFolder";-import { JSDOM } from "jsdom";-import { parse, Node, Element, TextNode } from "himalaya";import { hasPassedLimit } from "../../verifyCapacity";-import { ZipReader, BlobReader, TextWriter } from "@zip.js/zip.js";-import { Readable } from "stream";-import streamToBlob from "stream-to-blob";+import streamToBlob from "@/lib/shared/streamToBlob";+import JSZip from "jszip";type OmnivoreMetadata = {id: string;@@ -25,13 +22,13 @@ type OmnivoreMetadata = {export default async function importFromOmnivore(userId: number,- rawStream: Readable+ rawStream: ReadableStream) {const rawData: Blob = await streamToBlob(rawStream);- const zipFileReader = new BlobReader(rawData);- const importArchive = new ZipReader(zipFileReader);- const zipEntries = await importArchive.getEntries();+ const arrayBuffer = await rawData.arrayBuffer();++ const zip = await JSZip.loadAsync(arrayBuffer);await prisma.$transaction(async () => {const omnivoreCollection = await prisma.collection.create({@@ -52,18 +49,17 @@ export default async function importFromOmnivore(createFolder({ filePath: `archives/${omnivoreCollection.id}` });- for (const entry of zipEntries) {- if (entry.filename.startsWith("metadata_")) {- console.log(`Getting metadata from ${entry.filename}`);+ for (const filename of Object.keys(zip.files)) {+ if (filename.startsWith("metadata_")) {+ console.log(`Getting metadata from ${filename}`);- const jsonWriter = new TextWriter();- const jsonMetadatString: string = await entry.getData(jsonWriter);- const metadata: OmnivoreMetadata = JSON.parse(jsonMetadatString);+ const jsonMetadataString = await zip.files[filename].async("string");+ const metadata: OmnivoreMetadata = JSON.parse(jsonMetadataString);const hasTooManyLinks = await hasPassedLimit(userId, metadata.length);if (hasTooManyLinks) {return {- response: `Your subscription have reached the maximum number of links allowed.`,+ response: `Your subscription has reached the maximum number of links allowed.`,status: 400,};}@@ -75,7 +71,7 @@ export default async function importFromOmnivore(continue;}console.log("Extracting text data");- //const textData = await importArchive.get(`content/${data.slug}.html`)?.get_string()+ // const textData = await zip.file(`content/${data.slug}.html`)?.async("string") ?? "";await prisma.link.create({data: {@@ -121,5 +117,6 @@ export default async function importFromOmnivore(}}});+return { response: "Success.", status: 200 };}diff --git a/lib/api/controllers/migration/importFromWallabag.ts b/lib/api/controllers/migration/importFromWallabag.tsindex 0c5ad70b..23b8edef 100644--- a/lib/api/controllers/migration/importFromWallabag.ts+++ b/lib/api/controllers/migration/importFromWallabag.ts@@ -1,8 +1,7 @@import { prisma } from "@/lib/api/db";import createFolder from "@/lib/api/storage/createFolder";import { hasPassedLimit } from "../../verifyCapacity";-import { Readable } from "stream";-import streamToBlob from "stream-to-blob";+import streamToBlob from "@/lib/shared/streamToBlob";type WallabagBackup = {is_archived: number;@@ -29,7 +28,7 @@ type WallabagBackup = {export default async function importFromWallabag(userId: number,- rawStream: Readable+ rawStream: ReadableStream) {const rawData: Blob = await streamToBlob(rawStream);const data: WallabagBackup = JSON.parse(await rawData.text());@@ -42,7 +41,7 @@ export default async function importFromWallabag(if (hasTooManyLinks) {return {- response: `Your subscription have reached the maximum number of links allowed.`,+ response: `Your subscription has reached the maximum number of links allowed.`,status: 400,};}diff --git a/lib/shared/streamToBlob.ts b/lib/shared/streamToBlob.tsnew file mode 100644index 00000000..7fdabc8b--- /dev/null+++ b/lib/shared/streamToBlob.ts@@ -0,0 +1,20 @@+export default async function streamToBlob(+ stream: ReadableStream,+ mimeType?: string+): Promise<Blob> {+ if (mimeType != null && typeof mimeType !== "string") {+ throw new Error("Invalid mimeType, expected a string.");+ }++ const chunks: Uint8Array[] = [];+ for await (const chunk of stream as any) {+ // Ensure the chunk is a Uint8Array+ chunks.push(+ typeof chunk === "string"+ ? new Uint8Array(Buffer.from(chunk))+ : new Uint8Array(chunk)+ );+ }++ return new Blob(chunks, { type: mimeType });+}diff --git a/package.json b/package.jsonindex 6cd144c4..526a45b1 100644--- a/package.json+++ b/package.json@@ -37,8 +37,6 @@"@types/nodemailer": "^6.4.8","@types/react": "18.2.14","@types/react-dom": "18.2.7",- "@types/stream-to-blob": "^2.0.0",- "@zip.js/zip.js": "^2.7.53","axios": "^1.5.1","bcrypt": "^5.1.0","bootstrap-icons": "^1.11.2",@@ -60,6 +58,7 @@"i18next": "^23.11.5","jimp": "^0.22.10","jsdom": "^22.1.0",+ "jszip": "^3.10.1","lottie-web": "^5.12.2","micro": "^10.0.1","next": "13.4.12",@@ -82,7 +81,6 @@"rss-parser": "^3.13.0","socks-proxy-agent": "^8.0.2","stream-size": "^0.0.6",- "stream-to-blob": "^2.0.1","stripe": "^12.13.0","tailwind-merge": "^2.3.0","vaul": "^1.1.1",diff --git a/yarn.lock b/yarn.lockindex ad47913e..fcc41c2a 100644--- a/yarn.lock+++ b/yarn.lock@@ -2119,13 +2119,6 @@"@types/glob" "~7.2.0""@types/node" "*"-"@types/stream-to-blob@^2.0.0":- version "2.0.0"- resolved "https://registry.yarnpkg.com/@types/stream-to-blob/-/stream-to-blob-2.0.0.tgz#b38b6eb763831a37b91dfe4a5854f9da528acb5c"- integrity sha512-tZpPRVwoUMe4ZHr0xt2+RyvkqKFoKzdyCUuzW3NxC4HKxbNEwCMLZiumMuZ+FaizAQJx3rUXVJNXpMCnnOffAw==- dependencies:- "@types/node" "*"-"@types/tough-cookie@*":version "4.0.3"resolved "https://registry.yarnpkg.com/@types/tough-cookie/-/tough-cookie-4.0.3.tgz#3d06b6769518450871fbc40770b7586334bdfd90"@@ -2180,11 +2173,6 @@"@typescript-eslint/types" "5.49.0"eslint-visitor-keys "^3.3.0"-"@zip.js/zip.js@^2.7.53":- version "2.7.53"- resolved "https://registry.yarnpkg.com/@zip.js/zip.js/-/zip.js-2.7.53.tgz#bf88e90d8eed562182c01339643bc405446b0578"- integrity sha512-G6Bl5wN9EXXVaTUIox71vIX5Z454zEBe+akKpV4m1tUboIctT5h7ID3QXCJd/Lfy2rSvmkTmZIucf1jGRR4f5A==-abab@^2.0.6:version "2.0.6"resolved "https://registry.yarnpkg.com/abab/-/abab-2.0.6.tgz#41b80f2c871d19686216b82309231cfd3cb3d291"@@ -2749,6 +2737,11 @@ [email protected]:resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7"integrity sha512-3lqz5YjWTYnW6dlDa5TLaTCcShfar1e40rmcJVwCBJC6mWlFuj0eCHIElmG1g5kyuJ/GD+8Wn4FFCcz4gJPfaQ==+core-util-is@~1.0.0:+ version "1.0.3"+ resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.3.tgz#a6042d3634c2b27e9328f837b965fac83808db85"+ integrity sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==+cosmiconfig@^7.0.0:version "7.1.0"resolved "https://registry.yarnpkg.com/cosmiconfig/-/cosmiconfig-7.1.0.tgz#1443b9afa596b670082ea46cbd8f6a62b84635f6"@@ -4012,6 +4005,11 @@ image-q@^4.0.0:dependencies:"@types/node" "16.9.1"+immediate@~3.0.5:+ version "3.0.6"+ resolved "https://registry.yarnpkg.com/immediate/-/immediate-3.0.6.tgz#9db1dbd0faf8de6fbe0f5dd5e56bb606280de69b"+ integrity sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==+import-fresh@^3.2.1:version "3.3.0"resolved "https://registry.yarnpkg.com/import-fresh/-/import-fresh-3.3.0.tgz#37162c25fcb9ebaa2e6e53d5b4d88ce17d9e0c2b"@@ -4033,7 +4031,7 @@ inflight@^1.0.4:once "^1.3.0"wrappy "1"-inherits@2, [email protected], inherits@^2.0.3:+inherits@2, [email protected], inherits@^2.0.3, inherits@~2.0.3:version "2.0.4"resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c"integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==@@ -4276,6 +4274,11 @@ isarray@^2.0.5:resolved "https://registry.yarnpkg.com/isarray/-/isarray-2.0.5.tgz#8af1e4c1221244cc62459faf38940d4e644a5723"integrity sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==+isarray@~1.0.0:+ version "1.0.0"+ resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11"+ integrity sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==+isexe@^2.0.0:version "2.0.0"resolved "https://registry.yarnpkg.com/isexe/-/isexe-2.0.0.tgz#e8fbf374dc556ff8947a10dcb0572d633f2cfa10"@@ -4420,6 +4423,16 @@ jsprim@^1.2.2:array-includes "^3.1.5"object.assign "^4.1.3"+jszip@^3.10.1:+ version "3.10.1"+ resolved "https://registry.yarnpkg.com/jszip/-/jszip-3.10.1.tgz#34aee70eb18ea1faec2f589208a157d1feb091c2"+ integrity sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g==+ dependencies:+ lie "~3.3.0"+ pako "~1.0.2"+ readable-stream "~2.3.6"+ setimmediate "^1.0.5"+language-subtag-registry@~0.3.2:version "0.3.22"resolved "https://registry.yarnpkg.com/language-subtag-registry/-/language-subtag-registry-0.3.22.tgz#2e1500861b2e457eba7e7ae86877cbd08fa1fd1d"@@ -4440,6 +4453,13 @@ levn@^0.4.1:prelude-ls "^1.2.1"type-check "~0.4.0"+lie@~3.3.0:+ version "3.3.0"+ resolved "https://registry.yarnpkg.com/lie/-/lie-3.3.0.tgz#dcf82dee545f46074daf200c7c1c5a08e0f40f6a"+ integrity sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==+ dependencies:+ immediate "~3.0.5"+lilconfig@^2.0.5, lilconfig@^2.1.0:version "2.1.0"resolved "https://registry.yarnpkg.com/lilconfig/-/lilconfig-2.1.0.tgz#78e23ac89ebb7e1bfbf25b18043de756548e7f52"@@ -4954,7 +4974,7 @@ p-locate@^5.0.0:dependencies:p-limit "^3.0.2"-pako@^1.0.11:+pako@^1.0.11, pako@~1.0.2:version "1.0.11"resolved "https://registry.yarnpkg.com/pako/-/pako-1.0.11.tgz#6c9599d340d54dfd3946380252a35705a6b992bf"integrity sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==@@ -5209,6 +5229,11 @@ prisma@^5.21.1:optionalDependencies:fsevents "2.3.3"+process-nextick-args@~2.0.0:+ version "2.0.1"+ resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz#7820d9b16120cc55ca9ae7792680ae7dba6d7fe2"+ integrity sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==+process@^0.11.10:version "0.11.10"resolved "https://registry.yarnpkg.com/process/-/process-0.11.10.tgz#7332300e840161bda3e69a1d1d91a7d4bc16f182"@@ -5448,6 +5473,19 @@ readable-stream@^3.6.0:string_decoder "^1.1.1"util-deprecate "^1.0.1"+readable-stream@~2.3.6:+ version "2.3.8"+ resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.8.tgz#91125e8042bba1b9887f49345f6277027ce8be9b"+ integrity sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==+ dependencies:+ core-util-is "~1.0.0"+ inherits "~2.0.3"+ isarray "~1.0.0"+ process-nextick-args "~2.0.0"+ safe-buffer "~5.1.1"+ string_decoder "~1.1.1"+ util-deprecate "~1.0.1"+readable-web-to-node-stream@^3.0.0:version "3.0.2"resolved "https://registry.yarnpkg.com/readable-web-to-node-stream/-/readable-web-to-node-stream-3.0.2.tgz#5d52bb5df7b54861fd48d015e93a2cb87b3ee0bb"@@ -5591,6 +5629,11 @@ safe-buffer@^5.0.1, safe-buffer@^5.1.2, safe-buffer@~5.2.0:resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6"integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==+safe-buffer@~5.1.0, safe-buffer@~5.1.1:+ version "5.1.2"+ resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d"+ integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==+safe-regex-test@^1.0.0:version "1.0.0"resolved "https://registry.yarnpkg.com/safe-regex-test/-/safe-regex-test-1.0.0.tgz#793b874d524eb3640d1873aad03596db2d4f2295"@@ -5641,6 +5684,11 @@ set-blocking@^2.0.0:resolved "https://registry.yarnpkg.com/set-blocking/-/set-blocking-2.0.0.tgz#045f9782d011ae9a6803ddd382b24392b3d890f7"integrity sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw==+setimmediate@^1.0.5:+ version "1.0.5"+ resolved "https://registry.yarnpkg.com/setimmediate/-/setimmediate-1.0.5.tgz#290cbb232e306942d7d7ea9b83732ab7856f8285"+ integrity sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==+version "1.1.1"resolved "https://registry.yarnpkg.com/setprototypeof/-/setprototypeof-1.1.1.tgz#7e95acb24aa92f5885e0abef5ba131330d4ae683"@@ -5773,11 +5821,6 @@ stream-size@^0.0.6:resolved "https://registry.yarnpkg.com/stream-size/-/stream-size-0.0.6.tgz#05d7132e2d2fdb91b533a6e50aff882f040d7679"integrity sha512-TZsxxZzKPkiD7fxcKx0Ze9s0+WnGGbX1yWZNoXXmN6YjyEu/62pYkLKo7P627N27BT9kI93ZfZAx38brr/hq+Q==-stream-to-blob@^2.0.1:- version "2.0.1"- resolved "https://registry.yarnpkg.com/stream-to-blob/-/stream-to-blob-2.0.1.tgz#59ab71d7a7f0bfb899570e886e44d39f4ac4381a"- integrity sha512-GXlqXt3svqwIVWoICenix5Poxi4KbCF0BdXXUbpU1X4vq1V8wmjiEIU3aFJzCGNFpKxfbnG0uoowS3nKUgSPYg==-streamsearch@^1.1.0:version "1.1.0"resolved "https://registry.yarnpkg.com/streamsearch/-/streamsearch-1.1.0.tgz#404dd1e2247ca94af554e841a8ef0eaa238da764"@@ -5831,6 +5874,13 @@ string_decoder@^1.1.1:dependencies:safe-buffer "~5.2.0"+string_decoder@~1.1.1:+ version "1.1.1"+ resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.1.1.tgz#9cf1611ba62685d7030ae9e4ba34149c3af03fc8"+ integrity sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==+ dependencies:+ safe-buffer "~5.1.0"+strip-ansi@^6.0.0, strip-ansi@^6.0.1:version "6.0.1"resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"@@ -6298,7 +6348,7 @@ utif2@^4.0.1:dependencies:pako "^1.0.11"-util-deprecate@^1.0.1, util-deprecate@^1.0.2:+util-deprecate@^1.0.1, util-deprecate@^1.0.2, util-deprecate@~1.0.1:version "1.0.2"resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf"integrity sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==