Skip to content

⚡️⚡️⚡️ Self-hosted collaborative bookmark manager to collect, read, annotate, and fully preserve what matters, all in one place.

minimize the usage of external dependencies

daniel31x134bf220c78662
diff --git a/lib/api/controllers/links/postLink.ts b/lib/api/controllers/links/postLink.ts
index 3159ca64..fab02b38 100644
--- a/lib/api/controllers/links/postLink.ts
+++ b/lib/api/controllers/links/postLink.ts
@@ -66,7 +66,7 @@ export default async function postLink(
if (hasTooManyLinks) {
return {
- response: `Your subscription have reached the maximum number of links allowed.`,
+ response: `Your subscription has reached the maximum number of links allowed.`,
status: 400,
};
}
diff --git a/lib/api/controllers/migration/importFromHTMLFile.ts b/lib/api/controllers/migration/importFromHTMLFile.ts
index f255bac0..365e74db 100644
--- a/lib/api/controllers/migration/importFromHTMLFile.ts
+++ b/lib/api/controllers/migration/importFromHTMLFile.ts
@@ -3,12 +3,11 @@ import createFolder from "@/lib/api/storage/createFolder";
import { JSDOM } from "jsdom";
import { parse, Node, Element, TextNode } from "himalaya";
import { hasPassedLimit } from "../../verifyCapacity";
-import { Readable } from "stream";
-import streamToBlob from "stream-to-blob";
+import streamToBlob from "@/lib/shared/streamToBlob";
export default async function importFromHTMLFile(
userId: number,
- rawStream: Readable
+ rawStream: ReadableStream
) {
const rawData: Blob = await streamToBlob(rawStream);
const dom = new JSDOM(await rawData.arrayBuffer());
@@ -26,7 +25,7 @@ export default async function importFromHTMLFile(
if (hasTooManyLinks) {
return {
- response: `Your subscription have reached the maximum number of links allowed.`,
+ response: `Your subscription has reached the maximum number of links allowed.`,
status: 400,
};
}
diff --git a/lib/api/controllers/migration/importFromLinkwarden.ts b/lib/api/controllers/migration/importFromLinkwarden.ts
index a191d69c..fc521612 100644
--- a/lib/api/controllers/migration/importFromLinkwarden.ts
+++ b/lib/api/controllers/migration/importFromLinkwarden.ts
@@ -2,12 +2,11 @@ import { prisma } from "@/lib/api/db";
import { Backup } from "@/types/global";
import createFolder from "@/lib/api/storage/createFolder";
import { hasPassedLimit } from "../../verifyCapacity";
-import { Readable } from "stream";
-import streamToBlob from "stream-to-blob";
+import streamToBlob from "@/lib/shared/streamToBlob";
export default async function importFromLinkwarden(
userId: number,
- rawStream: Readable
+ rawStream: ReadableStream
) {
const rawData: Blob = await streamToBlob(rawStream);
const data: Backup = JSON.parse(await rawData.text());
@@ -22,7 +21,7 @@ export default async function importFromLinkwarden(
if (hasTooManyLinks) {
return {
- response: `Your subscription have reached the maximum number of links allowed.`,
+ response: `Your subscription has reached the maximum number of links allowed.`,
status: 400,
};
}
diff --git a/lib/api/controllers/migration/importFromOmnivore.ts b/lib/api/controllers/migration/importFromOmnivore.ts
index 83ba8357..3214524b 100644
--- a/lib/api/controllers/migration/importFromOmnivore.ts
+++ b/lib/api/controllers/migration/importFromOmnivore.ts
@@ -1,11 +1,8 @@
import { prisma } from "@/lib/api/db";
import createFolder from "@/lib/api/storage/createFolder";
-import { JSDOM } from "jsdom";
-import { parse, Node, Element, TextNode } from "himalaya";
import { hasPassedLimit } from "../../verifyCapacity";
-import { ZipReader, BlobReader, TextWriter } from "@zip.js/zip.js";
-import { Readable } from "stream";
-import streamToBlob from "stream-to-blob";
+import streamToBlob from "@/lib/shared/streamToBlob";
+import JSZip from "jszip";
type OmnivoreMetadata = {
id: string;
@@ -25,13 +22,13 @@ type OmnivoreMetadata = {
export default async function importFromOmnivore(
userId: number,
- rawStream: Readable
+ rawStream: ReadableStream
) {
const rawData: Blob = await streamToBlob(rawStream);
- const zipFileReader = new BlobReader(rawData);
- const importArchive = new ZipReader(zipFileReader);
- const zipEntries = await importArchive.getEntries();
+ const arrayBuffer = await rawData.arrayBuffer();
+
+ const zip = await JSZip.loadAsync(arrayBuffer);
await prisma.$transaction(async () => {
const omnivoreCollection = await prisma.collection.create({
@@ -52,18 +49,17 @@ export default async function importFromOmnivore(
createFolder({ filePath: `archives/${omnivoreCollection.id}` });
- for (const entry of zipEntries) {
- if (entry.filename.startsWith("metadata_")) {
- console.log(`Getting metadata from ${entry.filename}`);
+ for (const filename of Object.keys(zip.files)) {
+ if (filename.startsWith("metadata_")) {
+ console.log(`Getting metadata from ${filename}`);
- const jsonWriter = new TextWriter();
- const jsonMetadatString: string = await entry.getData(jsonWriter);
- const metadata: OmnivoreMetadata = JSON.parse(jsonMetadatString);
+ const jsonMetadataString = await zip.files[filename].async("string");
+ const metadata: OmnivoreMetadata = JSON.parse(jsonMetadataString);
const hasTooManyLinks = await hasPassedLimit(userId, metadata.length);
if (hasTooManyLinks) {
return {
- response: `Your subscription have reached the maximum number of links allowed.`,
+ response: `Your subscription has reached the maximum number of links allowed.`,
status: 400,
};
}
@@ -75,7 +71,7 @@ export default async function importFromOmnivore(
continue;
}
console.log("Extracting text data");
- //const textData = await importArchive.get(`content/${data.slug}.html`)?.get_string()
+ // const textData = await zip.file(`content/${data.slug}.html`)?.async("string") ?? "";
await prisma.link.create({
data: {
@@ -121,5 +117,6 @@ export default async function importFromOmnivore(
}
}
});
+
return { response: "Success.", status: 200 };
}
diff --git a/lib/api/controllers/migration/importFromWallabag.ts b/lib/api/controllers/migration/importFromWallabag.ts
index 0c5ad70b..23b8edef 100644
--- a/lib/api/controllers/migration/importFromWallabag.ts
+++ b/lib/api/controllers/migration/importFromWallabag.ts
@@ -1,8 +1,7 @@
import { prisma } from "@/lib/api/db";
import createFolder from "@/lib/api/storage/createFolder";
import { hasPassedLimit } from "../../verifyCapacity";
-import { Readable } from "stream";
-import streamToBlob from "stream-to-blob";
+import streamToBlob from "@/lib/shared/streamToBlob";
type WallabagBackup = {
is_archived: number;
@@ -29,7 +28,7 @@ type WallabagBackup = {
export default async function importFromWallabag(
userId: number,
- rawStream: Readable
+ rawStream: ReadableStream
) {
const rawData: Blob = await streamToBlob(rawStream);
const data: WallabagBackup = JSON.parse(await rawData.text());
@@ -42,7 +41,7 @@ export default async function importFromWallabag(
if (hasTooManyLinks) {
return {
- response: `Your subscription have reached the maximum number of links allowed.`,
+ response: `Your subscription has reached the maximum number of links allowed.`,
status: 400,
};
}
diff --git a/lib/shared/streamToBlob.ts b/lib/shared/streamToBlob.ts
new file mode 100644
index 00000000..7fdabc8b
--- /dev/null
+++ b/lib/shared/streamToBlob.ts
@@ -0,0 +1,20 @@
+export default async function streamToBlob(
+ stream: ReadableStream,
+ mimeType?: string
+): Promise<Blob> {
+ if (mimeType != null && typeof mimeType !== "string") {
+ throw new Error("Invalid mimeType, expected a string.");
+ }
+
+ const chunks: Uint8Array[] = [];
+ for await (const chunk of stream as any) {
+ // Ensure the chunk is a Uint8Array
+ chunks.push(
+ typeof chunk === "string"
+ ? new Uint8Array(Buffer.from(chunk))
+ : new Uint8Array(chunk)
+ );
+ }
+
+ return new Blob(chunks, { type: mimeType });
+}
diff --git a/package.json b/package.json
index 6cd144c4..526a45b1 100644
--- a/package.json
+++ b/package.json
@@ -37,8 +37,6 @@
"@types/nodemailer": "^6.4.8",
"@types/react": "18.2.14",
"@types/react-dom": "18.2.7",
- "@types/stream-to-blob": "^2.0.0",
- "@zip.js/zip.js": "^2.7.53",
"axios": "^1.5.1",
"bcrypt": "^5.1.0",
"bootstrap-icons": "^1.11.2",
@@ -60,6 +58,7 @@
"i18next": "^23.11.5",
"jimp": "^0.22.10",
"jsdom": "^22.1.0",
+ "jszip": "^3.10.1",
"lottie-web": "^5.12.2",
"micro": "^10.0.1",
"next": "13.4.12",
@@ -82,7 +81,6 @@
"rss-parser": "^3.13.0",
"socks-proxy-agent": "^8.0.2",
"stream-size": "^0.0.6",
- "stream-to-blob": "^2.0.1",
"stripe": "^12.13.0",
"tailwind-merge": "^2.3.0",
"vaul": "^1.1.1",
diff --git a/yarn.lock b/yarn.lock
index ad47913e..fcc41c2a 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -2119,13 +2119,6 @@
"@types/glob" "~7.2.0"
"@types/node" "*"
-"@types/stream-to-blob@^2.0.0":
- version "2.0.0"
- resolved "https://registry.yarnpkg.com/@types/stream-to-blob/-/stream-to-blob-2.0.0.tgz#b38b6eb763831a37b91dfe4a5854f9da528acb5c"
- integrity sha512-tZpPRVwoUMe4ZHr0xt2+RyvkqKFoKzdyCUuzW3NxC4HKxbNEwCMLZiumMuZ+FaizAQJx3rUXVJNXpMCnnOffAw==
- dependencies:
- "@types/node" "*"
-
"@types/tough-cookie@*":
version "4.0.3"
resolved "https://registry.yarnpkg.com/@types/tough-cookie/-/tough-cookie-4.0.3.tgz#3d06b6769518450871fbc40770b7586334bdfd90"
@@ -2180,11 +2173,6 @@
"@typescript-eslint/types" "5.49.0"
eslint-visitor-keys "^3.3.0"
-"@zip.js/zip.js@^2.7.53":
- version "2.7.53"
- resolved "https://registry.yarnpkg.com/@zip.js/zip.js/-/zip.js-2.7.53.tgz#bf88e90d8eed562182c01339643bc405446b0578"
- integrity sha512-G6Bl5wN9EXXVaTUIox71vIX5Z454zEBe+akKpV4m1tUboIctT5h7ID3QXCJd/Lfy2rSvmkTmZIucf1jGRR4f5A==
-
abab@^2.0.6:
version "2.0.6"
resolved "https://registry.yarnpkg.com/abab/-/abab-2.0.6.tgz#41b80f2c871d19686216b82309231cfd3cb3d291"
@@ -2749,6 +2737,11 @@ [email protected]:
resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7"
integrity sha512-3lqz5YjWTYnW6dlDa5TLaTCcShfar1e40rmcJVwCBJC6mWlFuj0eCHIElmG1g5kyuJ/GD+8Wn4FFCcz4gJPfaQ==
+core-util-is@~1.0.0:
+ version "1.0.3"
+ resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.3.tgz#a6042d3634c2b27e9328f837b965fac83808db85"
+ integrity sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==
+
cosmiconfig@^7.0.0:
version "7.1.0"
resolved "https://registry.yarnpkg.com/cosmiconfig/-/cosmiconfig-7.1.0.tgz#1443b9afa596b670082ea46cbd8f6a62b84635f6"
@@ -4012,6 +4005,11 @@ image-q@^4.0.0:
dependencies:
"@types/node" "16.9.1"
+immediate@~3.0.5:
+ version "3.0.6"
+ resolved "https://registry.yarnpkg.com/immediate/-/immediate-3.0.6.tgz#9db1dbd0faf8de6fbe0f5dd5e56bb606280de69b"
+ integrity sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==
+
import-fresh@^3.2.1:
version "3.3.0"
resolved "https://registry.yarnpkg.com/import-fresh/-/import-fresh-3.3.0.tgz#37162c25fcb9ebaa2e6e53d5b4d88ce17d9e0c2b"
@@ -4033,7 +4031,7 @@ inflight@^1.0.4:
once "^1.3.0"
wrappy "1"
-inherits@2, [email protected], inherits@^2.0.3:
+inherits@2, [email protected], inherits@^2.0.3, inherits@~2.0.3:
version "2.0.4"
resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c"
integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==
@@ -4276,6 +4274,11 @@ isarray@^2.0.5:
resolved "https://registry.yarnpkg.com/isarray/-/isarray-2.0.5.tgz#8af1e4c1221244cc62459faf38940d4e644a5723"
integrity sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==
+isarray@~1.0.0:
+ version "1.0.0"
+ resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11"
+ integrity sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==
+
isexe@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/isexe/-/isexe-2.0.0.tgz#e8fbf374dc556ff8947a10dcb0572d633f2cfa10"
@@ -4420,6 +4423,16 @@ jsprim@^1.2.2:
array-includes "^3.1.5"
object.assign "^4.1.3"
+jszip@^3.10.1:
+ version "3.10.1"
+ resolved "https://registry.yarnpkg.com/jszip/-/jszip-3.10.1.tgz#34aee70eb18ea1faec2f589208a157d1feb091c2"
+ integrity sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g==
+ dependencies:
+ lie "~3.3.0"
+ pako "~1.0.2"
+ readable-stream "~2.3.6"
+ setimmediate "^1.0.5"
+
language-subtag-registry@~0.3.2:
version "0.3.22"
resolved "https://registry.yarnpkg.com/language-subtag-registry/-/language-subtag-registry-0.3.22.tgz#2e1500861b2e457eba7e7ae86877cbd08fa1fd1d"
@@ -4440,6 +4453,13 @@ levn@^0.4.1:
prelude-ls "^1.2.1"
type-check "~0.4.0"
+lie@~3.3.0:
+ version "3.3.0"
+ resolved "https://registry.yarnpkg.com/lie/-/lie-3.3.0.tgz#dcf82dee545f46074daf200c7c1c5a08e0f40f6a"
+ integrity sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==
+ dependencies:
+ immediate "~3.0.5"
+
lilconfig@^2.0.5, lilconfig@^2.1.0:
version "2.1.0"
resolved "https://registry.yarnpkg.com/lilconfig/-/lilconfig-2.1.0.tgz#78e23ac89ebb7e1bfbf25b18043de756548e7f52"
@@ -4954,7 +4974,7 @@ p-locate@^5.0.0:
dependencies:
p-limit "^3.0.2"
-pako@^1.0.11:
+pako@^1.0.11, pako@~1.0.2:
version "1.0.11"
resolved "https://registry.yarnpkg.com/pako/-/pako-1.0.11.tgz#6c9599d340d54dfd3946380252a35705a6b992bf"
integrity sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==
@@ -5209,6 +5229,11 @@ prisma@^5.21.1:
optionalDependencies:
fsevents "2.3.3"
+process-nextick-args@~2.0.0:
+ version "2.0.1"
+ resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz#7820d9b16120cc55ca9ae7792680ae7dba6d7fe2"
+ integrity sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==
+
process@^0.11.10:
version "0.11.10"
resolved "https://registry.yarnpkg.com/process/-/process-0.11.10.tgz#7332300e840161bda3e69a1d1d91a7d4bc16f182"
@@ -5448,6 +5473,19 @@ readable-stream@^3.6.0:
string_decoder "^1.1.1"
util-deprecate "^1.0.1"
+readable-stream@~2.3.6:
+ version "2.3.8"
+ resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.8.tgz#91125e8042bba1b9887f49345f6277027ce8be9b"
+ integrity sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==
+ dependencies:
+ core-util-is "~1.0.0"
+ inherits "~2.0.3"
+ isarray "~1.0.0"
+ process-nextick-args "~2.0.0"
+ safe-buffer "~5.1.1"
+ string_decoder "~1.1.1"
+ util-deprecate "~1.0.1"
+
readable-web-to-node-stream@^3.0.0:
version "3.0.2"
resolved "https://registry.yarnpkg.com/readable-web-to-node-stream/-/readable-web-to-node-stream-3.0.2.tgz#5d52bb5df7b54861fd48d015e93a2cb87b3ee0bb"
@@ -5591,6 +5629,11 @@ safe-buffer@^5.0.1, safe-buffer@^5.1.2, safe-buffer@~5.2.0:
resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6"
integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==
+safe-buffer@~5.1.0, safe-buffer@~5.1.1:
+ version "5.1.2"
+ resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d"
+ integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==
+
safe-regex-test@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/safe-regex-test/-/safe-regex-test-1.0.0.tgz#793b874d524eb3640d1873aad03596db2d4f2295"
@@ -5641,6 +5684,11 @@ set-blocking@^2.0.0:
resolved "https://registry.yarnpkg.com/set-blocking/-/set-blocking-2.0.0.tgz#045f9782d011ae9a6803ddd382b24392b3d890f7"
integrity sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw==
+setimmediate@^1.0.5:
+ version "1.0.5"
+ resolved "https://registry.yarnpkg.com/setimmediate/-/setimmediate-1.0.5.tgz#290cbb232e306942d7d7ea9b83732ab7856f8285"
+ integrity sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==
+
version "1.1.1"
resolved "https://registry.yarnpkg.com/setprototypeof/-/setprototypeof-1.1.1.tgz#7e95acb24aa92f5885e0abef5ba131330d4ae683"
@@ -5773,11 +5821,6 @@ stream-size@^0.0.6:
resolved "https://registry.yarnpkg.com/stream-size/-/stream-size-0.0.6.tgz#05d7132e2d2fdb91b533a6e50aff882f040d7679"
integrity sha512-TZsxxZzKPkiD7fxcKx0Ze9s0+WnGGbX1yWZNoXXmN6YjyEu/62pYkLKo7P627N27BT9kI93ZfZAx38brr/hq+Q==
-stream-to-blob@^2.0.1:
- version "2.0.1"
- resolved "https://registry.yarnpkg.com/stream-to-blob/-/stream-to-blob-2.0.1.tgz#59ab71d7a7f0bfb899570e886e44d39f4ac4381a"
- integrity sha512-GXlqXt3svqwIVWoICenix5Poxi4KbCF0BdXXUbpU1X4vq1V8wmjiEIU3aFJzCGNFpKxfbnG0uoowS3nKUgSPYg==
-
streamsearch@^1.1.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/streamsearch/-/streamsearch-1.1.0.tgz#404dd1e2247ca94af554e841a8ef0eaa238da764"
@@ -5831,6 +5874,13 @@ string_decoder@^1.1.1:
dependencies:
safe-buffer "~5.2.0"
+string_decoder@~1.1.1:
+ version "1.1.1"
+ resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.1.1.tgz#9cf1611ba62685d7030ae9e4ba34149c3af03fc8"
+ integrity sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==
+ dependencies:
+ safe-buffer "~5.1.0"
+
strip-ansi@^6.0.0, strip-ansi@^6.0.1:
version "6.0.1"
resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"
@@ -6298,7 +6348,7 @@ utif2@^4.0.1:
dependencies:
pako "^1.0.11"
-util-deprecate@^1.0.1, util-deprecate@^1.0.2:
+util-deprecate@^1.0.1, util-deprecate@^1.0.2, util-deprecate@~1.0.1:
version "1.0.2"
resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf"
integrity sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==