Skip to content

⚡️⚡️⚡️ Self-hosted collaborative bookmark manager to collect, read, annotate, and fully preserve what matters, all in one place.

finalized the implementation

daniel31x13fe776252898c
diff --git a/components/ImportDropdown.tsx b/components/ImportDropdown.tsx
index c210e089..51e5f068 100644
--- a/components/ImportDropdown.tsx
+++ b/components/ImportDropdown.tsx
@@ -1,6 +1,5 @@
import React from "react";
import importBookmarks from "@/lib/client/importBookmarks";
-import importBookmarksBinary from "@/lib/client/importBookmarksBinary";
import { MigrationFormat } from "@/types/global";
import { useTranslation } from "next-i18next";
import { dropdownTriggerer } from "@/lib/client/utils";
@@ -93,9 +92,7 @@ const ImportDropdown = ({}: Props) => {
id="import-omnivore-file"
accept=".zip"
className="hidden"
- onChange={(e) =>
- importBookmarksBinary(e, MigrationFormat.omnivore)
- }
+ onChange={(e) => importBookmarks(e, MigrationFormat.omnivore)}
/>
</label>
</li>
diff --git a/lib/api/controllers/migration/importFromHTMLFile.ts b/lib/api/controllers/migration/importFromHTMLFile.ts
index 365e74db..dbdaaa23 100644
--- a/lib/api/controllers/migration/importFromHTMLFile.ts
+++ b/lib/api/controllers/migration/importFromHTMLFile.ts
@@ -3,14 +3,12 @@ import createFolder from "@/lib/api/storage/createFolder";
import { JSDOM } from "jsdom";
import { parse, Node, Element, TextNode } from "himalaya";
import { hasPassedLimit } from "../../verifyCapacity";
-import streamToBlob from "@/lib/shared/streamToBlob";
export default async function importFromHTMLFile(
userId: number,
- rawStream: ReadableStream
+ rawData: string
) {
- const rawData: Blob = await streamToBlob(rawStream);
- const dom = new JSDOM(await rawData.arrayBuffer());
+ const dom = new JSDOM(rawData);
const document = dom.window.document;
// remove bad tags
diff --git a/lib/api/controllers/migration/importFromLinkwarden.ts b/lib/api/controllers/migration/importFromLinkwarden.ts
index fc521612..fc29a996 100644
--- a/lib/api/controllers/migration/importFromLinkwarden.ts
+++ b/lib/api/controllers/migration/importFromLinkwarden.ts
@@ -2,14 +2,12 @@ import { prisma } from "@/lib/api/db";
import { Backup } from "@/types/global";
import createFolder from "@/lib/api/storage/createFolder";
import { hasPassedLimit } from "../../verifyCapacity";
-import streamToBlob from "@/lib/shared/streamToBlob";
export default async function importFromLinkwarden(
userId: number,
- rawStream: ReadableStream
+ rawData: string
) {
- const rawData: Blob = await streamToBlob(rawStream);
- const data: Backup = JSON.parse(await rawData.text());
+ const data: Backup = JSON.parse(rawData);
let totalImports = 0;
diff --git a/lib/api/controllers/migration/importFromOmnivore.ts b/lib/api/controllers/migration/importFromOmnivore.ts
index b1a8398d..864270e5 100644
--- a/lib/api/controllers/migration/importFromOmnivore.ts
+++ b/lib/api/controllers/migration/importFromOmnivore.ts
@@ -1,91 +1,80 @@
import { prisma } from "@/lib/api/db";
import createFolder from "@/lib/api/storage/createFolder";
import { hasPassedLimit } from "../../verifyCapacity";
-import { ZipReader, BlobReader, TextWriter } from "@zip.js/zip.js";
-import { Readable } from "stream";
-import streamToBlob from "@/lib/shared/streamToBlob";
-type OmnivoreMetadata = {
+type OmnivoreItem = {
id: string;
slug: string;
title: string;
description: string;
- author?: string;
+ author: string;
url: string;
state: string;
readingProgress: number;
thumbnail: string;
labels: string[];
- savedAt: Date;
- updatedAt: Date;
- publishedAt?: Date;
-}[];
+ savedAt: string;
+ updatedAt: string;
+ publishedAt: string;
+};
+
+type OmnivoreMetadata = OmnivoreItem[];
export default async function importFromOmnivore(
userId: number,
- rawStream: ReadableStream
+ rawData: string
) {
- const rawData: Blob = await streamToBlob(rawStream);
- const zipFileReader = new BlobReader(rawData);
- const importArchive = new ZipReader(zipFileReader);
+ const data: OmnivoreMetadata = JSON.parse(rawData);
- const zipEntries = await importArchive.getEntries();
+ const backup = data.filter((item) => !!item.url);
- await prisma.$transaction(async () => {
- const omnivoreCollection = await prisma.collection.create({
- data: {
- owner: {
- connect: {
- id: userId,
- },
- },
- name: "Omnivore Imports",
- createdBy: {
- connect: {
- id: userId,
- },
- },
- },
- });
+ const totalImports = backup.length;
+ const hasTooManyLinks = await hasPassedLimit(userId, totalImports);
+ if (hasTooManyLinks) {
+ return {
+ response: `Your subscription has reached the maximum number of links allowed.`,
+ status: 400,
+ };
+ }
- createFolder({ filePath: `archives/${omnivoreCollection.id}` });
-
- for (const entry of zipEntries) {
- if (entry.filename.startsWith("metadata_")) {
- console.log(`Getting metadata from ${entry.filename}`);
-
- const jsonWriter = new TextWriter();
- const jsonMetadatString: string =
- (await entry.getData?.(jsonWriter)) ?? "";
- const metadata: OmnivoreMetadata = JSON.parse(jsonMetadatString);
+ await prisma
+ .$transaction(
+ async () => {
+ const newCollection = await prisma.collection.create({
+ data: {
+ owner: {
+ connect: {
+ id: userId,
+ },
+ },
+ name: "Omnivore Imports",
+ createdBy: {
+ connect: {
+ id: userId,
+ },
+ },
+ },
+ });
- const hasTooManyLinks = await hasPassedLimit(userId, metadata.length);
- if (hasTooManyLinks) {
- return {
- response: `Your subscription have reached the maximum number of links allowed.`,
- status: 400,
- };
- }
+ createFolder({ filePath: `archives/${newCollection.id}` });
- for (const data of metadata) {
+ for (const item of backup) {
try {
- new URL(data.url.trim());
+ new URL(item.url.trim());
} catch (err) {
continue;
}
- console.log("Extracting text data");
- //const textData = await importArchive.get(`content/${data.slug}.html`)?.get_string()
await prisma.link.create({
data: {
- url: data.url.trim(),
- name: data.title.trim().slice(0, 254),
- importDate: data.savedAt,
- description: data.description ? data.description : "",
- //textContent: textData, // TODO maybe we need to cleanup the html
+ url: item.url?.trim().slice(0, 2047),
+ name: item.title?.trim().slice(0, 254) || "",
+ description: item.description?.trim().slice(0, 2047) || "",
+ image: item.thumbnail || "",
+ importDate: item.savedAt ? new Date(item.savedAt) : null,
collection: {
connect: {
- id: omnivoreCollection.id,
+ id: newCollection.id,
},
},
createdBy: {
@@ -93,10 +82,11 @@ export default async function importFromOmnivore(
id: userId,
},
},
+
tags:
- data.labels && data.labels[0]
+ item.labels && item.labels.length > 0
? {
- connectOrCreate: data.labels.map((label) => ({
+ connectOrCreate: item.labels.map((label) => ({
where: {
name_ownerId: {
name: label?.trim().slice(0, 49),
@@ -117,8 +107,13 @@ export default async function importFromOmnivore(
},
});
}
- }
- }
- });
+ },
+ { timeout: 30000 }
+ )
+ .catch((err) => {
+ console.error("Error during Omnivore import:", err);
+ throw err;
+ });
+
return { response: "Success.", status: 200 };
}
diff --git a/lib/api/controllers/migration/importFromWallabag.ts b/lib/api/controllers/migration/importFromWallabag.ts
index 23b8edef..9f8fe581 100644
--- a/lib/api/controllers/migration/importFromWallabag.ts
+++ b/lib/api/controllers/migration/importFromWallabag.ts
@@ -1,7 +1,6 @@
import { prisma } from "@/lib/api/db";
import createFolder from "@/lib/api/storage/createFolder";
import { hasPassedLimit } from "../../verifyCapacity";
-import streamToBlob from "@/lib/shared/streamToBlob";
type WallabagBackup = {
is_archived: number;
@@ -28,10 +27,9 @@ type WallabagBackup = {
export default async function importFromWallabag(
userId: number,
- rawStream: ReadableStream
+ rawData: string
) {
- const rawData: Blob = await streamToBlob(rawStream);
- const data: WallabagBackup = JSON.parse(await rawData.text());
+ const data: WallabagBackup = JSON.parse(rawData);
const backup = data.filter((e) => e.url);
@@ -83,7 +81,7 @@ export default async function importFromWallabag(
: undefined,
url: link.url?.trim().slice(0, 2047),
name: link.title?.trim().slice(0, 254) || "",
- textContent: link.content?.trim() || "",
+ textContent: link.content?.trim().slice(0, 2047) || "",
importDate: link.created_at || null,
collection: {
connect: {
diff --git a/lib/client/importBookmarks.ts b/lib/client/importBookmarks.ts
index 6398bfa0..29475ca8 100644
--- a/lib/client/importBookmarks.ts
+++ b/lib/client/importBookmarks.ts
@@ -1,5 +1,24 @@
import { MigrationFormat, MigrationRequest } from "@/types/global";
import { toast } from "react-hot-toast";
+import JSZip from "jszip";
+
+const processOmnivoreZipFile = async (zip: JSZip): Promise<string> => {
+ const metadataFiles = Object.keys(zip.files).filter((filePath) => {
+ const file = zip.files[filePath];
+ return filePath.startsWith("metadata_") && !file.dir;
+ });
+
+ const allMetadataArrays = await Promise.all(
+ metadataFiles.map(async (filePath) => {
+ const fileContent = await zip.files[filePath].async("string");
+ return JSON.parse(fileContent) || [];
+ })
+ );
+
+ const flattenedData = allMetadataArrays.flat();
+
+ return JSON.stringify(flattenedData);
+};
const importBookmarks = async (
e: React.ChangeEvent<HTMLInputElement>,
@@ -9,11 +28,26 @@ const importBookmarks = async (
if (file) {
const reader = new FileReader();
- reader.readAsText(file, "UTF-8");
+
+ if (format === MigrationFormat.omnivore) reader.readAsArrayBuffer(file);
+ else reader.readAsText(file, "UTF-8");
+
reader.onload = async function (e) {
const load = toast.loading("Importing...");
- const request: string = e.target?.result as string;
+ let request = e.target?.result as any;
+
+ if (format === MigrationFormat.omnivore) {
+ try {
+ const zip = await JSZip.loadAsync(request);
+ request = await processOmnivoreZipFile(zip);
+ } catch (zipError) {
+ console.error("Failed to parse zip file:", zipError);
+ toast.dismiss(load);
+ toast.error("Failed to parse the zip file. Please try again.");
+ return;
+ }
+ }
const body: MigrationRequest = {
format,
diff --git a/lib/client/importBookmarksBinary.ts b/lib/client/importBookmarksBinary.ts
deleted file mode 100644
index d9e59767..00000000
--- a/lib/client/importBookmarksBinary.ts
+++ /dev/null
@@ -1,64 +0,0 @@
-import { MigrationFormat } from "@/types/global";
-import { toast } from "react-hot-toast";
-
-const importBookmarksBinary = async (
- e: React.ChangeEvent<HTMLInputElement>,
- format: MigrationFormat
-) => {
- const file: File | null = e.target.files && e.target.files[0];
-
- if (file) {
- const reader = new FileReader();
-
- reader.readAsArrayBuffer(file);
- reader.onload = async function (e) {
- const load = toast.loading("Importing...");
-
- try {
- const migrationFormatName = MigrationFormat[format];
- const response = await fetch(
- `/api/v1/migration/upload/${migrationFormatName}`,
- {
- method: "POST",
- body: e.target?.result as ArrayBuffer,
- headers: { "Content-Type": "application/octet-stream" },
- }
- );
-
- if (!response.ok) {
- const errorData = await response.json();
- toast.dismiss(load);
-
- toast.error(
- errorData.response ||
- "Failed to import bookmarks. Please try again."
- );
- return;
- }
-
- await response.json();
- toast.dismiss(load);
- toast.success("Imported the Bookmarks! Reloading the page...");
-
- setTimeout(() => {
- location.reload();
- }, 2000);
- } catch (error) {
- console.error("Request failed", error);
- toast.dismiss(load);
- toast.error(
- "An error occurred while importing bookmarks. Please check the logs for more info."
- );
- }
- };
-
- reader.onerror = function (e) {
- console.log("Error reading file:", e);
- toast.error(
- "Failed to read the file. Please make sure the file is correct and try again."
- );
- };
- }
-};
-
-export default importBookmarksBinary;
diff --git a/lib/shared/streamToBlob.ts b/lib/shared/streamToBlob.ts
deleted file mode 100644
index 7fdabc8b..00000000
--- a/lib/shared/streamToBlob.ts
+++ /dev/null
@@ -1,20 +0,0 @@
-export default async function streamToBlob(
- stream: ReadableStream,
- mimeType?: string
-): Promise<Blob> {
- if (mimeType != null && typeof mimeType !== "string") {
- throw new Error("Invalid mimeType, expected a string.");
- }
-
- const chunks: Uint8Array[] = [];
- for await (const chunk of stream as any) {
- // Ensure the chunk is a Uint8Array
- chunks.push(
- typeof chunk === "string"
- ? new Uint8Array(Buffer.from(chunk))
- : new Uint8Array(chunk)
- );
- }
-
- return new Blob(chunks, { type: mimeType });
-}
diff --git a/package.json b/package.json
index b828de3b..d3e3ab3f 100644
--- a/package.json
+++ b/package.json
@@ -37,7 +37,6 @@
"@types/nodemailer": "^6.4.8",
"@types/react": "18.2.14",
"@types/react-dom": "18.2.7",
- "@zip.js/zip.js": "^2.7.53",
"axios": "^1.5.1",
"bcrypt": "^5.1.0",
"bootstrap-icons": "^1.11.2",
@@ -81,7 +80,6 @@
"react-window": "^1.8.10",
"rss-parser": "^3.13.0",
"socks-proxy-agent": "^8.0.2",
- "stream-size": "^0.0.6",
"stripe": "^12.13.0",
"tailwind-merge": "^2.3.0",
"vaul": "^1.1.1",
diff --git a/pages/api/v1/migration/[format].ts b/pages/api/v1/migration/[format].ts
deleted file mode 100644
index 31d8acd9..00000000
--- a/pages/api/v1/migration/[format].ts
+++ /dev/null
@@ -1,56 +0,0 @@
-import type { NextApiRequest, NextApiResponse } from "next";
-import exportData from "@/lib/api/controllers/migration/exportData";
-import importFromHTMLFile from "@/lib/api/controllers/migration/importFromHTMLFile";
-import importFromLinkwarden from "@/lib/api/controllers/migration/importFromLinkwarden";
-import { MigrationFormat, MigrationRequest } from "@/types/global";
-import verifyUser from "@/lib/api/verifyUser";
-import importFromWallabag from "@/lib/api/controllers/migration/importFromWallabag";
-import importFromOmnivore from "@/lib/api/controllers/migration/importFromOmnivore";
-import { Readable } from "stream";
-import getSizeTransform from "stream-size";
-
-export const config = {
- api: {
- // Disable body parser to be able to process binary uploads
- bodyParser: false,
- },
-};
-
-const IMPORT_SIZE_LIMIT_BYTES =
- parseInt(process.env.IMPORT_LIMIT ? process.env.IMPORT_LIMIT : "10") *
- 1024 *
- 1024;
-
-export default async function users(req: NextApiRequest, res: NextApiResponse) {
- const user = await verifyUser({ req, res });
- if (!user) return;
-
- if (req.method === "POST") {
- if (process.env.NEXT_PUBLIC_DEMO === "true")
- return res.status(400).json({
- response:
- "This action is disabled because this is a read-only demo of Linkwarden.",
- });
-
- const unlimitedReadable: Readable = Readable.from(req);
- const importData = unlimitedReadable.pipe(
- getSizeTransform(IMPORT_SIZE_LIMIT_BYTES)
- ) as unknown as ReadableStream;
- const importFormatName = String(
- req.query.format
- ) as keyof typeof MigrationFormat;
- const importFormat: MigrationFormat = MigrationFormat[importFormatName];
-
- let data;
- if (importFormat === MigrationFormat.htmlFile)
- data = await importFromHTMLFile(user.id, importData);
- else if (importFormat === MigrationFormat.linkwarden)
- data = await importFromLinkwarden(user.id, importData);
- else if (importFormat === MigrationFormat.wallabag)
- data = await importFromWallabag(user.id, importData);
- else if (importFormat == MigrationFormat.omnivore)
- data = await importFromOmnivore(user.id, importData);
-
- if (data) return res.status(data.status).json({ response: data.response });
- }
-}
diff --git a/pages/api/v1/migration/index.ts b/pages/api/v1/migration/index.ts
index 41090086..d7c52ffb 100644
--- a/pages/api/v1/migration/index.ts
+++ b/pages/api/v1/migration/index.ts
@@ -1,7 +1,21 @@
import type { NextApiRequest, NextApiResponse } from "next";
import exportData from "@/lib/api/controllers/migration/exportData";
-
+import importFromHTMLFile from "@/lib/api/controllers/migration/importFromHTMLFile";
+import importFromLinkwarden from "@/lib/api/controllers/migration/importFromLinkwarden";
+import { MigrationFormat, MigrationRequest } from "@/types/global";
import verifyUser from "@/lib/api/verifyUser";
+import importFromWallabag from "@/lib/api/controllers/migration/importFromWallabag";
+import importFromOmnivore from "@/lib/api/controllers/migration/importFromOmnivore";
+
+export const config = {
+ api: {
+ bodyParser: {
+ sizeLimit: process.env.IMPORT_LIMIT
+ ? process.env.IMPORT_LIMIT + "mb"
+ : "10mb",
+ },
+ },
+};
export default async function users(req: NextApiRequest, res: NextApiResponse) {
const user = await verifyUser({ req, res });
@@ -16,5 +30,25 @@ export default async function users(req: NextApiRequest, res: NextApiResponse) {
.setHeader("Content-Disposition", "attachment; filename=backup.json")
.status(data.status)
.json(data.response);
- } // TODO needs handling for invalid request type
+ } else if (req.method === "POST") {
+ if (process.env.NEXT_PUBLIC_DEMO === "true")
+ return res.status(400).json({
+ response:
+ "This action is disabled because this is a read-only demo of Linkwarden.",
+ });
+
+ const request: MigrationRequest = JSON.parse(req.body);
+
+ let data;
+ if (request.format === MigrationFormat.htmlFile)
+ data = await importFromHTMLFile(user.id, request.data);
+ else if (request.format === MigrationFormat.linkwarden)
+ data = await importFromLinkwarden(user.id, request.data);
+ else if (request.format === MigrationFormat.wallabag)
+ data = await importFromWallabag(user.id, request.data);
+ else if (request.format === MigrationFormat.omnivore)
+ data = await importFromOmnivore(user.id, request.data);
+
+ if (data) return res.status(data.status).json({ response: data.response });
+ }
}
diff --git a/pages/api/v1/migration/upload/[format].ts b/pages/api/v1/migration/upload/[format].ts
deleted file mode 100644
index 0a032981..00000000
--- a/pages/api/v1/migration/upload/[format].ts
+++ /dev/null
@@ -1,39 +0,0 @@
-import type { NextApiRequest, NextApiResponse } from "next";
-import { MigrationFormat, MigrationRequest } from "@/types/global";
-import verifyUser from "@/lib/api/verifyUser";
-import importFromOmnivore from "@/lib/api/controllers/migration/importFromOmnivore";
-import { Readable } from "stream";
-import getSizeTransform from "stream-size";
-import formidable from "formidable";
-
-export const config = {
- api: {
- // Disable body parser to be able to process binary uploads
- bodyParser: false,
- },
-};
-
-const IMPORT_SIZE_LIMIT_BYTES =
- parseInt(process.env.IMPORT_LIMIT ? process.env.IMPORT_LIMIT : "10") *
- 1024 *
- 1024;
-
-export default async function users(req: NextApiRequest, res: NextApiResponse) {
- const user = await verifyUser({ req, res });
- if (!user) return;
-
- const unlimitedReadable: Readable = Readable.from(req);
- const importData = unlimitedReadable.pipe(
- getSizeTransform(IMPORT_SIZE_LIMIT_BYTES)
- );
- const importFormatName = String(
- req.query.format
- ) as keyof typeof MigrationFormat;
- const importFormat: MigrationFormat = MigrationFormat[importFormatName];
-
- let data;
- if (importFormat == MigrationFormat.omnivore)
- data = await importFromOmnivore(user.id, importData);
-
- if (data) return res.status(data.status).json({ response: data.response });
-}
diff --git a/pages/settings/account.tsx b/pages/settings/account.tsx
index db54d2f0..0d107cee 100644
--- a/pages/settings/account.tsx
+++ b/pages/settings/account.tsx
@@ -39,7 +39,7 @@ export default function Account() {
emailVerified: null,
password: undefined,
image: "",
- isPrivate: true,
+ isPrivate: false,
// @ts-ignore
createdAt: null,
whitelistedUsers: [],
diff --git a/yarn.lock b/yarn.lock
index 4b7bf49f..cecd9b82 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -2173,11 +2173,6 @@
"@typescript-eslint/types" "5.49.0"
eslint-visitor-keys "^3.3.0"
-"@zip.js/zip.js@^2.7.53":
- version "2.7.53"
- resolved "https://registry.yarnpkg.com/@zip.js/zip.js/-/zip.js-2.7.53.tgz#bf88e90d8eed562182c01339643bc405446b0578"
- integrity sha512-G6Bl5wN9EXXVaTUIox71vIX5Z454zEBe+akKpV4m1tUboIctT5h7ID3QXCJd/Lfy2rSvmkTmZIucf1jGRR4f5A==
-
abab@^2.0.6:
version "2.0.6"
resolved "https://registry.yarnpkg.com/abab/-/abab-2.0.6.tgz#41b80f2c871d19686216b82309231cfd3cb3d291"
@@ -5821,11 +5816,6 @@ stop-iteration-iterator@^1.0.0:
dependencies:
internal-slot "^1.0.4"
-stream-size@^0.0.6:
- version "0.0.6"
- resolved "https://registry.yarnpkg.com/stream-size/-/stream-size-0.0.6.tgz#05d7132e2d2fdb91b533a6e50aff882f040d7679"
- integrity sha512-TZsxxZzKPkiD7fxcKx0Ze9s0+WnGGbX1yWZNoXXmN6YjyEu/62pYkLKo7P627N27BT9kI93ZfZAx38brr/hq+Q==
-
streamsearch@^1.1.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/streamsearch/-/streamsearch-1.1.0.tgz#404dd1e2247ca94af554e841a8ef0eaa238da764"