⚡️⚡️⚡️ Self-hosted collaborative bookmark manager to collect, read, annotate, and fully preserve what matters, all in one place.
TypeScript bookmarkbookmark-managercollaborationnextjsreact-nativeread-it-laterself-hostedtypescript
finalized the implementation
daniel31x13fe776252898c
diff --git a/components/ImportDropdown.tsx b/components/ImportDropdown.tsxindex c210e089..51e5f068 100644--- a/components/ImportDropdown.tsx+++ b/components/ImportDropdown.tsx@@ -1,6 +1,5 @@import React from "react";import importBookmarks from "@/lib/client/importBookmarks";-import importBookmarksBinary from "@/lib/client/importBookmarksBinary";import { MigrationFormat } from "@/types/global";import { useTranslation } from "next-i18next";import { dropdownTriggerer } from "@/lib/client/utils";@@ -93,9 +92,7 @@ const ImportDropdown = ({}: Props) => {id="import-omnivore-file"accept=".zip"className="hidden"- onChange={(e) =>- importBookmarksBinary(e, MigrationFormat.omnivore)- }+ onChange={(e) => importBookmarks(e, MigrationFormat.omnivore)}/></label></li>diff --git a/lib/api/controllers/migration/importFromHTMLFile.ts b/lib/api/controllers/migration/importFromHTMLFile.tsindex 365e74db..dbdaaa23 100644--- a/lib/api/controllers/migration/importFromHTMLFile.ts+++ b/lib/api/controllers/migration/importFromHTMLFile.ts@@ -3,14 +3,12 @@ import createFolder from "@/lib/api/storage/createFolder";import { JSDOM } from "jsdom";import { parse, Node, Element, TextNode } from "himalaya";import { hasPassedLimit } from "../../verifyCapacity";-import streamToBlob from "@/lib/shared/streamToBlob";export default async function importFromHTMLFile(userId: number,- rawStream: ReadableStream+ rawData: string) {- const rawData: Blob = await streamToBlob(rawStream);- const dom = new JSDOM(await rawData.arrayBuffer());+ const dom = new JSDOM(rawData);const document = dom.window.document;// remove bad tagsdiff --git a/lib/api/controllers/migration/importFromLinkwarden.ts b/lib/api/controllers/migration/importFromLinkwarden.tsindex fc521612..fc29a996 100644--- a/lib/api/controllers/migration/importFromLinkwarden.ts+++ b/lib/api/controllers/migration/importFromLinkwarden.ts@@ -2,14 +2,12 @@ import { prisma } from "@/lib/api/db";import { Backup } from "@/types/global";import createFolder from "@/lib/api/storage/createFolder";import { hasPassedLimit } from "../../verifyCapacity";-import streamToBlob from "@/lib/shared/streamToBlob";export default async function importFromLinkwarden(userId: number,- rawStream: ReadableStream+ rawData: string) {- const rawData: Blob = await streamToBlob(rawStream);- const data: Backup = JSON.parse(await rawData.text());+ const data: Backup = JSON.parse(rawData);let totalImports = 0;diff --git a/lib/api/controllers/migration/importFromOmnivore.ts b/lib/api/controllers/migration/importFromOmnivore.tsindex b1a8398d..864270e5 100644--- a/lib/api/controllers/migration/importFromOmnivore.ts+++ b/lib/api/controllers/migration/importFromOmnivore.ts@@ -1,91 +1,80 @@import { prisma } from "@/lib/api/db";import createFolder from "@/lib/api/storage/createFolder";import { hasPassedLimit } from "../../verifyCapacity";-import { ZipReader, BlobReader, TextWriter } from "@zip.js/zip.js";-import { Readable } from "stream";-import streamToBlob from "@/lib/shared/streamToBlob";-type OmnivoreMetadata = {+type OmnivoreItem = {id: string;slug: string;title: string;description: string;- author?: string;+ author: string;url: string;state: string;readingProgress: number;thumbnail: string;labels: string[];- savedAt: Date;- updatedAt: Date;- publishedAt?: Date;-}[];+ savedAt: string;+ updatedAt: string;+ publishedAt: string;+};++type OmnivoreMetadata = OmnivoreItem[];export default async function importFromOmnivore(userId: number,- rawStream: ReadableStream+ rawData: string) {- const rawData: Blob = await streamToBlob(rawStream);- const zipFileReader = new BlobReader(rawData);- const importArchive = new ZipReader(zipFileReader);+ const data: OmnivoreMetadata = JSON.parse(rawData);- const zipEntries = await importArchive.getEntries();+ const backup = data.filter((item) => !!item.url);- await prisma.$transaction(async () => {- const omnivoreCollection = await prisma.collection.create({- data: {- owner: {- connect: {- id: userId,- },- },- name: "Omnivore Imports",- createdBy: {- connect: {- id: userId,- },- },- },- });+ const totalImports = backup.length;+ const hasTooManyLinks = await hasPassedLimit(userId, totalImports);+ if (hasTooManyLinks) {+ return {+ response: `Your subscription has reached the maximum number of links allowed.`,+ status: 400,+ };+ }- createFolder({ filePath: `archives/${omnivoreCollection.id}` });-- for (const entry of zipEntries) {- if (entry.filename.startsWith("metadata_")) {- console.log(`Getting metadata from ${entry.filename}`);-- const jsonWriter = new TextWriter();- const jsonMetadatString: string =- (await entry.getData?.(jsonWriter)) ?? "";- const metadata: OmnivoreMetadata = JSON.parse(jsonMetadatString);+ await prisma+ .$transaction(+ async () => {+ const newCollection = await prisma.collection.create({+ data: {+ owner: {+ connect: {+ id: userId,+ },+ },+ name: "Omnivore Imports",+ createdBy: {+ connect: {+ id: userId,+ },+ },+ },+ });- const hasTooManyLinks = await hasPassedLimit(userId, metadata.length);- if (hasTooManyLinks) {- return {- response: `Your subscription have reached the maximum number of links allowed.`,- status: 400,- };- }+ createFolder({ filePath: `archives/${newCollection.id}` });- for (const data of metadata) {+ for (const item of backup) {try {- new URL(data.url.trim());+ new URL(item.url.trim());} catch (err) {continue;}- console.log("Extracting text data");- //const textData = await importArchive.get(`content/${data.slug}.html`)?.get_string()await prisma.link.create({data: {- url: data.url.trim(),- name: data.title.trim().slice(0, 254),- importDate: data.savedAt,- description: data.description ? data.description : "",- //textContent: textData, // TODO maybe we need to cleanup the html+ url: item.url?.trim().slice(0, 2047),+ name: item.title?.trim().slice(0, 254) || "",+ description: item.description?.trim().slice(0, 2047) || "",+ image: item.thumbnail || "",+ importDate: item.savedAt ? new Date(item.savedAt) : null,collection: {connect: {- id: omnivoreCollection.id,+ id: newCollection.id,},},createdBy: {@@ -93,10 +82,11 @@ export default async function importFromOmnivore(id: userId,},},+tags:- data.labels && data.labels[0]+ item.labels && item.labels.length > 0? {- connectOrCreate: data.labels.map((label) => ({+ connectOrCreate: item.labels.map((label) => ({where: {name_ownerId: {name: label?.trim().slice(0, 49),@@ -117,8 +107,13 @@ export default async function importFromOmnivore(},});}- }- }- });+ },+ { timeout: 30000 }+ )+ .catch((err) => {+ console.error("Error during Omnivore import:", err);+ throw err;+ });+return { response: "Success.", status: 200 };}diff --git a/lib/api/controllers/migration/importFromWallabag.ts b/lib/api/controllers/migration/importFromWallabag.tsindex 23b8edef..9f8fe581 100644--- a/lib/api/controllers/migration/importFromWallabag.ts+++ b/lib/api/controllers/migration/importFromWallabag.ts@@ -1,7 +1,6 @@import { prisma } from "@/lib/api/db";import createFolder from "@/lib/api/storage/createFolder";import { hasPassedLimit } from "../../verifyCapacity";-import streamToBlob from "@/lib/shared/streamToBlob";type WallabagBackup = {is_archived: number;@@ -28,10 +27,9 @@ type WallabagBackup = {export default async function importFromWallabag(userId: number,- rawStream: ReadableStream+ rawData: string) {- const rawData: Blob = await streamToBlob(rawStream);- const data: WallabagBackup = JSON.parse(await rawData.text());+ const data: WallabagBackup = JSON.parse(rawData);const backup = data.filter((e) => e.url);@@ -83,7 +81,7 @@ export default async function importFromWallabag(: undefined,url: link.url?.trim().slice(0, 2047),name: link.title?.trim().slice(0, 254) || "",- textContent: link.content?.trim() || "",+ textContent: link.content?.trim().slice(0, 2047) || "",importDate: link.created_at || null,collection: {connect: {diff --git a/lib/client/importBookmarks.ts b/lib/client/importBookmarks.tsindex 6398bfa0..29475ca8 100644--- a/lib/client/importBookmarks.ts+++ b/lib/client/importBookmarks.ts@@ -1,5 +1,24 @@import { MigrationFormat, MigrationRequest } from "@/types/global";import { toast } from "react-hot-toast";+import JSZip from "jszip";++const processOmnivoreZipFile = async (zip: JSZip): Promise<string> => {+ const metadataFiles = Object.keys(zip.files).filter((filePath) => {+ const file = zip.files[filePath];+ return filePath.startsWith("metadata_") && !file.dir;+ });++ const allMetadataArrays = await Promise.all(+ metadataFiles.map(async (filePath) => {+ const fileContent = await zip.files[filePath].async("string");+ return JSON.parse(fileContent) || [];+ })+ );++ const flattenedData = allMetadataArrays.flat();++ return JSON.stringify(flattenedData);+};const importBookmarks = async (e: React.ChangeEvent<HTMLInputElement>,@@ -9,11 +28,26 @@ const importBookmarks = async (if (file) {const reader = new FileReader();- reader.readAsText(file, "UTF-8");++ if (format === MigrationFormat.omnivore) reader.readAsArrayBuffer(file);+ else reader.readAsText(file, "UTF-8");+reader.onload = async function (e) {const load = toast.loading("Importing...");- const request: string = e.target?.result as string;+ let request = e.target?.result as any;++ if (format === MigrationFormat.omnivore) {+ try {+ const zip = await JSZip.loadAsync(request);+ request = await processOmnivoreZipFile(zip);+ } catch (zipError) {+ console.error("Failed to parse zip file:", zipError);+ toast.dismiss(load);+ toast.error("Failed to parse the zip file. Please try again.");+ return;+ }+ }const body: MigrationRequest = {format,diff --git a/lib/client/importBookmarksBinary.ts b/lib/client/importBookmarksBinary.tsdeleted file mode 100644index d9e59767..00000000--- a/lib/client/importBookmarksBinary.ts+++ /dev/null@@ -1,64 +0,0 @@-import { MigrationFormat } from "@/types/global";-import { toast } from "react-hot-toast";--const importBookmarksBinary = async (- e: React.ChangeEvent<HTMLInputElement>,- format: MigrationFormat-) => {- const file: File | null = e.target.files && e.target.files[0];-- if (file) {- const reader = new FileReader();-- reader.readAsArrayBuffer(file);- reader.onload = async function (e) {- const load = toast.loading("Importing...");-- try {- const migrationFormatName = MigrationFormat[format];- const response = await fetch(- `/api/v1/migration/upload/${migrationFormatName}`,- {- method: "POST",- body: e.target?.result as ArrayBuffer,- headers: { "Content-Type": "application/octet-stream" },- }- );-- if (!response.ok) {- const errorData = await response.json();- toast.dismiss(load);-- toast.error(- errorData.response ||- "Failed to import bookmarks. Please try again."- );- return;- }-- await response.json();- toast.dismiss(load);- toast.success("Imported the Bookmarks! Reloading the page...");-- setTimeout(() => {- location.reload();- }, 2000);- } catch (error) {- console.error("Request failed", error);- toast.dismiss(load);- toast.error(- "An error occurred while importing bookmarks. Please check the logs for more info."- );- }- };-- reader.onerror = function (e) {- console.log("Error reading file:", e);- toast.error(- "Failed to read the file. Please make sure the file is correct and try again."- );- };- }-};--export default importBookmarksBinary;diff --git a/lib/shared/streamToBlob.ts b/lib/shared/streamToBlob.tsdeleted file mode 100644index 7fdabc8b..00000000--- a/lib/shared/streamToBlob.ts+++ /dev/null@@ -1,20 +0,0 @@-export default async function streamToBlob(- stream: ReadableStream,- mimeType?: string-): Promise<Blob> {- if (mimeType != null && typeof mimeType !== "string") {- throw new Error("Invalid mimeType, expected a string.");- }-- const chunks: Uint8Array[] = [];- for await (const chunk of stream as any) {- // Ensure the chunk is a Uint8Array- chunks.push(- typeof chunk === "string"- ? new Uint8Array(Buffer.from(chunk))- : new Uint8Array(chunk)- );- }-- return new Blob(chunks, { type: mimeType });-}diff --git a/package.json b/package.jsonindex b828de3b..d3e3ab3f 100644--- a/package.json+++ b/package.json@@ -37,7 +37,6 @@"@types/nodemailer": "^6.4.8","@types/react": "18.2.14","@types/react-dom": "18.2.7",- "@zip.js/zip.js": "^2.7.53","axios": "^1.5.1","bcrypt": "^5.1.0","bootstrap-icons": "^1.11.2",@@ -81,7 +80,6 @@"react-window": "^1.8.10","rss-parser": "^3.13.0","socks-proxy-agent": "^8.0.2",- "stream-size": "^0.0.6","stripe": "^12.13.0","tailwind-merge": "^2.3.0","vaul": "^1.1.1",diff --git a/pages/api/v1/migration/[format].ts b/pages/api/v1/migration/[format].tsdeleted file mode 100644index 31d8acd9..00000000--- a/pages/api/v1/migration/[format].ts+++ /dev/null@@ -1,56 +0,0 @@-import type { NextApiRequest, NextApiResponse } from "next";-import exportData from "@/lib/api/controllers/migration/exportData";-import importFromHTMLFile from "@/lib/api/controllers/migration/importFromHTMLFile";-import importFromLinkwarden from "@/lib/api/controllers/migration/importFromLinkwarden";-import { MigrationFormat, MigrationRequest } from "@/types/global";-import verifyUser from "@/lib/api/verifyUser";-import importFromWallabag from "@/lib/api/controllers/migration/importFromWallabag";-import importFromOmnivore from "@/lib/api/controllers/migration/importFromOmnivore";-import { Readable } from "stream";-import getSizeTransform from "stream-size";--export const config = {- api: {- // Disable body parser to be able to process binary uploads- bodyParser: false,- },-};--const IMPORT_SIZE_LIMIT_BYTES =- parseInt(process.env.IMPORT_LIMIT ? process.env.IMPORT_LIMIT : "10") *- 1024 *- 1024;--export default async function users(req: NextApiRequest, res: NextApiResponse) {- const user = await verifyUser({ req, res });- if (!user) return;-- if (req.method === "POST") {- if (process.env.NEXT_PUBLIC_DEMO === "true")- return res.status(400).json({- response:- "This action is disabled because this is a read-only demo of Linkwarden.",- });-- const unlimitedReadable: Readable = Readable.from(req);- const importData = unlimitedReadable.pipe(- getSizeTransform(IMPORT_SIZE_LIMIT_BYTES)- ) as unknown as ReadableStream;- const importFormatName = String(- req.query.format- ) as keyof typeof MigrationFormat;- const importFormat: MigrationFormat = MigrationFormat[importFormatName];-- let data;- if (importFormat === MigrationFormat.htmlFile)- data = await importFromHTMLFile(user.id, importData);- else if (importFormat === MigrationFormat.linkwarden)- data = await importFromLinkwarden(user.id, importData);- else if (importFormat === MigrationFormat.wallabag)- data = await importFromWallabag(user.id, importData);- else if (importFormat == MigrationFormat.omnivore)- data = await importFromOmnivore(user.id, importData);-- if (data) return res.status(data.status).json({ response: data.response });- }-}diff --git a/pages/api/v1/migration/index.ts b/pages/api/v1/migration/index.tsindex 41090086..d7c52ffb 100644--- a/pages/api/v1/migration/index.ts+++ b/pages/api/v1/migration/index.ts@@ -1,7 +1,21 @@import type { NextApiRequest, NextApiResponse } from "next";import exportData from "@/lib/api/controllers/migration/exportData";-+import importFromHTMLFile from "@/lib/api/controllers/migration/importFromHTMLFile";+import importFromLinkwarden from "@/lib/api/controllers/migration/importFromLinkwarden";+import { MigrationFormat, MigrationRequest } from "@/types/global";import verifyUser from "@/lib/api/verifyUser";+import importFromWallabag from "@/lib/api/controllers/migration/importFromWallabag";+import importFromOmnivore from "@/lib/api/controllers/migration/importFromOmnivore";++export const config = {+ api: {+ bodyParser: {+ sizeLimit: process.env.IMPORT_LIMIT+ ? process.env.IMPORT_LIMIT + "mb"+ : "10mb",+ },+ },+};export default async function users(req: NextApiRequest, res: NextApiResponse) {const user = await verifyUser({ req, res });@@ -16,5 +30,25 @@ export default async function users(req: NextApiRequest, res: NextApiResponse) {.setHeader("Content-Disposition", "attachment; filename=backup.json").status(data.status).json(data.response);- } // TODO needs handling for invalid request type+ } else if (req.method === "POST") {+ if (process.env.NEXT_PUBLIC_DEMO === "true")+ return res.status(400).json({+ response:+ "This action is disabled because this is a read-only demo of Linkwarden.",+ });++ const request: MigrationRequest = JSON.parse(req.body);++ let data;+ if (request.format === MigrationFormat.htmlFile)+ data = await importFromHTMLFile(user.id, request.data);+ else if (request.format === MigrationFormat.linkwarden)+ data = await importFromLinkwarden(user.id, request.data);+ else if (request.format === MigrationFormat.wallabag)+ data = await importFromWallabag(user.id, request.data);+ else if (request.format === MigrationFormat.omnivore)+ data = await importFromOmnivore(user.id, request.data);++ if (data) return res.status(data.status).json({ response: data.response });+ }}diff --git a/pages/api/v1/migration/upload/[format].ts b/pages/api/v1/migration/upload/[format].tsdeleted file mode 100644index 0a032981..00000000--- a/pages/api/v1/migration/upload/[format].ts+++ /dev/null@@ -1,39 +0,0 @@-import type { NextApiRequest, NextApiResponse } from "next";-import { MigrationFormat, MigrationRequest } from "@/types/global";-import verifyUser from "@/lib/api/verifyUser";-import importFromOmnivore from "@/lib/api/controllers/migration/importFromOmnivore";-import { Readable } from "stream";-import getSizeTransform from "stream-size";-import formidable from "formidable";--export const config = {- api: {- // Disable body parser to be able to process binary uploads- bodyParser: false,- },-};--const IMPORT_SIZE_LIMIT_BYTES =- parseInt(process.env.IMPORT_LIMIT ? process.env.IMPORT_LIMIT : "10") *- 1024 *- 1024;--export default async function users(req: NextApiRequest, res: NextApiResponse) {- const user = await verifyUser({ req, res });- if (!user) return;-- const unlimitedReadable: Readable = Readable.from(req);- const importData = unlimitedReadable.pipe(- getSizeTransform(IMPORT_SIZE_LIMIT_BYTES)- );- const importFormatName = String(- req.query.format- ) as keyof typeof MigrationFormat;- const importFormat: MigrationFormat = MigrationFormat[importFormatName];-- let data;- if (importFormat == MigrationFormat.omnivore)- data = await importFromOmnivore(user.id, importData);-- if (data) return res.status(data.status).json({ response: data.response });-}diff --git a/pages/settings/account.tsx b/pages/settings/account.tsxindex db54d2f0..0d107cee 100644--- a/pages/settings/account.tsx+++ b/pages/settings/account.tsx@@ -39,7 +39,7 @@ export default function Account() {emailVerified: null,password: undefined,image: "",- isPrivate: true,+ isPrivate: false,// @ts-ignorecreatedAt: null,whitelistedUsers: [],diff --git a/yarn.lock b/yarn.lockindex 4b7bf49f..cecd9b82 100644--- a/yarn.lock+++ b/yarn.lock@@ -2173,11 +2173,6 @@"@typescript-eslint/types" "5.49.0"eslint-visitor-keys "^3.3.0"-"@zip.js/zip.js@^2.7.53":- version "2.7.53"- resolved "https://registry.yarnpkg.com/@zip.js/zip.js/-/zip.js-2.7.53.tgz#bf88e90d8eed562182c01339643bc405446b0578"- integrity sha512-G6Bl5wN9EXXVaTUIox71vIX5Z454zEBe+akKpV4m1tUboIctT5h7ID3QXCJd/Lfy2rSvmkTmZIucf1jGRR4f5A==-abab@^2.0.6:version "2.0.6"resolved "https://registry.yarnpkg.com/abab/-/abab-2.0.6.tgz#41b80f2c871d19686216b82309231cfd3cb3d291"@@ -5821,11 +5816,6 @@ stop-iteration-iterator@^1.0.0:dependencies:internal-slot "^1.0.4"-stream-size@^0.0.6:- version "0.0.6"- resolved "https://registry.yarnpkg.com/stream-size/-/stream-size-0.0.6.tgz#05d7132e2d2fdb91b533a6e50aff882f040d7679"- integrity sha512-TZsxxZzKPkiD7fxcKx0Ze9s0+WnGGbX1yWZNoXXmN6YjyEu/62pYkLKo7P627N27BT9kI93ZfZAx38brr/hq+Q==-streamsearch@^1.1.0:version "1.1.0"resolved "https://registry.yarnpkg.com/streamsearch/-/streamsearch-1.1.0.tgz#404dd1e2247ca94af554e841a8ef0eaa238da764"