⚡️⚡️⚡️ Self-hosted collaborative bookmark manager to collect, read, annotate, and fully preserve what matters, all in one place.
TypeScript bookmarkbookmark-managercollaborationnextjsreact-nativeread-it-laterself-hostedtypescript
Import should now work on testdata
dereulenspiegela733cc69a309
diff --git a/lib/api/controllers/migration/importFromHTMLFile.ts b/lib/api/controllers/migration/importFromHTMLFile.tsindex f7454713..f8fe2de3 100644--- a/lib/api/controllers/migration/importFromHTMLFile.ts+++ b/lib/api/controllers/migration/importFromHTMLFile.ts@@ -6,7 +6,7 @@ import { hasPassedLimit } from "../../verifyCapacity";export default async function importFromHTMLFile(userId: number,- rawData: string+ rawData: any) {const dom = new JSDOM(rawData);const document = dom.window.document;@@ -167,10 +167,10 @@ const createCollection = async (name: collectionName,parent: parentId? {- connect: {- id: parentId,- },- }+ connect: {+ id: parentId,+ },+ }: undefined,owner: {connect: {@@ -225,25 +225,25 @@ const createLink = async (tags:tags && tags[0]? {- connectOrCreate: tags.map((tag: string) => {- return {- where: {- name_ownerId: {- name: tag.trim(),- ownerId: userId,- },- },- create: {+ connectOrCreate: tags.map((tag: string) => {+ return {+ where: {+ name_ownerId: {name: tag.trim(),- owner: {- connect: {- id: userId,- },+ ownerId: userId,+ },+ },+ create: {+ name: tag.trim(),+ owner: {+ connect: {+ id: userId,},},- };- }),- }+ },+ };+ }),+ }: undefined,importDate: importDate || undefined,},diff --git a/lib/api/controllers/migration/importFromLinkwarden.ts b/lib/api/controllers/migration/importFromLinkwarden.tsindex e782846b..0ef440c8 100644--- a/lib/api/controllers/migration/importFromLinkwarden.ts+++ b/lib/api/controllers/migration/importFromLinkwarden.ts@@ -5,7 +5,7 @@ import { hasPassedLimit } from "../../verifyCapacity";export default async function importFromLinkwarden(userId: number,- rawData: string+ rawData: any) {const data: Backup = JSON.parse(rawData);diff --git a/lib/api/controllers/migration/importFromOmnivore.ts b/lib/api/controllers/migration/importFromOmnivore.tsindex f17482ee..67c00eea 100644--- a/lib/api/controllers/migration/importFromOmnivore.ts+++ b/lib/api/controllers/migration/importFromOmnivore.ts@@ -3,10 +3,126 @@ import createFolder from "@/lib/api/storage/createFolder";import { JSDOM } from "jsdom";import { parse, Node, Element, TextNode } from "himalaya";import { hasPassedLimit } from "../../verifyCapacity";+import { ZipReader, BlobReader, TextWriter } from "@zip.js/zip.js";+import { Readable } from 'stream';+import streamToBlob from "stream-to-blob";++type OmnivoreMetadata = {+ id: string;+ slug: string;+ title: string;+ description: string;+ author?: string;+ url: string;+ state: string;+ readingProgress: number;+ thumbnail: string;+ labels: string[];+ savedAt: Date;+ updatedAt: Date;+ publishedAt?: Date;+}[]export default async function importFromOmnivore(- userId: number,- rawData: string+ userId: number,+ rawStream: Readable) {- console.log("Received data:\n" + rawData);+ const rawData: Blob = await streamToBlob(rawStream);+ const zipFileReader = new BlobReader(rawData);+ const importArchive = new ZipReader(zipFileReader);++ const zipEntries = await importArchive.getEntries();++ for (const entry of zipEntries) {++ if (entry.filename.startsWith("metadata_")) {+ console.log(`Getting metadata from ${entry.filename}`);++ const jsonWriter = new TextWriter();+ const jsonMetadatString: string = await entry.getData(jsonWriter);+ const metadata: OmnivoreMetadata = JSON.parse(jsonMetadatString);++ const hasTooManyLinks = await hasPassedLimit(userId, metadata.length);+ if (hasTooManyLinks) {+ return {+ response: `Your subscription have reached the maximum number of links allowed.`,+ status: 400,+ };+ }++ await prisma.$transaction(+ async () => {+ const omnivoreCollection = await prisma.collection.create({+ data: {+ owner: {+ connect: {+ id: userId+ }+ },+ name: "Omnivore Imports",+ createdBy: {+ connect: {+ id: userId+ }+ }+ }+ });++ createFolder({ filePath: `archives/${omnivoreCollection.id}` });++ for (const data of metadata) {+ try {+ new URL(data.url.trim());+ } catch (err) {+ continue;+ }+ console.log("Extracting text data");+ //const textData = await importArchive.get(`content/${data.slug}.html`)?.get_string()++ await prisma.link.create(+ {+ data: {+ url: data.url.trim(),+ name: data.title.trim().slice(0, 254),+ importDate: data.savedAt,+ description: data.description ? data.description : "",+ //textContent: textData, // TODO maybe we need to cleanup the html+ collection: {+ connect: {+ id: omnivoreCollection.id,+ },+ },+ createdBy: {+ connect: {+ id: userId,+ },+ },+ tags: data.labels && data.labels[0]+ ? {+ connectOrCreate: data.labels.map((label) => ({+ where: {+ name_ownerId: {+ name: label?.trim().slice(0, 49),+ ownerId: userId,+ },+ },+ create: {+ name: label?.trim().slice(0, 49),+ owner: {+ connect: {+ id: userId,+ },+ },+ },+ })),+ }+ : undefined,+ }+ }+ )+ }+ }+ )+ }+ }}diff --git a/lib/api/controllers/migration/importFromWallabag.ts b/lib/api/controllers/migration/importFromWallabag.tsindex 3c5fa001..f07bdf24 100644--- a/lib/api/controllers/migration/importFromWallabag.ts+++ b/lib/api/controllers/migration/importFromWallabag.ts@@ -27,7 +27,7 @@ type WallabagBackup = {export default async function importFromWallabag(userId: number,- rawData: string+ rawData: any) {const data: WallabagBackup = JSON.parse(rawData);@@ -96,23 +96,23 @@ export default async function importFromWallabag(tags:link.tags && link.tags[0]? {- connectOrCreate: link.tags.map((tag) => ({- where: {- name_ownerId: {- name: tag?.trim().slice(0, 49),- ownerId: userId,- },- },- create: {+ connectOrCreate: link.tags.map((tag) => ({+ where: {+ name_ownerId: {name: tag?.trim().slice(0, 49),- owner: {- connect: {- id: userId,- },+ ownerId: userId,+ },+ },+ create: {+ name: tag?.trim().slice(0, 49),+ owner: {+ connect: {+ id: userId,},},- })),- }+ },+ })),+ }: undefined,},});diff --git a/package.json b/package.jsonindex 4a8eb6d0..dc24b63e 100644--- a/package.json+++ b/package.json@@ -37,6 +37,8 @@"@types/nodemailer": "^6.4.8","@types/react": "18.2.14","@types/react-dom": "18.2.7",+ "@types/stream-to-blob": "^2.0.0",+ "@zip.js/zip.js": "^2.7.53","axios": "^1.5.1","bcrypt": "^5.1.0","bootstrap-icons": "^1.11.2",@@ -79,6 +81,7 @@"react-window": "^1.8.10","rss-parser": "^3.13.0","socks-proxy-agent": "^8.0.2",+ "stream-to-blob": "^2.0.1","stripe": "^12.13.0","tailwind-merge": "^2.3.0","vaul": "^1.1.1",diff --git a/pages/api/v1/migration/[format].ts b/pages/api/v1/migration/[format].tsnew file mode 100644index 00000000..9246c79c--- /dev/null+++ b/pages/api/v1/migration/[format].ts@@ -0,0 +1,50 @@+import type { NextApiRequest, NextApiResponse } from "next";+import exportData from "@/lib/api/controllers/migration/exportData";+import importFromHTMLFile from "@/lib/api/controllers/migration/importFromHTMLFile";+import importFromLinkwarden from "@/lib/api/controllers/migration/importFromLinkwarden";+import { MigrationFormat, MigrationRequest } from "@/types/global";+import verifyUser from "@/lib/api/verifyUser";+import importFromWallabag from "@/lib/api/controllers/migration/importFromWallabag";+import importFromOmnivore from "@/lib/api/controllers/migration/importFromOmnivore";+import { Readable } from 'stream';++export const config = {+ api: {+ // bodyParser: {+ // sizeLimit: process.env.IMPORT_LIMIT+ // ? process.env.IMPORT_LIMIT + "mb"+ // : "10mb",+ // },+ bodyParser: false,+ },+};++export default async function users(req: NextApiRequest, res: NextApiResponse) {+ const user = await verifyUser({ req, res });+ if (!user) return;++ if (req.method === "POST") {+ if (process.env.NEXT_PUBLIC_DEMO === "true")+ return res.status(400).json({+ response:+ "This action is disabled because this is a read-only demo of Linkwarden.",+ });++ const importData: Readable = Readable.from(req); // TODO have something like a limit reader+ console.log(`received body type ${typeof importData}`);+ const importFormatName = String(req.query.format) as keyof typeof MigrationFormat;+ const importFormat: MigrationFormat = MigrationFormat[importFormatName];++ let data;+ if (importFormat === MigrationFormat.htmlFile)+ data = await importFromHTMLFile(user.id, importData);+ else if (importFormat === MigrationFormat.linkwarden)+ data = await importFromLinkwarden(user.id, importData);+ else if (importFormat === MigrationFormat.wallabag)+ data = await importFromWallabag(user.id, importData);+ else if (importFormat == MigrationFormat.omnivore)+ data = await importFromOmnivore(user.id, importData);++ if (data) return res.status(data.status).json({ response: data.response });+ }+}diff --git a/pages/api/v1/migration/index.ts b/pages/api/v1/migration/index.tsindex 790af031..41090086 100644--- a/pages/api/v1/migration/index.ts+++ b/pages/api/v1/migration/index.ts@@ -1,21 +1,7 @@import type { NextApiRequest, NextApiResponse } from "next";import exportData from "@/lib/api/controllers/migration/exportData";-import importFromHTMLFile from "@/lib/api/controllers/migration/importFromHTMLFile";-import importFromLinkwarden from "@/lib/api/controllers/migration/importFromLinkwarden";-import { MigrationFormat, MigrationRequest } from "@/types/global";-import verifyUser from "@/lib/api/verifyUser";-import importFromWallabag from "@/lib/api/controllers/migration/importFromWallabag";-import importFromOmnivore from "@/lib/api/controllers/migration/importFromOmnivore";-export const config = {- api: {- bodyParser: {- sizeLimit: process.env.IMPORT_LIMIT- ? process.env.IMPORT_LIMIT + "mb"- : "10mb",- },- },-};+import verifyUser from "@/lib/api/verifyUser";export default async function users(req: NextApiRequest, res: NextApiResponse) {const user = await verifyUser({ req, res });@@ -30,25 +16,5 @@ export default async function users(req: NextApiRequest, res: NextApiResponse) {.setHeader("Content-Disposition", "attachment; filename=backup.json").status(data.status).json(data.response);- } else if (req.method === "POST") {- if (process.env.NEXT_PUBLIC_DEMO === "true")- return res.status(400).json({- response:- "This action is disabled because this is a read-only demo of Linkwarden.",- });-- const request: MigrationRequest = JSON.parse(req.body);-- let data;- if (request.format === MigrationFormat.htmlFile)- data = await importFromHTMLFile(user.id, request.data);- else if (request.format === MigrationFormat.linkwarden)- data = await importFromLinkwarden(user.id, request.data);- else if (request.format === MigrationFormat.wallabag)- data = await importFromWallabag(user.id, request.data);- else if (request.format == MigrationFormat.omnivore)- data = await importFromOmnivore(user.id, request.data);-- if (data) return res.status(data.status).json({ response: data.response });- }+ } // TODO needs handling for invalid request type}diff --git a/pages/dashboard.tsx b/pages/dashboard.tsxindex 2c92b4a0..2917f014 100644--- a/pages/dashboard.tsx+++ b/pages/dashboard.tsx@@ -81,21 +81,24 @@ export default function Dashboard() {if (file) {const reader = new FileReader();- reader.readAsText(file, "UTF-8");+ //reader.readAsText(file, "UTF-8");+ reader.readAsArrayBuffer(file);reader.onload = async function (e) {const load = toast.loading("Importing...");- const request: string = e.target?.result as string;+ // const request: string = e.target?.result as string;- const body: MigrationRequest = {- format,- data: request,- };+ // const body: MigrationRequest = {+ // format,+ // data: request,+ // };try {- const response = await fetch("/api/v1/migration", {+ const migrationFormatName = MigrationFormat[format];+ const response = await fetch(`/api/v1/migration/${migrationFormatName}`, {method: "POST",- body: JSON.stringify(body),+ body: e.target?.result as ArrayBuffer,+ headers: { "Content-Type": "application/octet-stream" },});if (!response.ok) {diff --git a/pages/register.tsx b/pages/register.tsxindex 282558b4..fdb7e966 100644--- a/pages/register.tsx+++ b/pages/register.tsx@@ -149,8 +149,8 @@ export default function Register({text={process.env.NEXT_PUBLIC_STRIPE? t("trial_offer_desc", {- count: Number(process.env.NEXT_PUBLIC_TRIAL_PERIOD_DAYS || 14),- })+ count: Number(process.env.NEXT_PUBLIC_TRIAL_PERIOD_DAYS || 14),+ }): t("register_desc")}data-testid="registration-form"diff --git a/yarn.lock b/yarn.lockindex 4dba2268..c079ba73 100644--- a/yarn.lock+++ b/yarn.lock@@ -2119,6 +2119,13 @@"@types/glob" "~7.2.0""@types/node" "*"+"@types/stream-to-blob@^2.0.0":+ version "2.0.0"+ resolved "https://registry.yarnpkg.com/@types/stream-to-blob/-/stream-to-blob-2.0.0.tgz#b38b6eb763831a37b91dfe4a5854f9da528acb5c"+ integrity sha512-tZpPRVwoUMe4ZHr0xt2+RyvkqKFoKzdyCUuzW3NxC4HKxbNEwCMLZiumMuZ+FaizAQJx3rUXVJNXpMCnnOffAw==+ dependencies:+ "@types/node" "*"+"@types/tough-cookie@*":version "4.0.3"resolved "https://registry.yarnpkg.com/@types/tough-cookie/-/tough-cookie-4.0.3.tgz#3d06b6769518450871fbc40770b7586334bdfd90"@@ -2173,6 +2180,11 @@"@typescript-eslint/types" "5.49.0"eslint-visitor-keys "^3.3.0"+"@zip.js/zip.js@^2.7.53":+ version "2.7.53"+ resolved "https://registry.yarnpkg.com/@zip.js/zip.js/-/zip.js-2.7.53.tgz#bf88e90d8eed562182c01339643bc405446b0578"+ integrity sha512-G6Bl5wN9EXXVaTUIox71vIX5Z454zEBe+akKpV4m1tUboIctT5h7ID3QXCJd/Lfy2rSvmkTmZIucf1jGRR4f5A==+abab@^2.0.6:version "2.0.6"resolved "https://registry.yarnpkg.com/abab/-/abab-2.0.6.tgz#41b80f2c871d19686216b82309231cfd3cb3d291"@@ -5756,6 +5768,11 @@ stop-iteration-iterator@^1.0.0:dependencies:internal-slot "^1.0.4"+stream-to-blob@^2.0.1:+ version "2.0.1"+ resolved "https://registry.yarnpkg.com/stream-to-blob/-/stream-to-blob-2.0.1.tgz#59ab71d7a7f0bfb899570e886e44d39f4ac4381a"+ integrity sha512-GXlqXt3svqwIVWoICenix5Poxi4KbCF0BdXXUbpU1X4vq1V8wmjiEIU3aFJzCGNFpKxfbnG0uoowS3nKUgSPYg==+streamsearch@^1.1.0:version "1.1.0"resolved "https://registry.yarnpkg.com/streamsearch/-/streamsearch-1.1.0.tgz#404dd1e2247ca94af554e841a8ef0eaa238da764"