Skip to content

⚡️⚡️⚡️ Self-hosted collaborative bookmark manager to collect, read, annotate, and fully preserve what matters, all in one place.

Import should now work on testdata

dereulenspiegela733cc69a309
diff --git a/lib/api/controllers/migration/importFromHTMLFile.ts b/lib/api/controllers/migration/importFromHTMLFile.ts
index f7454713..f8fe2de3 100644
--- a/lib/api/controllers/migration/importFromHTMLFile.ts
+++ b/lib/api/controllers/migration/importFromHTMLFile.ts
@@ -6,7 +6,7 @@ import { hasPassedLimit } from "../../verifyCapacity";
export default async function importFromHTMLFile(
userId: number,
- rawData: string
+ rawData: any
) {
const dom = new JSDOM(rawData);
const document = dom.window.document;
@@ -167,10 +167,10 @@ const createCollection = async (
name: collectionName,
parent: parentId
? {
- connect: {
- id: parentId,
- },
- }
+ connect: {
+ id: parentId,
+ },
+ }
: undefined,
owner: {
connect: {
@@ -225,25 +225,25 @@ const createLink = async (
tags:
tags && tags[0]
? {
- connectOrCreate: tags.map((tag: string) => {
- return {
- where: {
- name_ownerId: {
- name: tag.trim(),
- ownerId: userId,
- },
- },
- create: {
+ connectOrCreate: tags.map((tag: string) => {
+ return {
+ where: {
+ name_ownerId: {
name: tag.trim(),
- owner: {
- connect: {
- id: userId,
- },
+ ownerId: userId,
+ },
+ },
+ create: {
+ name: tag.trim(),
+ owner: {
+ connect: {
+ id: userId,
},
},
- };
- }),
- }
+ },
+ };
+ }),
+ }
: undefined,
importDate: importDate || undefined,
},
diff --git a/lib/api/controllers/migration/importFromLinkwarden.ts b/lib/api/controllers/migration/importFromLinkwarden.ts
index e782846b..0ef440c8 100644
--- a/lib/api/controllers/migration/importFromLinkwarden.ts
+++ b/lib/api/controllers/migration/importFromLinkwarden.ts
@@ -5,7 +5,7 @@ import { hasPassedLimit } from "../../verifyCapacity";
export default async function importFromLinkwarden(
userId: number,
- rawData: string
+ rawData: any
) {
const data: Backup = JSON.parse(rawData);
diff --git a/lib/api/controllers/migration/importFromOmnivore.ts b/lib/api/controllers/migration/importFromOmnivore.ts
index f17482ee..67c00eea 100644
--- a/lib/api/controllers/migration/importFromOmnivore.ts
+++ b/lib/api/controllers/migration/importFromOmnivore.ts
@@ -3,10 +3,126 @@ import createFolder from "@/lib/api/storage/createFolder";
import { JSDOM } from "jsdom";
import { parse, Node, Element, TextNode } from "himalaya";
import { hasPassedLimit } from "../../verifyCapacity";
+import { ZipReader, BlobReader, TextWriter } from "@zip.js/zip.js";
+import { Readable } from 'stream';
+import streamToBlob from "stream-to-blob";
+
+type OmnivoreMetadata = {
+ id: string;
+ slug: string;
+ title: string;
+ description: string;
+ author?: string;
+ url: string;
+ state: string;
+ readingProgress: number;
+ thumbnail: string;
+ labels: string[];
+ savedAt: Date;
+ updatedAt: Date;
+ publishedAt?: Date;
+}[]
export default async function importFromOmnivore(
- userId: number,
- rawData: string
+ userId: number,
+ rawStream: Readable
) {
- console.log("Received data:\n" + rawData);
+ const rawData: Blob = await streamToBlob(rawStream);
+ const zipFileReader = new BlobReader(rawData);
+ const importArchive = new ZipReader(zipFileReader);
+
+ const zipEntries = await importArchive.getEntries();
+
+ for (const entry of zipEntries) {
+
+ if (entry.filename.startsWith("metadata_")) {
+ console.log(`Getting metadata from ${entry.filename}`);
+
+ const jsonWriter = new TextWriter();
+ const jsonMetadatString: string = await entry.getData(jsonWriter);
+ const metadata: OmnivoreMetadata = JSON.parse(jsonMetadatString);
+
+ const hasTooManyLinks = await hasPassedLimit(userId, metadata.length);
+ if (hasTooManyLinks) {
+ return {
+ response: `Your subscription have reached the maximum number of links allowed.`,
+ status: 400,
+ };
+ }
+
+ await prisma.$transaction(
+ async () => {
+ const omnivoreCollection = await prisma.collection.create({
+ data: {
+ owner: {
+ connect: {
+ id: userId
+ }
+ },
+ name: "Omnivore Imports",
+ createdBy: {
+ connect: {
+ id: userId
+ }
+ }
+ }
+ });
+
+ createFolder({ filePath: `archives/${omnivoreCollection.id}` });
+
+ for (const data of metadata) {
+ try {
+ new URL(data.url.trim());
+ } catch (err) {
+ continue;
+ }
+ console.log("Extracting text data");
+ //const textData = await importArchive.get(`content/${data.slug}.html`)?.get_string()
+
+ await prisma.link.create(
+ {
+ data: {
+ url: data.url.trim(),
+ name: data.title.trim().slice(0, 254),
+ importDate: data.savedAt,
+ description: data.description ? data.description : "",
+ //textContent: textData, // TODO maybe we need to cleanup the html
+ collection: {
+ connect: {
+ id: omnivoreCollection.id,
+ },
+ },
+ createdBy: {
+ connect: {
+ id: userId,
+ },
+ },
+ tags: data.labels && data.labels[0]
+ ? {
+ connectOrCreate: data.labels.map((label) => ({
+ where: {
+ name_ownerId: {
+ name: label?.trim().slice(0, 49),
+ ownerId: userId,
+ },
+ },
+ create: {
+ name: label?.trim().slice(0, 49),
+ owner: {
+ connect: {
+ id: userId,
+ },
+ },
+ },
+ })),
+ }
+ : undefined,
+ }
+ }
+ )
+ }
+ }
+ )
+ }
+ }
}
diff --git a/lib/api/controllers/migration/importFromWallabag.ts b/lib/api/controllers/migration/importFromWallabag.ts
index 3c5fa001..f07bdf24 100644
--- a/lib/api/controllers/migration/importFromWallabag.ts
+++ b/lib/api/controllers/migration/importFromWallabag.ts
@@ -27,7 +27,7 @@ type WallabagBackup = {
export default async function importFromWallabag(
userId: number,
- rawData: string
+ rawData: any
) {
const data: WallabagBackup = JSON.parse(rawData);
@@ -96,23 +96,23 @@ export default async function importFromWallabag(
tags:
link.tags && link.tags[0]
? {
- connectOrCreate: link.tags.map((tag) => ({
- where: {
- name_ownerId: {
- name: tag?.trim().slice(0, 49),
- ownerId: userId,
- },
- },
- create: {
+ connectOrCreate: link.tags.map((tag) => ({
+ where: {
+ name_ownerId: {
name: tag?.trim().slice(0, 49),
- owner: {
- connect: {
- id: userId,
- },
+ ownerId: userId,
+ },
+ },
+ create: {
+ name: tag?.trim().slice(0, 49),
+ owner: {
+ connect: {
+ id: userId,
},
},
- })),
- }
+ },
+ })),
+ }
: undefined,
},
});
diff --git a/package.json b/package.json
index 4a8eb6d0..dc24b63e 100644
--- a/package.json
+++ b/package.json
@@ -37,6 +37,8 @@
"@types/nodemailer": "^6.4.8",
"@types/react": "18.2.14",
"@types/react-dom": "18.2.7",
+ "@types/stream-to-blob": "^2.0.0",
+ "@zip.js/zip.js": "^2.7.53",
"axios": "^1.5.1",
"bcrypt": "^5.1.0",
"bootstrap-icons": "^1.11.2",
@@ -79,6 +81,7 @@
"react-window": "^1.8.10",
"rss-parser": "^3.13.0",
"socks-proxy-agent": "^8.0.2",
+ "stream-to-blob": "^2.0.1",
"stripe": "^12.13.0",
"tailwind-merge": "^2.3.0",
"vaul": "^1.1.1",
diff --git a/pages/api/v1/migration/[format].ts b/pages/api/v1/migration/[format].ts
new file mode 100644
index 00000000..9246c79c
--- /dev/null
+++ b/pages/api/v1/migration/[format].ts
@@ -0,0 +1,50 @@
+import type { NextApiRequest, NextApiResponse } from "next";
+import exportData from "@/lib/api/controllers/migration/exportData";
+import importFromHTMLFile from "@/lib/api/controllers/migration/importFromHTMLFile";
+import importFromLinkwarden from "@/lib/api/controllers/migration/importFromLinkwarden";
+import { MigrationFormat, MigrationRequest } from "@/types/global";
+import verifyUser from "@/lib/api/verifyUser";
+import importFromWallabag from "@/lib/api/controllers/migration/importFromWallabag";
+import importFromOmnivore from "@/lib/api/controllers/migration/importFromOmnivore";
+import { Readable } from 'stream';
+
+export const config = {
+ api: {
+ // bodyParser: {
+ // sizeLimit: process.env.IMPORT_LIMIT
+ // ? process.env.IMPORT_LIMIT + "mb"
+ // : "10mb",
+ // },
+ bodyParser: false,
+ },
+};
+
+export default async function users(req: NextApiRequest, res: NextApiResponse) {
+ const user = await verifyUser({ req, res });
+ if (!user) return;
+
+ if (req.method === "POST") {
+ if (process.env.NEXT_PUBLIC_DEMO === "true")
+ return res.status(400).json({
+ response:
+ "This action is disabled because this is a read-only demo of Linkwarden.",
+ });
+
+ const importData: Readable = Readable.from(req); // TODO have something like a limit reader
+ console.log(`received body type ${typeof importData}`);
+ const importFormatName = String(req.query.format) as keyof typeof MigrationFormat;
+ const importFormat: MigrationFormat = MigrationFormat[importFormatName];
+
+ let data;
+ if (importFormat === MigrationFormat.htmlFile)
+ data = await importFromHTMLFile(user.id, importData);
+ else if (importFormat === MigrationFormat.linkwarden)
+ data = await importFromLinkwarden(user.id, importData);
+ else if (importFormat === MigrationFormat.wallabag)
+ data = await importFromWallabag(user.id, importData);
+ else if (importFormat == MigrationFormat.omnivore)
+ data = await importFromOmnivore(user.id, importData);
+
+ if (data) return res.status(data.status).json({ response: data.response });
+ }
+}
diff --git a/pages/api/v1/migration/index.ts b/pages/api/v1/migration/index.ts
index 790af031..41090086 100644
--- a/pages/api/v1/migration/index.ts
+++ b/pages/api/v1/migration/index.ts
@@ -1,21 +1,7 @@
import type { NextApiRequest, NextApiResponse } from "next";
import exportData from "@/lib/api/controllers/migration/exportData";
-import importFromHTMLFile from "@/lib/api/controllers/migration/importFromHTMLFile";
-import importFromLinkwarden from "@/lib/api/controllers/migration/importFromLinkwarden";
-import { MigrationFormat, MigrationRequest } from "@/types/global";
-import verifyUser from "@/lib/api/verifyUser";
-import importFromWallabag from "@/lib/api/controllers/migration/importFromWallabag";
-import importFromOmnivore from "@/lib/api/controllers/migration/importFromOmnivore";
-export const config = {
- api: {
- bodyParser: {
- sizeLimit: process.env.IMPORT_LIMIT
- ? process.env.IMPORT_LIMIT + "mb"
- : "10mb",
- },
- },
-};
+import verifyUser from "@/lib/api/verifyUser";
export default async function users(req: NextApiRequest, res: NextApiResponse) {
const user = await verifyUser({ req, res });
@@ -30,25 +16,5 @@ export default async function users(req: NextApiRequest, res: NextApiResponse) {
.setHeader("Content-Disposition", "attachment; filename=backup.json")
.status(data.status)
.json(data.response);
- } else if (req.method === "POST") {
- if (process.env.NEXT_PUBLIC_DEMO === "true")
- return res.status(400).json({
- response:
- "This action is disabled because this is a read-only demo of Linkwarden.",
- });
-
- const request: MigrationRequest = JSON.parse(req.body);
-
- let data;
- if (request.format === MigrationFormat.htmlFile)
- data = await importFromHTMLFile(user.id, request.data);
- else if (request.format === MigrationFormat.linkwarden)
- data = await importFromLinkwarden(user.id, request.data);
- else if (request.format === MigrationFormat.wallabag)
- data = await importFromWallabag(user.id, request.data);
- else if (request.format == MigrationFormat.omnivore)
- data = await importFromOmnivore(user.id, request.data);
-
- if (data) return res.status(data.status).json({ response: data.response });
- }
+ } // TODO needs handling for invalid request type
}
diff --git a/pages/dashboard.tsx b/pages/dashboard.tsx
index 2c92b4a0..2917f014 100644
--- a/pages/dashboard.tsx
+++ b/pages/dashboard.tsx
@@ -81,21 +81,24 @@ export default function Dashboard() {
if (file) {
const reader = new FileReader();
- reader.readAsText(file, "UTF-8");
+ //reader.readAsText(file, "UTF-8");
+ reader.readAsArrayBuffer(file);
reader.onload = async function (e) {
const load = toast.loading("Importing...");
- const request: string = e.target?.result as string;
+ // const request: string = e.target?.result as string;
- const body: MigrationRequest = {
- format,
- data: request,
- };
+ // const body: MigrationRequest = {
+ // format,
+ // data: request,
+ // };
try {
- const response = await fetch("/api/v1/migration", {
+ const migrationFormatName = MigrationFormat[format];
+ const response = await fetch(`/api/v1/migration/${migrationFormatName}`, {
method: "POST",
- body: JSON.stringify(body),
+ body: e.target?.result as ArrayBuffer,
+ headers: { "Content-Type": "application/octet-stream" },
});
if (!response.ok) {
diff --git a/pages/register.tsx b/pages/register.tsx
index 282558b4..fdb7e966 100644
--- a/pages/register.tsx
+++ b/pages/register.tsx
@@ -149,8 +149,8 @@ export default function Register({
text={
process.env.NEXT_PUBLIC_STRIPE
? t("trial_offer_desc", {
- count: Number(process.env.NEXT_PUBLIC_TRIAL_PERIOD_DAYS || 14),
- })
+ count: Number(process.env.NEXT_PUBLIC_TRIAL_PERIOD_DAYS || 14),
+ })
: t("register_desc")
}
data-testid="registration-form"
diff --git a/yarn.lock b/yarn.lock
index 4dba2268..c079ba73 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -2119,6 +2119,13 @@
"@types/glob" "~7.2.0"
"@types/node" "*"
+"@types/stream-to-blob@^2.0.0":
+ version "2.0.0"
+ resolved "https://registry.yarnpkg.com/@types/stream-to-blob/-/stream-to-blob-2.0.0.tgz#b38b6eb763831a37b91dfe4a5854f9da528acb5c"
+ integrity sha512-tZpPRVwoUMe4ZHr0xt2+RyvkqKFoKzdyCUuzW3NxC4HKxbNEwCMLZiumMuZ+FaizAQJx3rUXVJNXpMCnnOffAw==
+ dependencies:
+ "@types/node" "*"
+
"@types/tough-cookie@*":
version "4.0.3"
resolved "https://registry.yarnpkg.com/@types/tough-cookie/-/tough-cookie-4.0.3.tgz#3d06b6769518450871fbc40770b7586334bdfd90"
@@ -2173,6 +2180,11 @@
"@typescript-eslint/types" "5.49.0"
eslint-visitor-keys "^3.3.0"
+"@zip.js/zip.js@^2.7.53":
+ version "2.7.53"
+ resolved "https://registry.yarnpkg.com/@zip.js/zip.js/-/zip.js-2.7.53.tgz#bf88e90d8eed562182c01339643bc405446b0578"
+ integrity sha512-G6Bl5wN9EXXVaTUIox71vIX5Z454zEBe+akKpV4m1tUboIctT5h7ID3QXCJd/Lfy2rSvmkTmZIucf1jGRR4f5A==
+
abab@^2.0.6:
version "2.0.6"
resolved "https://registry.yarnpkg.com/abab/-/abab-2.0.6.tgz#41b80f2c871d19686216b82309231cfd3cb3d291"
@@ -5756,6 +5768,11 @@ stop-iteration-iterator@^1.0.0:
dependencies:
internal-slot "^1.0.4"
+stream-to-blob@^2.0.1:
+ version "2.0.1"
+ resolved "https://registry.yarnpkg.com/stream-to-blob/-/stream-to-blob-2.0.1.tgz#59ab71d7a7f0bfb899570e886e44d39f4ac4381a"
+ integrity sha512-GXlqXt3svqwIVWoICenix5Poxi4KbCF0BdXXUbpU1X4vq1V8wmjiEIU3aFJzCGNFpKxfbnG0uoowS3nKUgSPYg==
+
streamsearch@^1.1.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/streamsearch/-/streamsearch-1.1.0.tgz#404dd1e2247ca94af554e841a8ef0eaa238da764"