Skip to content

⚡️⚡️⚡️ Self-hosted collaborative bookmark manager to collect, read, annotate, and fully preserve what matters, all in one place.

Moving omnivore import to specific binary upload route

dereulenspiegel8932b9929a13
diff --git a/lib/api/controllers/migration/importFromHTMLFile.ts b/lib/api/controllers/migration/importFromHTMLFile.ts
index f255bac0..f7454713 100644
--- a/lib/api/controllers/migration/importFromHTMLFile.ts
+++ b/lib/api/controllers/migration/importFromHTMLFile.ts
@@ -3,15 +3,12 @@ import createFolder from "@/lib/api/storage/createFolder";
import { JSDOM } from "jsdom";
import { parse, Node, Element, TextNode } from "himalaya";
import { hasPassedLimit } from "../../verifyCapacity";
-import { Readable } from "stream";
-import streamToBlob from "stream-to-blob";
export default async function importFromHTMLFile(
userId: number,
- rawStream: Readable
+ rawData: string
) {
- const rawData: Blob = await streamToBlob(rawStream);
- const dom = new JSDOM(await rawData.arrayBuffer());
+ const dom = new JSDOM(rawData);
const document = dom.window.document;
// remove bad tags
diff --git a/lib/api/controllers/migration/importFromLinkwarden.ts b/lib/api/controllers/migration/importFromLinkwarden.ts
index a191d69c..e782846b 100644
--- a/lib/api/controllers/migration/importFromLinkwarden.ts
+++ b/lib/api/controllers/migration/importFromLinkwarden.ts
@@ -2,15 +2,12 @@ import { prisma } from "@/lib/api/db";
import { Backup } from "@/types/global";
import createFolder from "@/lib/api/storage/createFolder";
import { hasPassedLimit } from "../../verifyCapacity";
-import { Readable } from "stream";
-import streamToBlob from "stream-to-blob";
export default async function importFromLinkwarden(
userId: number,
- rawStream: Readable
+ rawData: string
) {
- const rawData: Blob = await streamToBlob(rawStream);
- const data: Backup = JSON.parse(await rawData.text());
+ const data: Backup = JSON.parse(rawData);
let totalImports = 0;
diff --git a/lib/api/controllers/migration/importFromWallabag.ts b/lib/api/controllers/migration/importFromWallabag.ts
index 0c5ad70b..3c5fa001 100644
--- a/lib/api/controllers/migration/importFromWallabag.ts
+++ b/lib/api/controllers/migration/importFromWallabag.ts
@@ -1,8 +1,6 @@
import { prisma } from "@/lib/api/db";
import createFolder from "@/lib/api/storage/createFolder";
import { hasPassedLimit } from "../../verifyCapacity";
-import { Readable } from "stream";
-import streamToBlob from "stream-to-blob";
type WallabagBackup = {
is_archived: number;
@@ -29,10 +27,9 @@ type WallabagBackup = {
export default async function importFromWallabag(
userId: number,
- rawStream: Readable
+ rawData: string
) {
- const rawData: Blob = await streamToBlob(rawStream);
- const data: WallabagBackup = JSON.parse(await rawData.text());
+ const data: WallabagBackup = JSON.parse(rawData);
const backup = data.filter((e) => e.url);
diff --git a/pages/api/v1/migration/[format].ts b/pages/api/v1/migration/[format].ts
deleted file mode 100644
index 212ea379..00000000
--- a/pages/api/v1/migration/[format].ts
+++ /dev/null
@@ -1,56 +0,0 @@
-import type { NextApiRequest, NextApiResponse } from "next";
-import exportData from "@/lib/api/controllers/migration/exportData";
-import importFromHTMLFile from "@/lib/api/controllers/migration/importFromHTMLFile";
-import importFromLinkwarden from "@/lib/api/controllers/migration/importFromLinkwarden";
-import { MigrationFormat, MigrationRequest } from "@/types/global";
-import verifyUser from "@/lib/api/verifyUser";
-import importFromWallabag from "@/lib/api/controllers/migration/importFromWallabag";
-import importFromOmnivore from "@/lib/api/controllers/migration/importFromOmnivore";
-import { Readable } from "stream";
-import getSizeTransform from "stream-size";
-
-export const config = {
- api: {
- // Disable body parser to be able to process binary uploads
- bodyParser: false,
- },
-};
-
-const IMPORT_SIZE_LIMIT_BYTES =
- parseInt(process.env.IMPORT_LIMIT ? process.env.IMPORT_LIMIT : "10") *
- 1024 *
- 1024;
-
-export default async function users(req: NextApiRequest, res: NextApiResponse) {
- const user = await verifyUser({ req, res });
- if (!user) return;
-
- if (req.method === "POST") {
- if (process.env.NEXT_PUBLIC_DEMO === "true")
- return res.status(400).json({
- response:
- "This action is disabled because this is a read-only demo of Linkwarden.",
- });
-
- const unlimitedReadable: Readable = Readable.from(req);
- const importData = unlimitedReadable.pipe(
- getSizeTransform(IMPORT_SIZE_LIMIT_BYTES)
- );
- const importFormatName = String(
- req.query.format
- ) as keyof typeof MigrationFormat;
- const importFormat: MigrationFormat = MigrationFormat[importFormatName];
-
- let data;
- if (importFormat === MigrationFormat.htmlFile)
- data = await importFromHTMLFile(user.id, importData);
- else if (importFormat === MigrationFormat.linkwarden)
- data = await importFromLinkwarden(user.id, importData);
- else if (importFormat === MigrationFormat.wallabag)
- data = await importFromWallabag(user.id, importData);
- else if (importFormat == MigrationFormat.omnivore)
- data = await importFromOmnivore(user.id, importData);
-
- if (data) return res.status(data.status).json({ response: data.response });
- }
-}
diff --git a/pages/api/v1/migration/index.ts b/pages/api/v1/migration/index.ts
index 41090086..4e479847 100644
--- a/pages/api/v1/migration/index.ts
+++ b/pages/api/v1/migration/index.ts
@@ -1,7 +1,20 @@
import type { NextApiRequest, NextApiResponse } from "next";
import exportData from "@/lib/api/controllers/migration/exportData";
-
+import importFromHTMLFile from "@/lib/api/controllers/migration/importFromHTMLFile";
+import importFromLinkwarden from "@/lib/api/controllers/migration/importFromLinkwarden";
+import { MigrationFormat, MigrationRequest } from "@/types/global";
import verifyUser from "@/lib/api/verifyUser";
+import importFromWallabag from "@/lib/api/controllers/migration/importFromWallabag";
+
+export const config = {
+ api: {
+ bodyParser: {
+ sizeLimit: process.env.IMPORT_LIMIT
+ ? process.env.IMPORT_LIMIT + "mb"
+ : "10mb",
+ },
+ },
+};
export default async function users(req: NextApiRequest, res: NextApiResponse) {
const user = await verifyUser({ req, res });
@@ -16,5 +29,23 @@ export default async function users(req: NextApiRequest, res: NextApiResponse) {
.setHeader("Content-Disposition", "attachment; filename=backup.json")
.status(data.status)
.json(data.response);
- } // TODO needs handling for invalid request type
+ } else if (req.method === "POST") {
+ if (process.env.NEXT_PUBLIC_DEMO === "true")
+ return res.status(400).json({
+ response:
+ "This action is disabled because this is a read-only demo of Linkwarden.",
+ });
+
+ const request: MigrationRequest = JSON.parse(req.body);
+
+ let data;
+ if (request.format === MigrationFormat.htmlFile)
+ data = await importFromHTMLFile(user.id, request.data);
+ else if (request.format === MigrationFormat.linkwarden)
+ data = await importFromLinkwarden(user.id, request.data);
+ else if (request.format === MigrationFormat.wallabag)
+ data = await importFromWallabag(user.id, request.data);
+
+ if (data) return res.status(data.status).json({ response: data.response });
+ }
}
diff --git a/pages/api/v1/migration/upload/[format].ts b/pages/api/v1/migration/upload/[format].ts
new file mode 100644
index 00000000..556e7f60
--- /dev/null
+++ b/pages/api/v1/migration/upload/[format].ts
@@ -0,0 +1,38 @@
+import type { NextApiRequest, NextApiResponse } from "next";
+import { MigrationFormat, MigrationRequest } from "@/types/global";
+import verifyUser from "@/lib/api/verifyUser";
+import importFromOmnivore from "@/lib/api/controllers/migration/importFromOmnivore";
+import { Readable } from "stream";
+import getSizeTransform from "stream-size";
+
+export const config = {
+ api: {
+ // Disable body parser to be able to process binary uploads
+ bodyParser: false,
+ },
+};
+
+const IMPORT_SIZE_LIMIT_BYTES =
+ parseInt(process.env.IMPORT_LIMIT ? process.env.IMPORT_LIMIT : "10") *
+ 1024 *
+ 1024;
+
+export default async function users(req: NextApiRequest, res: NextApiResponse) {
+ const user = await verifyUser({ req, res });
+ if (!user) return;
+
+ const unlimitedReadable: Readable = Readable.from(req);
+ const importData = unlimitedReadable.pipe(
+ getSizeTransform(IMPORT_SIZE_LIMIT_BYTES)
+ );
+ const importFormatName = String(
+ req.query.format
+ ) as keyof typeof MigrationFormat;
+ const importFormat: MigrationFormat = MigrationFormat[importFormatName];
+
+ let data;
+ if (importFormat == MigrationFormat.omnivore)
+ data = await importFromOmnivore(user.id, importData);
+
+ if (data) return res.status(data.status).json({ response: data.response });
+}
\ No newline at end of file