⚡️⚡️⚡️ Self-hosted collaborative bookmark manager to collect, read, annotate, and fully preserve what matters, all in one place.
TypeScript bookmarkbookmark-managercollaborationnextjsreact-nativeread-it-laterself-hostedtypescript
Moving omnivore import to specific binary upload route
dereulenspiegel8932b9929a13
diff --git a/lib/api/controllers/migration/importFromHTMLFile.ts b/lib/api/controllers/migration/importFromHTMLFile.tsindex f255bac0..f7454713 100644--- a/lib/api/controllers/migration/importFromHTMLFile.ts+++ b/lib/api/controllers/migration/importFromHTMLFile.ts@@ -3,15 +3,12 @@ import createFolder from "@/lib/api/storage/createFolder";import { JSDOM } from "jsdom";import { parse, Node, Element, TextNode } from "himalaya";import { hasPassedLimit } from "../../verifyCapacity";-import { Readable } from "stream";-import streamToBlob from "stream-to-blob";export default async function importFromHTMLFile(userId: number,- rawStream: Readable+ rawData: string) {- const rawData: Blob = await streamToBlob(rawStream);- const dom = new JSDOM(await rawData.arrayBuffer());+ const dom = new JSDOM(rawData);const document = dom.window.document;// remove bad tagsdiff --git a/lib/api/controllers/migration/importFromLinkwarden.ts b/lib/api/controllers/migration/importFromLinkwarden.tsindex a191d69c..e782846b 100644--- a/lib/api/controllers/migration/importFromLinkwarden.ts+++ b/lib/api/controllers/migration/importFromLinkwarden.ts@@ -2,15 +2,12 @@ import { prisma } from "@/lib/api/db";import { Backup } from "@/types/global";import createFolder from "@/lib/api/storage/createFolder";import { hasPassedLimit } from "../../verifyCapacity";-import { Readable } from "stream";-import streamToBlob from "stream-to-blob";export default async function importFromLinkwarden(userId: number,- rawStream: Readable+ rawData: string) {- const rawData: Blob = await streamToBlob(rawStream);- const data: Backup = JSON.parse(await rawData.text());+ const data: Backup = JSON.parse(rawData);let totalImports = 0;diff --git a/lib/api/controllers/migration/importFromWallabag.ts b/lib/api/controllers/migration/importFromWallabag.tsindex 0c5ad70b..3c5fa001 100644--- a/lib/api/controllers/migration/importFromWallabag.ts+++ b/lib/api/controllers/migration/importFromWallabag.ts@@ -1,8 +1,6 @@import { prisma } from "@/lib/api/db";import createFolder from "@/lib/api/storage/createFolder";import { hasPassedLimit } from "../../verifyCapacity";-import { Readable } from "stream";-import streamToBlob from "stream-to-blob";type WallabagBackup = {is_archived: number;@@ -29,10 +27,9 @@ type WallabagBackup = {export default async function importFromWallabag(userId: number,- rawStream: Readable+ rawData: string) {- const rawData: Blob = await streamToBlob(rawStream);- const data: WallabagBackup = JSON.parse(await rawData.text());+ const data: WallabagBackup = JSON.parse(rawData);const backup = data.filter((e) => e.url);diff --git a/pages/api/v1/migration/[format].ts b/pages/api/v1/migration/[format].tsdeleted file mode 100644index 212ea379..00000000--- a/pages/api/v1/migration/[format].ts+++ /dev/null@@ -1,56 +0,0 @@-import type { NextApiRequest, NextApiResponse } from "next";-import exportData from "@/lib/api/controllers/migration/exportData";-import importFromHTMLFile from "@/lib/api/controllers/migration/importFromHTMLFile";-import importFromLinkwarden from "@/lib/api/controllers/migration/importFromLinkwarden";-import { MigrationFormat, MigrationRequest } from "@/types/global";-import verifyUser from "@/lib/api/verifyUser";-import importFromWallabag from "@/lib/api/controllers/migration/importFromWallabag";-import importFromOmnivore from "@/lib/api/controllers/migration/importFromOmnivore";-import { Readable } from "stream";-import getSizeTransform from "stream-size";--export const config = {- api: {- // Disable body parser to be able to process binary uploads- bodyParser: false,- },-};--const IMPORT_SIZE_LIMIT_BYTES =- parseInt(process.env.IMPORT_LIMIT ? process.env.IMPORT_LIMIT : "10") *- 1024 *- 1024;--export default async function users(req: NextApiRequest, res: NextApiResponse) {- const user = await verifyUser({ req, res });- if (!user) return;-- if (req.method === "POST") {- if (process.env.NEXT_PUBLIC_DEMO === "true")- return res.status(400).json({- response:- "This action is disabled because this is a read-only demo of Linkwarden.",- });-- const unlimitedReadable: Readable = Readable.from(req);- const importData = unlimitedReadable.pipe(- getSizeTransform(IMPORT_SIZE_LIMIT_BYTES)- );- const importFormatName = String(- req.query.format- ) as keyof typeof MigrationFormat;- const importFormat: MigrationFormat = MigrationFormat[importFormatName];-- let data;- if (importFormat === MigrationFormat.htmlFile)- data = await importFromHTMLFile(user.id, importData);- else if (importFormat === MigrationFormat.linkwarden)- data = await importFromLinkwarden(user.id, importData);- else if (importFormat === MigrationFormat.wallabag)- data = await importFromWallabag(user.id, importData);- else if (importFormat == MigrationFormat.omnivore)- data = await importFromOmnivore(user.id, importData);-- if (data) return res.status(data.status).json({ response: data.response });- }-}diff --git a/pages/api/v1/migration/index.ts b/pages/api/v1/migration/index.tsindex 41090086..4e479847 100644--- a/pages/api/v1/migration/index.ts+++ b/pages/api/v1/migration/index.ts@@ -1,7 +1,20 @@import type { NextApiRequest, NextApiResponse } from "next";import exportData from "@/lib/api/controllers/migration/exportData";-+import importFromHTMLFile from "@/lib/api/controllers/migration/importFromHTMLFile";+import importFromLinkwarden from "@/lib/api/controllers/migration/importFromLinkwarden";+import { MigrationFormat, MigrationRequest } from "@/types/global";import verifyUser from "@/lib/api/verifyUser";+import importFromWallabag from "@/lib/api/controllers/migration/importFromWallabag";++export const config = {+ api: {+ bodyParser: {+ sizeLimit: process.env.IMPORT_LIMIT+ ? process.env.IMPORT_LIMIT + "mb"+ : "10mb",+ },+ },+};export default async function users(req: NextApiRequest, res: NextApiResponse) {const user = await verifyUser({ req, res });@@ -16,5 +29,23 @@ export default async function users(req: NextApiRequest, res: NextApiResponse) {.setHeader("Content-Disposition", "attachment; filename=backup.json").status(data.status).json(data.response);- } // TODO needs handling for invalid request type+ } else if (req.method === "POST") {+ if (process.env.NEXT_PUBLIC_DEMO === "true")+ return res.status(400).json({+ response:+ "This action is disabled because this is a read-only demo of Linkwarden.",+ });++ const request: MigrationRequest = JSON.parse(req.body);++ let data;+ if (request.format === MigrationFormat.htmlFile)+ data = await importFromHTMLFile(user.id, request.data);+ else if (request.format === MigrationFormat.linkwarden)+ data = await importFromLinkwarden(user.id, request.data);+ else if (request.format === MigrationFormat.wallabag)+ data = await importFromWallabag(user.id, request.data);++ if (data) return res.status(data.status).json({ response: data.response });+ }}diff --git a/pages/api/v1/migration/upload/[format].ts b/pages/api/v1/migration/upload/[format].tsnew file mode 100644index 00000000..556e7f60--- /dev/null+++ b/pages/api/v1/migration/upload/[format].ts@@ -0,0 +1,38 @@+import type { NextApiRequest, NextApiResponse } from "next";+import { MigrationFormat, MigrationRequest } from "@/types/global";+import verifyUser from "@/lib/api/verifyUser";+import importFromOmnivore from "@/lib/api/controllers/migration/importFromOmnivore";+import { Readable } from "stream";+import getSizeTransform from "stream-size";++export const config = {+ api: {+ // Disable body parser to be able to process binary uploads+ bodyParser: false,+ },+};++const IMPORT_SIZE_LIMIT_BYTES =+ parseInt(process.env.IMPORT_LIMIT ? process.env.IMPORT_LIMIT : "10") *+ 1024 *+ 1024;++export default async function users(req: NextApiRequest, res: NextApiResponse) {+ const user = await verifyUser({ req, res });+ if (!user) return;++ const unlimitedReadable: Readable = Readable.from(req);+ const importData = unlimitedReadable.pipe(+ getSizeTransform(IMPORT_SIZE_LIMIT_BYTES)+ );+ const importFormatName = String(+ req.query.format+ ) as keyof typeof MigrationFormat;+ const importFormat: MigrationFormat = MigrationFormat[importFormatName];++ let data;+ if (importFormat == MigrationFormat.omnivore)+ data = await importFromOmnivore(user.id, importData);++ if (data) return res.status(data.status).json({ response: data.response });+}\ No newline at end of file