A fully open source & end-to-end encrypted note taking alternative to Evernote.
TypeScript asp-net-coredotnet-coreelectronfosshacktoberfestnote-managmentnote-takingnote-taking-appnotebooknotesnotes-appopen-sourceopensourceprivacyproductivityreactreact-nativereactjsself-hosted
web: make importer error resistant
Abdullah Attaafc9b96a1107
diff --git a/apps/web/src/dialogs/settings/components/importer.tsx b/apps/web/src/dialogs/settings/components/importer.tsxindex ff6ff2a93..8e2fb3c5d 100644--- a/apps/web/src/dialogs/settings/components/importer.tsx+++ b/apps/web/src/dialogs/settings/components/importer.tsx@@ -75,7 +75,7 @@ export function Importer() {<><CheckCircleOutline color="accent" sx={{ mt: 150 }} /><Text variant="body" my={2} sx={{ textAlign: "center" }}>- Import successful. {errors.length} errors occured.+ Import completed. {errors.length} errors occured.</Text><Buttonvariant="secondary"@@ -136,23 +136,29 @@ export function Importer() {</Text></Flex><Button+ variant="accent"onClick={async () => {setIsDone(false);setIsImporting(true);await db.syncer?.acquireLock(async () => {try {- for await (const {- count,- filesRead,- totalFiles- } of importFiles(files)) {- if (notesCounter.current)- notesCounter.current.innerText = `${count}`;- if (importProgress.current)- importProgress.current.style.width = `${- (filesRead / totalFiles) * 100- }%`;+ for await (const message of importFiles(files)) {+ switch (message.type) {+ case "error":+ setErrors((errors) => [...errors, message.error]);+ break;+ case "progress": {+ const { count, filesRead, totalFiles } = message;+ if (notesCounter.current)+ notesCounter.current.innerText = `${count}`;+ if (importProgress.current)+ importProgress.current.style.width = `${+ (filesRead / totalFiles) * 100+ }%`;+ break;+ }+ }}} catch (e) {console.error(e);diff --git a/apps/web/src/utils/importer.ts b/apps/web/src/utils/importer.tsindex 647fa9fb5..5b473aaf4 100644--- a/apps/web/src/utils/importer.ts+++ b/apps/web/src/utils/importer.ts@@ -18,25 +18,24 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.*/import { db } from "../common/db";-import type { Note, Notebook } from "@notesnook-importer/core";+import {+ Note,+ Notebook,+ ContentType+} from "@notesnook-importer/core/dist/src/models";import {ATTACHMENTS_DIRECTORY_NAME,NOTE_DATA_FILENAME} from "@notesnook-importer/core/dist/src/utils/note-stream";-import { Cipher } from "@notesnook/crypto/dist/src/types";import { Reader, Entry } from "./zip-reader";import { path } from "@notesnook-importer/core/dist/src/utils/path";export async function* importFiles(zipFiles: File[]) {- const { default: FS } = await import("../interfaces/fs");for (const zip of zipFiles) {let count = 0;let filesRead = 0;- const attachments: Record<- string,- Omit<Cipher, "cipher" | "format"> & { key: any; chunkSize: number }- > = {};+ const attachments: Record<string, any> = {};const { read, totalFiles } = await Reader(zip);for await (const entry of read()) {@@ -47,43 +46,19 @@ export async function* importFiles(zipFiles: File[]) {);const isNote = !isAttachment && entry.name.endsWith(NOTE_DATA_FILENAME);- if (isAttachment) {- const name = path.basename(entry.name);- if (!name || attachments[name] || db.attachments?.exists(name))- continue;-- const data = await entry.arrayBuffer();- const { hash } = await FS.hashBuffer(new Uint8Array(data));- if (hash !== name) {- throw new Error(`integrity check failed: ${name} !== ${hash}`);- }-- const file = new File([data], name, {- type: "application/octet-stream"- });- const key = await db.attachments?.generateKey();- const cipherData = await FS.writeEncryptedFile(file, key, name);- attachments[name] = { ...cipherData, key };- } else if (isNote) {- const note = await fileToJson<Note>(entry);- for (const attachment of note.attachments || []) {- const cipherData = attachments[attachment.hash];- if (!cipherData || db.attachments?.exists(attachment.hash)) continue;-- await db.attachments?.add({- ...cipherData,- hash: attachment.hash,- hashType: attachment.hashType,- filename: attachment.filename,- type: attachment.mime- });+ try {+ if (isAttachment) {+ await processAttachment(entry, attachments);+ } else if (isNote) {+ await processNote(entry, attachments);+ ++count;}- await importNote(note);-- ++count;+ } catch (e) {+ if (e instanceof Error) yield { type: "error" as const, error: e };}yield {+ type: "progress" as const,count,totalFiles,filesRead@@ -92,7 +67,50 @@ export async function* importFiles(zipFiles: File[]) {}}-async function importNote(note: Note) {+async function processAttachment(+ entry: Entry,+ attachments: Record<string, any>+) {+ const name = path.basename(entry.name);+ if (!name || attachments[name] || db.attachments?.exists(name)) return;++ const { default: FS } = await import("../interfaces/fs");++ const data = await entry.arrayBuffer();+ const { hash } = await FS.hashBuffer(new Uint8Array(data));+ if (hash !== name) {+ throw new Error(`integrity check failed: ${name} !== ${hash}`);+ }++ const file = new File([data], name, {+ type: "application/octet-stream"+ });+ const key = await db.attachments?.generateKey();+ const cipherData = await FS.writeEncryptedFile(file, key, name);+ attachments[name] = { ...cipherData, key };+}++async function processNote(entry: Entry, attachments: Record<string, any>) {+ const note = await fileToJson<Note>(entry);+ for (const attachment of note.attachments || []) {+ const cipherData = attachments[attachment.hash];+ if (!cipherData || db.attachments?.exists(attachment.hash)) continue;++ await db.attachments?.add({+ ...cipherData,+ hash: attachment.hash,+ hashType: attachment.hashType,+ filename: attachment.filename,+ type: attachment.mime+ });+ }++ if (!note.content)+ note.content = {+ data: "<p></p>",+ type: ContentType.HTML+ };+if (note.content?.type === "html") (note.content.type as string) = "tiptap";else throw new Error("Invalid content type: " + note.content?.type);@@ -101,7 +119,7 @@ async function importNote(note: Note) {const noteId = await db.notes?.add(note);for (const nb of notebooks) {- const notebook = await importNotebook(nb);+ const notebook = await importNotebook(nb).catch(() => ({ id: undefined }));if (!notebook.id) continue;await db.notes?.addToNotebook({ id: notebook.id, topic: notebook.topic },@@ -126,9 +144,10 @@ async function importNotebook(title: notebook.notebook,topics: notebook.topic ? [notebook.topic] : []});- nb = db.notebooks?.notebook(nbId).data;+ nb = db.notebooks?.notebook(nbId)?.data;+ if (!nb) return {};}- const topic = nb?.topics.find((t: any) => t.title === notebook.topic);+ const topic = nb.topics.find((t: any) => t.title === notebook.topic);return { id: nb ? nb.id : undefined, topic: topic ? topic.id : undefined };}