A fully open source & end-to-end encrypted note taking alternative to Evernote.
TypeScript asp-net-coredotnet-coreelectronfosshacktoberfestnote-managmentnote-takingnote-taking-appnotebooknotesnotes-appopen-sourceopensourceprivacyproductivityreactreact-nativereactjsself-hosted
fs: get rid of chunks property in file metadata
Abdullah Atta053f70cdb44f
diff --git a/packages/streamable-fs/index.ts b/packages/streamable-fs/index.tsindex d2bc9b758..0a200d7bb 100644--- a/packages/streamable-fs/index.ts+++ b/packages/streamable-fs/index.ts@@ -20,6 +20,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.import FileHandle from "./src/filehandle";import { IFileStorage, IStreamableFS } from "./src/interfaces";import { File } from "./src/types";+import { chunkPrefix } from "./src/utils";export class StreamableFS implements IStreamableFS {/**@@ -37,17 +38,19 @@ export class StreamableFS implements IStreamableFS {const file: File = {filename,size,- type,- chunks: 0+ type};await this.storage.setMetadata(filename, file);- return new FileHandle(this.storage, file);+ return new FileHandle(this.storage, file, []);}async readFile(filename: string): Promise<FileHandle | undefined> {const file = await this.storage.getMetadata(filename);if (!file) return undefined;- return new FileHandle(this.storage, file);+ const chunks = (await this.storage.listChunks(chunkPrefix(filename))).sort(+ (a, b) => a.localeCompare(b, undefined, { numeric: true })+ );+ return new FileHandle(this.storage, file, chunks);}async exists(filename: string): Promise<boolean> {diff --git a/packages/streamable-fs/src/filehandle.ts b/packages/streamable-fs/src/filehandle.tsindex 04f66c8bd..db1a20239 100644--- a/packages/streamable-fs/src/filehandle.ts+++ b/packages/streamable-fs/src/filehandle.ts@@ -20,12 +20,19 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.import FileStreamSource from "./filestreamsource";import { IFileStorage } from "./interfaces";import { File } from "./types";+import { chunkPrefix } from "./utils";export default class FileHandle {- constructor(private readonly storage: IFileStorage, readonly file: File) {}+ constructor(+ private readonly storage: IFileStorage,+ readonly file: File,+ readonly chunks: string[]+ ) {}get readable() {- return new ReadableStream(new FileStreamSource(this.storage, this.file));+ return new ReadableStream(+ new FileStreamSource(this.storage, this.file, this.chunks)+ );}get writeable() {@@ -33,20 +40,22 @@ export default class FileHandle {write: async (chunk, controller) => {if (controller.signal.aborted) return;- await this.storage.writeChunk(- this.getChunkKey(this.file.chunks++),- chunk- );- await this.storage.setMetadata(this.file.filename, this.file);+ const lastOffset = this.lastOffset();+ await this.storage.writeChunk(this.getChunkKey(lastOffset + 1), chunk);+ this.chunks.push(this.getChunkKey(lastOffset + 1));},abort: async () => {- for (let i = 0; i < this.file.chunks; ++i) {- await this.storage.deleteChunk(this.getChunkKey(i));+ for (const chunk of this.chunks) {+ await this.storage.deleteChunk(chunk);}}});}+ async writeChunkAtOffset(offset: number, chunk: Uint8Array) {+ await this.storage.writeChunk(this.getChunkKey(offset), chunk);+ }+async addAdditionalData<T>(key: string, value: T) {this.file.additionalData = this.file.additionalData || {};this.file.additionalData[key] = value;@@ -54,14 +63,14 @@ export default class FileHandle {}async delete() {- for (let i = 0; i < this.file.chunks; ++i) {- await this.storage.deleteChunk(this.getChunkKey(i));+ for (const chunk of this.chunks) {+ await this.storage.deleteChunk(chunk);}await this.storage.deleteMetadata(this.file.filename);}private getChunkKey(offset: number): string {- return `${this.file.filename}-chunk-${offset}`;+ return `${chunkPrefix(this.file.filename)}${offset}`;}async readChunk(offset: number): Promise<Uint8Array | null> {@@ -71,9 +80,9 @@ export default class FileHandle {async readChunks(from: number, length: number): Promise<Blob> {const blobParts: BlobPart[] = [];- for (let i = from; i < Math.min(from + length, this.file.chunks); ++i) {+ for (let i = from; i < from + length; ++i) {const array = await this.readChunk(i);- if (!array) continue;+ if (!array) throw new Error(`No data found for chunk at offset ${i}.`);blobParts.push(array.buffer);}return new Blob(blobParts, { type: this.file.type });@@ -81,8 +90,8 @@ export default class FileHandle {async toBlob() {const blobParts: BlobPart[] = [];- for (let i = 0; i < this.file.chunks; ++i) {- const array = await this.readChunk(i);+ for (const chunk of this.chunks) {+ const array = await this.storage.readChunk(chunk);if (!array) continue;blobParts.push(array.buffer);}@@ -91,11 +100,23 @@ export default class FileHandle {async size() {let size = 0;- for (let i = 0; i < this.file.chunks; ++i) {- const array = await this.readChunk(i);+ for (const chunk of this.chunks) {+ const array = await this.storage.readChunk(chunk);if (!array) continue;size += array.length;}return size;}++ async listChunks() {+ return (+ await this.storage.listChunks(chunkPrefix(this.file.filename))+ ).sort((a, b) => a.localeCompare(b, undefined, { numeric: true }));+ }++ private lastOffset() {+ const lastChunk = this.chunks.at(-1);+ if (!lastChunk) return -1;+ return parseInt(lastChunk.replace(chunkPrefix(this.file.filename), ""));+ }}diff --git a/packages/streamable-fs/src/filestreamsource.ts b/packages/streamable-fs/src/filestreamsource.tsindex 8a554466d..1b86d8ba3 100644--- a/packages/streamable-fs/src/filestreamsource.ts+++ b/packages/streamable-fs/src/filestreamsource.ts@@ -21,30 +21,27 @@ import { File } from "./types";import { IFileStorage } from "./interfaces";export default class FileStreamSource {- private offset = 0;+ private index = 0;constructor(private readonly storage: IFileStorage,- private readonly file: File+ private readonly file: File,+ private readonly chunks: string[]) {}start() {}async pull(controller: ReadableStreamDefaultController<Uint8Array>) {- const data = await this.readChunk(this.offset++);+ const data = await this.readChunk(this.index++);if (data) controller.enqueue(data);- const isFinalChunk = this.offset === this.file.chunks;+ const isFinalChunk = this.index === this.chunks.length;if (isFinalChunk || !data) controller.close();}- private readChunk(offset: number) {- if (offset > this.file.chunks) return;- return this.storage.readChunk(this.getChunkKey(offset));- }-- private getChunkKey(offset: number): string {- return `${this.file.filename}-chunk-${offset}`;+ private readChunk(index: number) {+ if (index > this.chunks.length) return;+ return this.storage.readChunk(this.chunks[index]);}}diff --git a/packages/streamable-fs/src/interfaces.ts b/packages/streamable-fs/src/interfaces.tsindex a4f646c22..74af3382d 100644--- a/packages/streamable-fs/src/interfaces.ts+++ b/packages/streamable-fs/src/interfaces.ts@@ -36,4 +36,5 @@ export interface IFileStorage {writeChunk(chunkName: string, data: Uint8Array): Promise<void>;deleteChunk(chunkName: string): Promise<void>;readChunk(chunkName: string): Promise<Uint8Array | undefined>;+ listChunks(chunkPrefix: string): Promise<string[]>;}diff --git a/packages/streamable-fs/src/types.ts b/packages/streamable-fs/src/types.tsindex accbb60ed..453303779 100644--- a/packages/streamable-fs/src/types.ts+++ b/packages/streamable-fs/src/types.ts@@ -21,6 +21,6 @@ export type File = {filename: string;size: number;type: string;- chunks: number;+ // chunks: number;additionalData?: { [key: string]: unknown };};diff --git a/packages/streamable-fs/src/utils.ts b/packages/streamable-fs/src/utils.tsnew file mode 100644index 000000000..d53be8e67--- /dev/null+++ b/packages/streamable-fs/src/utils.ts@@ -0,0 +1,22 @@+/*+This file is part of the Notesnook project (https://notesnook.com/)++Copyright (C) 2023 Streetwriters (Private) Limited++This program is free software: you can redistribute it and/or modify+it under the terms of the GNU General Public License as published by+the Free Software Foundation, either version 3 of the License, or+(at your option) any later version.++This program is distributed in the hope that it will be useful,+but WITHOUT ANY WARRANTY; without even the implied warranty of+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+GNU General Public License for more details.++You should have received a copy of the GNU General Public License+along with this program. If not, see <http://www.gnu.org/licenses/>.+*/++export function chunkPrefix(filename: string) {+ return `${filename}-chunk-`;+}