A fully open source & end-to-end encrypted note taking alternative to Evernote.
TypeScript asp-net-coredotnet-coreelectronfosshacktoberfestnote-managmentnote-takingnote-taking-appnotebooknotesnotes-appopen-sourceopensourceprivacyproductivityreactreact-nativereactjsself-hosted
core: use sqlite for storing logs
Abdullah Attab6f95ff7cd28
diff --git a/packages/core/src/api/index.ts b/packages/core/src/api/index.tsindex 8f77ab052..a8e9017d4 100644--- a/packages/core/src/api/index.ts+++ b/packages/core/src/api/index.ts@@ -63,6 +63,7 @@ import { Settings } from "../collections/settings";import {DatabaseAccessor,DatabaseSchema,+ RawDatabaseSchema,SQLiteOptions,changeDatabasePassword,createDatabase,@@ -74,7 +75,8 @@ import { Vaults } from "../collections/vaults";import { KVStorage } from "../database/kv";import { QueueValue } from "../utils/queue-value";import { Sanitizer } from "../database/sanitizer";-import { dropTriggers } from "../database/triggers";+import { createTriggers, dropTriggers } from "../database/triggers";+import { NNMigrationProvider } from "../database/migrations";type EventSourceConstructor = new (uri: string,@@ -244,7 +246,10 @@ class Database {await sql.raw(statement).execute(this.sql());}- await initializeDatabase(this.sql().withTables());+ await initializeDatabase(+ this.sql().withTables(),+ new NNMigrationProvider()+ );await this.initCollections();return true;}@@ -274,10 +279,11 @@ class Database {this.disconnectSSE();});- this._sql = (await createDatabase(- "notesnook",- this.options.sqliteOptions- )) as unknown as Kysely<DatabaseSchema>;+ this._sql = (await createDatabase<RawDatabaseSchema>("notesnook", {+ ...this.options.sqliteOptions,+ migrationProvider: new NNMigrationProvider(),+ onInit: (db) => createTriggers(db)+ })) as unknown as Kysely<DatabaseSchema>;await this.sanitizer.init();diff --git a/packages/core/src/database/index.ts b/packages/core/src/database/index.tsindex 021d2060d..505c8f5a8 100644--- a/packages/core/src/database/index.ts+++ b/packages/core/src/database/index.ts@@ -34,7 +34,8 @@ import {ColumnType,ExpressionBuilder,ReferenceExpression,- Dialect+ Dialect,+ MigrationProvider} from "kysely";import {Attachment,@@ -58,8 +59,6 @@ import {Vault,isDeleted} from "../types";-import { NNMigrationProvider } from "./migrations";-import { createTriggers } from "./triggers";import { logger } from "../logger";// type FilteredKeys<T, U> = {@@ -258,10 +257,11 @@ const DataMappers: Partial<Record<ItemType, (row: any) => void>> = {}};-async function setupDatabase(- db: Kysely<RawDatabaseSchema>,+async function setupDatabase<Schema>(+ db: Kysely<Schema>,options: SQLiteOptions) {+ await sql`PRAGMA init;`.execute(db);if (options.password)await sql`PRAGMA key = ${sql.ref(options.password)}`.execute(db);await sql`PRAGMA journal_mode = ${sql.raw(@@ -295,11 +295,14 @@ async function setupDatabase();}-export async function initializeDatabase(db: Kysely<RawDatabaseSchema>) {+export async function initializeDatabase<Schema>(+ db: Kysely<Schema>,+ migrationProvider: MigrationProvider+) {try {const migrator = new Migrator({db,- provider: new NNMigrationProvider()+ provider: migrationProvider});const { error, results } = await migrator.migrateToLatest();@@ -314,8 +317,6 @@ export async function initializeDatabase(db: Kysely<RawDatabaseSchema>) {.join(", ")}`);- await createTriggers(db);-return db;} catch (e) {logger.error(e, "Failed to initialized database.");@@ -336,8 +337,14 @@ export type SQLiteOptions = {skipInitialization?: boolean;};-export async function createDatabase(name: string, options: SQLiteOptions) {- const db = new Kysely<RawDatabaseSchema>({+export async function createDatabase<Schema>(+ name: string,+ options: SQLiteOptions & {+ migrationProvider: MigrationProvider;+ onInit?: (db: Kysely<Schema>) => Promise<void>;+ }+) {+ const db = new Kysely<Schema>({// log: (event) => {// if (event.queryDurationMillis > 5)// console.warn(event.query.sql, event.queryDurationMillis);@@ -345,7 +352,8 @@ export async function createDatabase(name: string, options: SQLiteOptions) {dialect: options.dialect(name, async () => {await db.connection().execute(async (db) => {await setupDatabase(db, options);- await initializeDatabase(db);+ await initializeDatabase(db, options.migrationProvider);+ if (options.onInit) await options.onInit(db);});}),plugins: [new SqliteBooleanPlugin()]@@ -353,7 +361,8 @@ export async function createDatabase(name: string, options: SQLiteOptions) {if (!options.skipInitialization)await db.connection().execute(async (db) => {await setupDatabase(db, options);- await initializeDatabase(db);+ await initializeDatabase(db, options.migrationProvider);+ if (options.onInit) await options.onInit(db);});return db;diff --git a/packages/core/src/logger.ts b/packages/core/src/logger.tsindex 1438215b9..81ba4d55c 100644--- a/packages/core/src/logger.ts+++ b/packages/core/src/logger.ts@@ -28,7 +28,8 @@ import {format,ILogger} from "@notesnook/logger";-import { IStorage } from "./interfaces";+import { Kysely, Migration, MigrationProvider, sql } from "kysely";+import { SQLiteOptions, createDatabase } from "./database";const WEEK = 86400000 * 7;@@ -38,10 +39,44 @@ const WEEK = 86400000 * 7;// 3. Keep 7 days of logs// 4. Implement functions for log retrieval & filtering+type SQLiteItem<T> = {+ [P in keyof T]?: T[P] | null;+};++export type LogDatabaseSchema = {+ logs: SQLiteItem<LogMessage>;+};++class NNLogsMigrationProvider implements MigrationProvider {+ async getMigrations(): Promise<Record<string, Migration>> {+ return {+ "1": {+ async up(db) {+ await db.schema+ .createTable("logs")+ .addColumn("timestamp", "integer", (c) => c.notNull())+ .addColumn("message", "text", (c) => c.notNull())+ .addColumn("level", "integer", (c) => c.notNull())+ .addColumn("scope", "text")+ .addColumn("extras", "text")+ .addColumn("elapsed", "integer")+ .execute();++ await db.schema+ .createIndex("log_timestamp_index")+ .on("logs")+ .column("timestamp")+ .execute();+ }+ }+ };+ }+}+class DatabaseLogReporter {writer: DatabaseLogWriter;- constructor(storage: IStorage) {- this.writer = new DatabaseLogWriter(storage);+ constructor(db: Kysely<LogDatabaseSchema>) {+ this.writer = new DatabaseLogWriter(db);}write(log: LogMessage) {@@ -50,10 +85,10 @@ class DatabaseLogReporter {}class DatabaseLogWriter {- private queue: Map<string, LogMessage> = new Map();+ private queue: LogMessage[] = [];private hasCleared = false;- constructor(private readonly storage: IStorage) {+ constructor(private readonly db: Kysely<LogDatabaseSchema>) {setInterval(() => {setTimeout(() => {if (!this.hasCleared) {@@ -66,86 +101,90 @@ class DatabaseLogWriter {}push(message: LogMessage) {- const key = new Date(message.timestamp).toLocaleDateString();- this.queue.set(`${key}:${message.timestamp}`, message);+ this.queue.push(message);}async flush() {- if (this.queue.size === 0) return;- const queueCopy = Array.from(this.queue.entries());- this.queue = new Map();-- await this.storage.writeMulti(queueCopy);+ if (this.queue.length === 0) return;+ const queueCopy = this.queue.slice();+ this.queue = [];+ await this.db.insertInto("logs").values(queueCopy).execute();}async rotate() {- const logKeys = (await this.storage.getAllKeys()).sort();- const keysToRemove = [];- for (const key of logKeys) {- const keyParts = key.split(":");- if (keyParts.length === 1 || parseInt(keyParts[1]) < Date.now() - WEEK) {- keysToRemove.push(key);- }- }-- if (keysToRemove.length) await this.storage.removeMulti(keysToRemove);+ const range = Date.now() - WEEK;+ await this.db.deleteFrom("logs").where("timestamp", "<", range).execute();}}class DatabaseLogManager {- constructor(private readonly storage: IStorage) {}+ constructor(private readonly db: Kysely<LogDatabaseSchema>) {}async get() {- const logKeys = await this.storage.getAllKeys();- const logEntries = await this.storage.readMulti<LogMessage>(logKeys);- const logs: Record<string, LogMessage[]> = {};-- for (const [logKey, log] of logEntries) {- const keyParts = logKey.split(":");- if (keyParts.length === 1) continue;-- const key = keyParts[0];- if (!logs[key]) logs[key] = [];- logs[key].push(log);+ const logs = await this.db+ .selectFrom("logs")+ .select([+ "timestamp",+ "message",+ "level",+ "scope",+ "extras",+ "elapsed",+ sql<string>`strftime('%Y-%m-%d', date(timestamp / 1000, 'unixepoch', 'localtime'))`.as(+ "date"+ )+ ])+ .execute();+ const groupedLogs: Record<string, LogMessage[]> = {};++ for (const log of logs) {+ const key = log.date;+ if (!groupedLogs[key]) groupedLogs[key] = [];+ groupedLogs[key].push(log as LogMessage);}- return Object.keys(logs)+ return Object.keys(groupedLogs).sort((a, b) => b.localeCompare(a, undefined, { numeric: true })).map((key) => ({key,- logs: logs[key]?.sort((a, b) => a.timestamp - b.timestamp)+ logs: groupedLogs[key]?.sort((a, b) => a.timestamp - b.timestamp)}));}async clear() {- const logKeys = await this.storage.getAllKeys();- await this.storage.removeMulti(logKeys);+ await this.db.deleteFrom("logs").execute();}async delete(key: string) {- const logKeys = await this.storage.getAllKeys();- const keysToRemove = [];- for (const logKey of logKeys) {- const keyParts = logKey.split(":");- if (keyParts.length === 1) continue;- const currKey = keyParts[0];- if (currKey === key) keysToRemove.push(logKey);- }- if (keysToRemove.length) await this.storage.removeMulti(keysToRemove);+ // const logKeys = await this.storage.getAllKeys();+ // const keysToRemove = [];+ // for (const logKey of logKeys) {+ // const keyParts = logKey.split(":");+ // if (keyParts.length === 1) continue;+ // const currKey = keyParts[0];+ // if (currKey === key) keysToRemove.push(logKey);+ // }+ // if (keysToRemove.length) await this.storage.removeMulti(keysToRemove);}}-function initialize(storage: IStorage, disableConsoleLogs?: boolean) {- if (storage) {- const reporters: ILogReporter[] = [new DatabaseLogReporter(storage)];- if (process.env.NODE_ENV !== "production" && !disableConsoleLogs)- reporters.push(consoleReporter);- logger = new Logger({- reporter: combineReporters(reporters),- lastTime: Date.now()- });- logManager = new DatabaseLogManager(storage);- }+async function initialize(+ options: SQLiteOptions,+ disableConsoleLogs?: boolean+) {+ const db = await createDatabase<LogDatabaseSchema>("notesnook-logs", {+ ...options,+ migrationProvider: new NNLogsMigrationProvider()+ });++ const reporters: ILogReporter[] = [new DatabaseLogReporter(db)];+ if (process.env.NODE_ENV !== "production" && !disableConsoleLogs)+ reporters.push(consoleReporter);+ logger = new Logger({+ reporter: combineReporters(reporters),+ lastTime: Date.now()+ });+ logManager = new DatabaseLogManager(db);}let logger: ILogger = new NoopLogger();