Skip to content

A fully open source & end-to-end encrypted note taking alternative to Evernote.

core: use sqlite for storing logs

Abdullah Attab6f95ff7cd28
diff --git a/packages/core/src/api/index.ts b/packages/core/src/api/index.ts
index 8f77ab052..a8e9017d4 100644
--- a/packages/core/src/api/index.ts
+++ b/packages/core/src/api/index.ts
@@ -63,6 +63,7 @@ import { Settings } from "../collections/settings";
import {
DatabaseAccessor,
DatabaseSchema,
+ RawDatabaseSchema,
SQLiteOptions,
changeDatabasePassword,
createDatabase,
@@ -74,7 +75,8 @@ import { Vaults } from "../collections/vaults";
import { KVStorage } from "../database/kv";
import { QueueValue } from "../utils/queue-value";
import { Sanitizer } from "../database/sanitizer";
-import { dropTriggers } from "../database/triggers";
+import { createTriggers, dropTriggers } from "../database/triggers";
+import { NNMigrationProvider } from "../database/migrations";
type EventSourceConstructor = new (
uri: string,
@@ -244,7 +246,10 @@ class Database {
await sql.raw(statement).execute(this.sql());
}
- await initializeDatabase(this.sql().withTables());
+ await initializeDatabase(
+ this.sql().withTables(),
+ new NNMigrationProvider()
+ );
await this.initCollections();
return true;
}
@@ -274,10 +279,11 @@ class Database {
this.disconnectSSE();
});
- this._sql = (await createDatabase(
- "notesnook",
- this.options.sqliteOptions
- )) as unknown as Kysely<DatabaseSchema>;
+ this._sql = (await createDatabase<RawDatabaseSchema>("notesnook", {
+ ...this.options.sqliteOptions,
+ migrationProvider: new NNMigrationProvider(),
+ onInit: (db) => createTriggers(db)
+ })) as unknown as Kysely<DatabaseSchema>;
await this.sanitizer.init();
diff --git a/packages/core/src/database/index.ts b/packages/core/src/database/index.ts
index 021d2060d..505c8f5a8 100644
--- a/packages/core/src/database/index.ts
+++ b/packages/core/src/database/index.ts
@@ -34,7 +34,8 @@ import {
ColumnType,
ExpressionBuilder,
ReferenceExpression,
- Dialect
+ Dialect,
+ MigrationProvider
} from "kysely";
import {
Attachment,
@@ -58,8 +59,6 @@ import {
Vault,
isDeleted
} from "../types";
-import { NNMigrationProvider } from "./migrations";
-import { createTriggers } from "./triggers";
import { logger } from "../logger";
// type FilteredKeys<T, U> = {
@@ -258,10 +257,11 @@ const DataMappers: Partial<Record<ItemType, (row: any) => void>> = {
}
};
-async function setupDatabase(
- db: Kysely<RawDatabaseSchema>,
+async function setupDatabase<Schema>(
+ db: Kysely<Schema>,
options: SQLiteOptions
) {
+ await sql`PRAGMA init;`.execute(db);
if (options.password)
await sql`PRAGMA key = ${sql.ref(options.password)}`.execute(db);
await sql`PRAGMA journal_mode = ${sql.raw(
@@ -295,11 +295,14 @@ async function setupDatabase(
);
}
-export async function initializeDatabase(db: Kysely<RawDatabaseSchema>) {
+export async function initializeDatabase<Schema>(
+ db: Kysely<Schema>,
+ migrationProvider: MigrationProvider
+) {
try {
const migrator = new Migrator({
db,
- provider: new NNMigrationProvider()
+ provider: migrationProvider
});
const { error, results } = await migrator.migrateToLatest();
@@ -314,8 +317,6 @@ export async function initializeDatabase(db: Kysely<RawDatabaseSchema>) {
.join(", ")}`
);
- await createTriggers(db);
-
return db;
} catch (e) {
logger.error(e, "Failed to initialized database.");
@@ -336,8 +337,14 @@ export type SQLiteOptions = {
skipInitialization?: boolean;
};
-export async function createDatabase(name: string, options: SQLiteOptions) {
- const db = new Kysely<RawDatabaseSchema>({
+export async function createDatabase<Schema>(
+ name: string,
+ options: SQLiteOptions & {
+ migrationProvider: MigrationProvider;
+ onInit?: (db: Kysely<Schema>) => Promise<void>;
+ }
+) {
+ const db = new Kysely<Schema>({
// log: (event) => {
// if (event.queryDurationMillis > 5)
// console.warn(event.query.sql, event.queryDurationMillis);
@@ -345,7 +352,8 @@ export async function createDatabase(name: string, options: SQLiteOptions) {
dialect: options.dialect(name, async () => {
await db.connection().execute(async (db) => {
await setupDatabase(db, options);
- await initializeDatabase(db);
+ await initializeDatabase(db, options.migrationProvider);
+ if (options.onInit) await options.onInit(db);
});
}),
plugins: [new SqliteBooleanPlugin()]
@@ -353,7 +361,8 @@ export async function createDatabase(name: string, options: SQLiteOptions) {
if (!options.skipInitialization)
await db.connection().execute(async (db) => {
await setupDatabase(db, options);
- await initializeDatabase(db);
+ await initializeDatabase(db, options.migrationProvider);
+ if (options.onInit) await options.onInit(db);
});
return db;
diff --git a/packages/core/src/logger.ts b/packages/core/src/logger.ts
index 1438215b9..81ba4d55c 100644
--- a/packages/core/src/logger.ts
+++ b/packages/core/src/logger.ts
@@ -28,7 +28,8 @@ import {
format,
ILogger
} from "@notesnook/logger";
-import { IStorage } from "./interfaces";
+import { Kysely, Migration, MigrationProvider, sql } from "kysely";
+import { SQLiteOptions, createDatabase } from "./database";
const WEEK = 86400000 * 7;
@@ -38,10 +39,44 @@ const WEEK = 86400000 * 7;
// 3. Keep 7 days of logs
// 4. Implement functions for log retrieval & filtering
+type SQLiteItem<T> = {
+ [P in keyof T]?: T[P] | null;
+};
+
+export type LogDatabaseSchema = {
+ logs: SQLiteItem<LogMessage>;
+};
+
+class NNLogsMigrationProvider implements MigrationProvider {
+ async getMigrations(): Promise<Record<string, Migration>> {
+ return {
+ "1": {
+ async up(db) {
+ await db.schema
+ .createTable("logs")
+ .addColumn("timestamp", "integer", (c) => c.notNull())
+ .addColumn("message", "text", (c) => c.notNull())
+ .addColumn("level", "integer", (c) => c.notNull())
+ .addColumn("scope", "text")
+ .addColumn("extras", "text")
+ .addColumn("elapsed", "integer")
+ .execute();
+
+ await db.schema
+ .createIndex("log_timestamp_index")
+ .on("logs")
+ .column("timestamp")
+ .execute();
+ }
+ }
+ };
+ }
+}
+
class DatabaseLogReporter {
writer: DatabaseLogWriter;
- constructor(storage: IStorage) {
- this.writer = new DatabaseLogWriter(storage);
+ constructor(db: Kysely<LogDatabaseSchema>) {
+ this.writer = new DatabaseLogWriter(db);
}
write(log: LogMessage) {
@@ -50,10 +85,10 @@ class DatabaseLogReporter {
}
class DatabaseLogWriter {
- private queue: Map<string, LogMessage> = new Map();
+ private queue: LogMessage[] = [];
private hasCleared = false;
- constructor(private readonly storage: IStorage) {
+ constructor(private readonly db: Kysely<LogDatabaseSchema>) {
setInterval(() => {
setTimeout(() => {
if (!this.hasCleared) {
@@ -66,86 +101,90 @@ class DatabaseLogWriter {
}
push(message: LogMessage) {
- const key = new Date(message.timestamp).toLocaleDateString();
- this.queue.set(`${key}:${message.timestamp}`, message);
+ this.queue.push(message);
}
async flush() {
- if (this.queue.size === 0) return;
- const queueCopy = Array.from(this.queue.entries());
- this.queue = new Map();
-
- await this.storage.writeMulti(queueCopy);
+ if (this.queue.length === 0) return;
+ const queueCopy = this.queue.slice();
+ this.queue = [];
+ await this.db.insertInto("logs").values(queueCopy).execute();
}
async rotate() {
- const logKeys = (await this.storage.getAllKeys()).sort();
- const keysToRemove = [];
- for (const key of logKeys) {
- const keyParts = key.split(":");
- if (keyParts.length === 1 || parseInt(keyParts[1]) < Date.now() - WEEK) {
- keysToRemove.push(key);
- }
- }
-
- if (keysToRemove.length) await this.storage.removeMulti(keysToRemove);
+ const range = Date.now() - WEEK;
+ await this.db.deleteFrom("logs").where("timestamp", "<", range).execute();
}
}
class DatabaseLogManager {
- constructor(private readonly storage: IStorage) {}
+ constructor(private readonly db: Kysely<LogDatabaseSchema>) {}
async get() {
- const logKeys = await this.storage.getAllKeys();
- const logEntries = await this.storage.readMulti<LogMessage>(logKeys);
- const logs: Record<string, LogMessage[]> = {};
-
- for (const [logKey, log] of logEntries) {
- const keyParts = logKey.split(":");
- if (keyParts.length === 1) continue;
-
- const key = keyParts[0];
- if (!logs[key]) logs[key] = [];
- logs[key].push(log);
+ const logs = await this.db
+ .selectFrom("logs")
+ .select([
+ "timestamp",
+ "message",
+ "level",
+ "scope",
+ "extras",
+ "elapsed",
+ sql<string>`strftime('%Y-%m-%d', date(timestamp / 1000, 'unixepoch', 'localtime'))`.as(
+ "date"
+ )
+ ])
+ .execute();
+ const groupedLogs: Record<string, LogMessage[]> = {};
+
+ for (const log of logs) {
+ const key = log.date;
+ if (!groupedLogs[key]) groupedLogs[key] = [];
+ groupedLogs[key].push(log as LogMessage);
}
- return Object.keys(logs)
+ return Object.keys(groupedLogs)
.sort((a, b) => b.localeCompare(a, undefined, { numeric: true }))
.map((key) => ({
key,
- logs: logs[key]?.sort((a, b) => a.timestamp - b.timestamp)
+ logs: groupedLogs[key]?.sort((a, b) => a.timestamp - b.timestamp)
}));
}
async clear() {
- const logKeys = await this.storage.getAllKeys();
- await this.storage.removeMulti(logKeys);
+ await this.db.deleteFrom("logs").execute();
}
async delete(key: string) {
- const logKeys = await this.storage.getAllKeys();
- const keysToRemove = [];
- for (const logKey of logKeys) {
- const keyParts = logKey.split(":");
- if (keyParts.length === 1) continue;
- const currKey = keyParts[0];
- if (currKey === key) keysToRemove.push(logKey);
- }
- if (keysToRemove.length) await this.storage.removeMulti(keysToRemove);
+ // const logKeys = await this.storage.getAllKeys();
+ // const keysToRemove = [];
+ // for (const logKey of logKeys) {
+ // const keyParts = logKey.split(":");
+ // if (keyParts.length === 1) continue;
+ // const currKey = keyParts[0];
+ // if (currKey === key) keysToRemove.push(logKey);
+ // }
+ // if (keysToRemove.length) await this.storage.removeMulti(keysToRemove);
}
}
-function initialize(storage: IStorage, disableConsoleLogs?: boolean) {
- if (storage) {
- const reporters: ILogReporter[] = [new DatabaseLogReporter(storage)];
- if (process.env.NODE_ENV !== "production" && !disableConsoleLogs)
- reporters.push(consoleReporter);
- logger = new Logger({
- reporter: combineReporters(reporters),
- lastTime: Date.now()
- });
- logManager = new DatabaseLogManager(storage);
- }
+async function initialize(
+ options: SQLiteOptions,
+ disableConsoleLogs?: boolean
+) {
+ const db = await createDatabase<LogDatabaseSchema>("notesnook-logs", {
+ ...options,
+ migrationProvider: new NNLogsMigrationProvider()
+ });
+
+ const reporters: ILogReporter[] = [new DatabaseLogReporter(db)];
+ if (process.env.NODE_ENV !== "production" && !disableConsoleLogs)
+ reporters.push(consoleReporter);
+ logger = new Logger({
+ reporter: combineReporters(reporters),
+ lastTime: Date.now()
+ });
+ logManager = new DatabaseLogManager(db);
}
let logger: ILogger = new NoopLogger();