A fully open source & end-to-end encrypted note taking alternative to Evernote.
TypeScript asp-net-coredotnet-coreelectronfosshacktoberfestnote-managmentnote-takingnote-taking-appnotebooknotesnotes-appopen-sourceopensourceprivacyproductivityreactreact-nativereactjsself-hosted
core: fix data encrypted with the wrong key after a password change
Abdullah Attaa90f418db1e8
diff --git a/packages/core/__e2e__/user-keys.test.ts b/packages/core/__e2e__/user-keys.test.tsnew file mode 100644index 000000000..c79bd3aeb--- /dev/null+++ b/packages/core/__e2e__/user-keys.test.ts@@ -0,0 +1,261 @@+/*+This file is part of the Notesnook project (https://notesnook.com/)++Copyright (C) 2023 Streetwriters (Private) Limited++This program is free software: you can redistribute it and/or modify+it under the terms of the GNU General Public License as published by+the Free Software Foundation, either version 3 of the License, or+(at your option) any later version.++This program is distributed in the hope that it will be useful,+but WITHOUT ANY WARRANTY; without even the implied warranty of+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+GNU General Public License for more details.++You should have received a copy of the GNU General Public License+along with this program. If not, see <http://www.gnu.org/licenses/>.+*/++import { test, expect, vi, afterEach } from "vitest";+import { databaseTest } from "../__tests__/utils/index.ts";+import { KeyManager } from "../src/api/key-manager.js";+import http from "../src/utils/http.js";+import { EVENTS } from "../src/common.js";+import { login, USER } from "./utils.js";++const NEW_PASSWORD = `${USER.password}-new`;++afterEach(() => {+ vi.restoreAllMocks();+});++async function changePassword(db: any) {+ await db.user.changePassword(USER.password, NEW_PASSWORD);+}++async function restorePassword(db: any) {+ await db.user.changePassword(NEW_PASSWORD, USER.password);+}++async function collectNotes(db: any) {+ const chunks: any[] = [];+ for await (const chunk of db.syncer.sync.collector.collect(100))+ if (chunk.type === "note") chunks.push(chunk);+ return chunks;+}++// what another device does when it receives these items+async function receive(db: any, chunks: any[]) {+ await db.user.fetchUser();+ const keys = await db.user.getDataEncryptionKeys();+ for (const chunk of chunks)+ await db.syncer.sync.processChunk(chunk, keys, { type: "full" });+}++function deferred() {+ let resolve!: () => void;+ const promise = new Promise<void>((r) => (resolve = r));+ return { promise, resolve };+}++function keyVersions(chunks: any[]) {+ return chunks.flatMap((c) => c.items.map((i: any) => i.keyVersion));+}++async function unwrapDataEncryptionKey(db: any) {+ return await new KeyManager(db).unwrapKey(+ (+ await db.user.getUser()+ ).dataEncryptionKey,+ await db.user.getMasterKey()+ );+}++async function storeKeysNewerThanMasterKey(db: any) {+ const user = await db.user.getUser();+ // e.g. the app was closed right after a password change stored the user+ await db.user.setUser({+ ...user,+ revision: user.revision + 1,+ dataEncryptionKey: await new KeyManager(db).wrapKey(+ await db.crypto().generateRandomKey(),+ await db.storage().generateCryptoKey(NEW_PASSWORD, user.salt)+ )+ });+}++test(+ "sync during a password change fails instead of using the wrong key",+ () =>+ databaseTest().then(async (db) => {+ await login(db);+ await db.notes.add({ title: "note" });+ const sessionExpired = vi.fn();+ db.eventManager.subscribe(EVENTS.userSessionExpired, sessionExpired);+ let collectDuringChange!: Promise<unknown>;+ const storage = db.storage();+ const derive = storage.deriveCryptoKey.bind(storage);+ // the new user is already stored but the master key is still the old one+ vi.spyOn(storage, "deriveCryptoKey").mockImplementationOnce(+ async (credentials) => {+ collectDuringChange = collectNotes(db).catch((e) => e);+ await collectDuringChange;+ await derive(credentials);+ }+ );++ await changePassword(db);+ try {+ expect(await collectDuringChange).toBeInstanceOf(Error);+ expect(sessionExpired).not.toHaveBeenCalled();+ const pushed = await collectNotes(db);+ expect(keyVersions(pushed)).toStrictEqual([1]);+ await expect(receive(db, pushed)).resolves.toBeUndefined();+ } finally {+ await restorePassword(db);+ }+ }),+ 60000+);++test(+ "keys newer than the master key expire the session",+ () =>+ databaseTest().then(async (db) => {+ await login(db);+ const sessionExpired = vi.fn();+ db.eventManager.subscribe(EVENTS.userSessionExpired, sessionExpired);+ await storeKeysNewerThanMasterKey(db);++ await expect(db.user.getDataEncryptionKeys()).rejects.toThrow();+ expect(sessionExpired).toHaveBeenCalled();+ }),+ 30000+);++test(+ "keys newer than the master key don't expire the session while another tab changes the master key",+ () =>+ databaseTest().then(async (db) => {+ await login(db);+ const sessionExpired = vi.fn();+ db.eventManager.subscribe(EVENTS.userSessionExpired, sessionExpired);+ await storeKeysNewerThanMasterKey(db);+ await db.kv().write("masterKeyChangeStartedAt", Date.now());++ await expect(db.user.getDataEncryptionKeys()).rejects.toThrow();+ expect(sessionExpired).not.toHaveBeenCalled();+ }),+ 30000+);++test(+ "keys newer than the master key expire the session if a master key change was interrupted",+ () =>+ databaseTest().then(async (db) => {+ await login(db);+ const sessionExpired = vi.fn();+ db.eventManager.subscribe(EVENTS.userSessionExpired, sessionExpired);+ await storeKeysNewerThanMasterKey(db);+ await db.kv().write("masterKeyChangeStartedAt", Date.now() - 60 * 1000);++ await expect(db.user.getDataEncryptionKeys()).rejects.toThrow();+ expect(sessionExpired).toHaveBeenCalled();+ }),+ 30000+);++test(+ "older user doesn't replace a newer one",+ () =>+ databaseTest().then(async (db) => {+ await login(db);+ const preChangeUser = await db.user.getUser();+ await changePassword(db);+ try {+ // e.g. written by another tab that still had the old user+ await db.user.setUser(preChangeUser!);++ await expect(unwrapDataEncryptionKey(db)).resolves.toBeDefined();+ await db.notes.add({ title: "note" });+ expect(keyVersions(await collectNotes(db))).toStrictEqual([1]);+ } finally {+ await restorePassword(db);+ }+ }),+ 60000+);++test(+ "user fetched before a password change doesn't overwrite the new keys",+ () =>+ databaseTest().then(async (db) => {+ await login(db);+ const getStarted = deferred();+ const releaseGet = deferred();+ const get = http.get;+ vi.spyOn(http, "get").mockImplementationOnce(async (...args) => {+ const preChangeUser = await get(...args);+ getStarted.resolve();+ await releaseGet.promise;+ return preChangeUser;+ });++ const fetchUser = db.user.fetchUser();+ await getStarted.promise;+ await changePassword(db);+ try {+ releaseGet.resolve();+ await fetchUser;++ await expect(unwrapDataEncryptionKey(db)).resolves.toBeDefined();+ } finally {+ await restorePassword(db);+ }+ }),+ 60000+);++test(+ "password change whose response is lost still switches to the new password",+ () =>+ databaseTest().then(async (db) => {+ await login(db);+ const patch = http.patch.json;+ vi.spyOn(http.patch, "json").mockImplementationOnce(async (...args) => {+ await patch(...args);+ throw new Error("Network error.");+ });++ await changePassword(db);+ try {+ const { salt } = (await db.user.getUser())!;+ expect(await db.user.getMasterKey()).toStrictEqual(+ await db.storage().generateCryptoKey(NEW_PASSWORD, salt)+ );+ await expect(unwrapDataEncryptionKey(db)).resolves.toBeDefined();+ } finally {+ await restorePassword(db);+ }+ }),+ 60000+);++test(+ "password change that failed before changing anything keeps the old password",+ () =>+ databaseTest().then(async (db) => {+ await login(db);+ const masterKey = await db.user.getMasterKey();+ vi.spyOn(http.patch, "json").mockRejectedValueOnce(+ new Error("Network error.")+ );++ await expect(changePassword(db)).rejects.toThrow("Network error.");++ expect(await db.user.getMasterKey()).toStrictEqual(masterKey);+ await expect(unwrapDataEncryptionKey(db)).resolves.toBeDefined();+ }),+ 30000+);diff --git a/packages/core/__tests__/password-change.test.ts b/packages/core/__tests__/password-change.test.tsindex 41f715ca4..7cb7d6cf7 100644--- a/packages/core/__tests__/password-change.test.ts+++ b/packages/core/__tests__/password-change.test.ts@@ -31,16 +31,26 @@ const FULL_USER: any = {subscription: { appId: 0, cancelURL: null, expiry: 0, productId: null, provider: "none", start: 0, plan: "free", status: "trial", updateURL: null, googlePurchaseToken: null }};-vi.mock("../src/utils/http.js", () => ({+// the password change responds with the user and its rewrapped keys+const server: { db?: any } = {};+vi.mock("../src/utils/http.js", async (importOriginal) => ({+ ...(await importOriginal<typeof import("../src/utils/http.js")>()),default: {get: vi.fn().mockResolvedValue(undefined),post: vi.fn().mockResolvedValue(undefined),- patch: { json: vi.fn().mockResolvedValue(undefined) },+ patch: {+ json: vi.fn(async (url: string, body: any) => {+ if (!url.includes("/users/password/")) return;+ const user = await server.db.user.getUser();+ return { ...user, ...body.userKeys, revision: (user.revision ?? 0) + 1 };+ })+ },delete: vi.fn().mockResolvedValue(undefined)}}));async function setupLoggedInUser(db: any, password: string = "oldpassword") {+ server.db = db;const salt = randomBytes(16).toString("base64");const user = { ...FULL_USER, salt };await db.user.setUser(user);diff --git a/packages/core/__tests__/user-keys.test.ts b/packages/core/__tests__/user-keys.test.tsnew file mode 100644index 000000000..b3af999c3--- /dev/null+++ b/packages/core/__tests__/user-keys.test.ts@@ -0,0 +1,401 @@+/*+This file is part of the Notesnook project (https://notesnook.com/)++Copyright (C) 2023 Streetwriters (Private) Limited++This program is free software: you can redistribute it and/or modify+it under the terms of the GNU General Public License as published by+the Free Software Foundation, either version 3 of the License, or+(at your option) any later version.++This program is distributed in the hope that it will be useful,+but WITHOUT ANY WARRANTY; without even the implied warranty of+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+GNU General Public License for more details.++You should have received a copy of the GNU General Public License+along with this program. If not, see <http://www.gnu.org/licenses/>.+*/++// Server states that only a failure produces (keys left pending by a password+// change, a missing key, a session that wasn't revoked). The rest is covered+// against a real server in __e2e__/user-keys.test.ts.+import { test, expect, vi } from "vitest";+import { databaseTest } from "./utils/index.ts";+import { KeyManager } from "../src/api/key-manager.js";+import { randomBytes } from "../src/utils/random.js";++// in-memory fake API server holding the user document+const server: any = { user: undefined, hooks: {} };+const KEY_IDS = [+ "attachmentsKey",+ "monographPasswordsKey",+ "dataEncryptionKey",+ "legacyDataEncryptionKey",+ "inboxKeys"+];+vi.mock("../src/utils/http.js", async (importOriginal) => {+ const original = await importOriginal<+ typeof import("../src/utils/http.js")+ >();+ const conflict = (code: string) =>+ new original.RequestError({ code, description: code, data: undefined });+ const keysChanged = () => conflict("keys_changed");+ const user = () => JSON.parse(JSON.stringify(server.user));+ return {+ ...original,+ default: {+ get: vi.fn(async (url: string) => {+ if (server.hooks.get) return server.hooks.get(url);+ if (url.endsWith("/users")) return user();+ }),+ post: Object.assign(+ vi.fn(async (url: string) => server.hooks.post?.(url)),+ {+ json: vi.fn(async (url: string, body: any) => {+ const pending = url.match(+ /\/users\/keys\/pending\/(commit|discard)/+ );+ if (pending) {+ if (+ Number(new URL(url).searchParams.get("revision")) !==+ server.user.revision+ )+ throw keysChanged();+ if (pending[1] === "commit") {+ server.hooks.onCommitPendingKeys?.();+ Object.assign(server.user, server.user.pendingKeys);+ } else server.hooks.onDiscardPendingKeys?.();+ delete server.user.pendingKeys;+ server.user.revision++;+ return user();+ }+ if (!url.includes("/users/keys")) return;+ await server.hooks.beforeKeysCreate?.(body);+ const revision = new URL(url).searchParams.get("revision");+ if (revision !== null && +revision < server.masterKeyRevision)+ throw conflict("password_changed");+ for (const id in body) {+ if (server.user[id]) continue;+ server.user[id] = body[id];+ server.user.revision++;+ }+ return user();+ })+ }+ ),+ patch: {+ json: vi.fn(async (url: string, body: any) => {+ if (!url.includes("/users/password/")) return;+ server.hooks.beforePasswordPatch?.(body);+ if (+ (body.revision !== undefined &&+ body.revision !== server.user.revision) ||+ KEY_IDS.some((id) => server.user[id] && !body.userKeys[id])+ )+ throw keysChanged();+ Object.assign(server.user, body.userKeys);+ server.masterKeyRevision = ++server.user.revision;+ return user();+ })+ },+ delete: vi.fn().mockResolvedValue(undefined)+ }+ };+});++const SUBSCRIPTION = {+ appId: 0,+ cancelURL: null,+ expiry: 0,+ productId: null,+ provider: "none",+ start: 0,+ plan: "free",+ status: "trial",+ updateURL: null,+ googlePurchaseToken: null+};++// A pre-DEK ("legacy") account: data is encrypted directly with the master key.+async function setupLegacyUser(db: any) {+ server.hooks = {};+ server.masterKeyRevision = 0;+ const salt = randomBytes(16).toString("base64");+ const user: any = {+ id: "user",+ email: "[email protected]",+ isEmailConfirmed: true,+ salt,+ mfa: { isEnabled: false },+ subscription: SUBSCRIPTION,+ revision: 0+ };+ await db.storage().deriveCryptoKey({ password: "old", salt });+ await db.kv().write("token", {+ access_token: "fake-token",+ t: Date.now(),+ expires_in: 3600,+ scope: "notesnook.sync offline_access IdentityServerApi",+ refresh_token: "fake-refresh"+ });+ await db.user.setUser(user);+ await db.kv().write("masterKeyRevision", 0);+ user.attachmentsKey = await new KeyManager(db).wrapKey(+ await db.crypto().generateRandomKey(),+ (await db.user.getMasterKey())!+ );+ server.user = JSON.parse(JSON.stringify(user));+ await db.user.fetchUser();+}++// what logging in does once the server accepted the password+async function logIn(db: any, password: string) {+ const user = await db.user.fetchUser();+ await db.user.setMasterKey(user, { password });+}++function deferred() {+ let resolve!: () => void;+ const promise = new Promise<void>((r) => (resolve = r));+ return { promise, resolve };+}++async function wrapForPassword(db: any, password: string) {+ const { salt } = (await db.user.getUser())!;+ return await new KeyManager(db).wrapKey(+ await db.crypto().generateRandomKey(),+ await db.storage().generateCryptoKey(password, salt)+ );+}++test("key generated during password change is rewrapped with the new master key", () =>+ databaseTest().then(async (db) => {+ await setupLegacyUser(db);+ const createStarted = deferred();+ const releaseCreate = deferred();+ server.hooks.beforeKeysCreate = async () => {+ server.hooks.beforeKeysCreate = undefined;+ createStarted.resolve();+ await releaseCreate.promise;+ };++ // wrapped with the old master key; the request lands after the change+ const generateKey = db.user.getMonographPasswordsKey();+ await createStarted.promise;+ await db.user.changePassword("old", "new");+ releaseCreate.resolve();+ const key = await generateKey;++ const masterKey = (await db.user.getMasterKey())!;+ const keyManager = new KeyManager(db);+ for (const user of [server.user, await db.user.getUser()]) {+ expect(user.dataEncryptionKey).toBeDefined();+ expect(+ await keyManager.unwrapKey(user.monographPasswordsKey, masterKey)+ ).toStrictEqual(key);+ }+ }));++test("concurrent requests for a missing key end up with the same key", () =>+ databaseTest().then(async (db) => {+ await setupLegacyUser(db);+ const user = (await db.user.getUser())!;+ delete user.attachmentsKey;+ delete server.user.attachmentsKey;+ await db.user.setUser(user);++ const [first, second] = await Promise.all([+ db.user.getAttachmentsKey(),+ db.user.getAttachmentsKey()+ ]);++ expect(first).toStrictEqual(second);+ }));++test("missing key created by another device first is adopted", () =>+ databaseTest().then(async (db) => {+ await setupLegacyUser(db);+ const keyManager = new KeyManager(db);+ const masterKey = (await db.user.getMasterKey())!;+ const otherDeviceKey = await db.crypto().generateRandomKey();+ server.hooks.beforeKeysCreate = async () => {+ server.user.monographPasswordsKey = await keyManager.wrapKey(+ otherDeviceKey,+ masterKey+ );+ };++ const key = await db.user.getMonographPasswordsKey();++ expect(key).toStrictEqual(otherDeviceKey);+ expect((await db.user.getUser())?.monographPasswordsKey).toStrictEqual(+ server.user.monographPasswordsKey+ );+ }));++test("password change retries with keys added on another device", () =>+ databaseTest().then(async (db) => {+ await setupLegacyUser(db);+ const keyManager = new KeyManager(db);+ const monographPasswordsKey = await db.crypto().generateRandomKey();+ // added on another device; this device doesn't know about it yet+ server.user.monographPasswordsKey = await keyManager.wrapKey(+ monographPasswordsKey,+ (await db.user.getMasterKey())!+ );++ await db.user.changePassword("old", "new");++ const masterKey = (await db.user.getMasterKey())!;+ expect(+ await keyManager.unwrapKey(server.user.monographPasswordsKey, masterKey)+ ).toStrictEqual(monographPasswordsKey);+ }));++test("password change retries if the user changed since it was fetched", () =>+ databaseTest().then(async (db) => {+ await setupLegacyUser(db);+ // e.g. inbox keys replaced on another device+ server.user.revision = 1;+ const revisions: number[] = [];+ server.hooks.beforePasswordPatch = (body: any) =>+ revisions.push(body.revision);++ await db.user.changePassword("old", "new");++ expect(revisions).toStrictEqual([0, 1]);+ expect((await db.user.getUser())?.revision).toBe(2);+ }));++test("keys of a password change that failed midway are committed when logging in with the new password", () =>+ databaseTest().then(async (db) => {+ await setupLegacyUser(db);+ const keyManager = new KeyManager(db);+ const dataEncryptionKey = await db.crypto().generateRandomKey();+ const { salt } = (await db.user.getUser())!;+ server.user.pendingKeys = {+ dataEncryptionKey: await keyManager.wrapKey(+ dataEncryptionKey,+ await db.storage().generateCryptoKey("new", salt)+ )+ };++ await logIn(db, "new");++ const user = await db.user.getUser();+ expect(user?.pendingKeys).toBeUndefined();+ expect(+ await keyManager.unwrapKey(+ user!.dataEncryptionKey!,+ (await db.user.getMasterKey())!+ )+ ).toStrictEqual(dataEncryptionKey);+ }));++test("logging in fails without changing anything if the pending keys can't be committed", () =>+ databaseTest().then(async (db) => {+ await setupLegacyUser(db);+ server.user.pendingKeys = {+ dataEncryptionKey: await wrapForPassword(db, "new")+ };+ server.hooks.onCommitPendingKeys = () => {+ throw new Error("Network error.");+ };+ const masterKey = await db.user.getMasterKey();++ await expect(logIn(db, "new")).rejects.toThrow("Network error.");++ expect(await db.user.getMasterKey()).toStrictEqual(masterKey);+ expect(await db.kv().read("masterKeyChangeStartedAt")).toBeUndefined();+ }));++test("keys of a password change that failed midway are discarded when logging in with another password", () =>+ databaseTest().then(async (db) => {+ await setupLegacyUser(db);+ server.user.pendingKeys = {+ dataEncryptionKey: await wrapForPassword(db, "new")+ };+ const commit = vi.fn();+ const discard = vi.fn();+ server.hooks.onCommitPendingKeys = commit;+ server.hooks.onDiscardPendingKeys = discard;++ await logIn(db, "old");++ expect(commit).not.toHaveBeenCalled();+ expect(discard).toHaveBeenCalled();+ expect((await db.user.getUser())?.pendingKeys).toBeUndefined();+ expect((await db.user.getUser())?.dataEncryptionKey).toBeUndefined();+ }));++test("logging in with another password succeeds even if the pending keys can't be discarded", () =>+ databaseTest().then(async (db) => {+ await setupLegacyUser(db);+ server.user.pendingKeys = {+ dataEncryptionKey: await wrapForPassword(db, "new")+ };+ server.hooks.onDiscardPendingKeys = () => {+ throw new Error("Network error.");+ };++ await expect(logIn(db, "old")).resolves.toBeUndefined();+ expect(await db.user.getMasterKey()).toBeDefined();+ }));++test("failed login stores nothing", () =>+ databaseTest().then(async (db) => {+ await setupLegacyUser(db);+ await db.kv().delete("user");+ await db.kv().write("token", {+ access_token: "fake-token",+ t: Date.now(),+ expires_in: 3600,+ scope: "auth:grant_types:mfa_password"+ } as any);+ server.hooks.post = async (url: string) => {+ if (url.endsWith("/connect/token"))+ return {+ access_token: "fake-token",+ expires_in: 3600,+ scope: "notesnook.sync offline_access IdentityServerApi",+ refresh_token: "fake-refresh"+ };+ };+ server.user.pendingKeys = {+ dataEncryptionKey: await new KeyManager(db).wrapKey(+ await db.crypto().generateRandomKey(),+ await db.storage().generateCryptoKey("new", server.user.salt)+ )+ };+ server.hooks.onCommitPendingKeys = () => {+ throw new Error("Network error.");+ };++ await expect(+ db.user.authenticatePassword("[email protected]", "new", undefined, true)+ ).rejects.toThrow("Network error.");++ expect(await db.user.getUser()).toBeUndefined();+ }));++test("key created after a password change on another device is rejected", () =>+ databaseTest().then(async (db) => {+ await setupLegacyUser(db);+ const keyManager = new KeyManager(db);+ const oldMasterKey = (await db.user.getMasterKey())!;+ // this device's session wasn't revoked, so it still has the old master key+ const newMasterKey = await db+ .storage()+ .generateCryptoKey("new", server.user.salt);+ server.user.attachmentsKey = await keyManager.rewrapKey(+ server.user.attachmentsKey,+ oldMasterKey,+ newMasterKey+ );+ server.masterKeyRevision = ++server.user.revision;+ await db.user.fetchUser();++ await expect(db.user.getMonographPasswordsKey()).rejects.toThrow();+ expect(server.user.monographPasswordsKey).toBeUndefined();+ }));diff --git a/packages/core/src/api/user-manager.ts b/packages/core/src/api/user-manager.tsindex 5f9149708..9c92733da 100644--- a/packages/core/src/api/user-manager.ts+++ b/packages/core/src/api/user-manager.ts@@ -17,8 +17,8 @@ You should have received a copy of the GNU General Public Licensealong with this program. If not, see <http://www.gnu.org/licenses/>.*/-import { User } from "../types.js";-import http from "../utils/http.js";+import { User, UserKeys } from "../types.js";+import http, { RequestError } from "../utils/http.js";import constants from "../utils/constants.js";import TokenManager from "./token-manager.js";import { EV, EVENTS } from "../common.js";@@ -44,9 +44,32 @@ const ENDPOINTS = {revoke: "/connect/revocation",recoverAccount: "/account/recover",resetUser: "/users/reset",+ userKeys: "/users/keys",+ commitPendingKeys: "/users/keys/pending/commit",+ discardPendingKeys: "/users/keys/pending/discard",+ inboxKeys: "/users/keys/inbox",activateTrial: "/subscriptions/trial"};+const MASTER_KEY_CHANGE_TIMEOUT = 30 * 1000;++function revisionQuery(revision?: number) {+ return revision === undefined ? "" : `?revision=${revision}`;+}++/**+ * Any one of the keys, to check which master key wrapped them.+ */+function anyWrappedKey(keys?: UserKeys) {+ return (+ keys?.dataEncryptionKey ||+ keys?.legacyDataEncryptionKey ||+ keys?.attachmentsKey ||+ keys?.monographPasswordsKey ||+ keys?.inboxKeys?.private+ );+}+class UserManager {private tokenManager: TokenManager;private keyManager: KeyManager;@@ -89,20 +112,19 @@ class UserManager {}));- const user = await this.fetchUser();+ // stored by setMasterKey, so a failed signup stores nothing+ const user = await this.getRemoteUser();if (!user) throw new Error("Failed to fetch user after signup.");await this.db.setLastSynced(0);await this.db.syncer.devices.register();- await this.db.storage().deriveCryptoKey({- password,- salt: user.salt- });-- const masterKey = await this.getMasterKey();- if (!masterKey) throw new Error("User encryption key not generated.");- await this.updateUser({+ // the keys are added before the master key is set so that the master key+ // is never used without a data encryption key+ const masterKey = await this.db+ .storage()+ .generateCryptoKey(password, user.salt);+ const keys: UserKeys = {dataEncryptionKey: await this.keyManager.wrapKey(await this.db.crypto().generateRandomKey(),masterKey@@ -115,9 +137,11 @@ class UserManager {await this.db.crypto().generateRandomKey(),masterKey)- });+ };+ const updatedUser = await this.addUserKeys(keys, user.revision);+ await this.setMasterKey(updatedUser, { password });- this.db.eventManager.publish(EVENTS.userLoggedIn, user);+ this.db.eventManager.publish(EVENTS.userLoggedIn, updatedUser);}async authenticateEmail(email: string) {@@ -209,7 +233,8 @@ class UserManager {}));- const user = await this.fetchUser();+ // stored by setMasterKey, so a failed login stores nothing+ const user = await this.getRemoteUser();if (!user) throw new Error("Failed to fetch user.");if (!sessionExpired) {@@ -217,17 +242,7 @@ class UserManager {await this.db.syncer.devices.register();}- if (usesFallback) {- await this.db.storage().deriveCryptoKeyFallback({- password,- salt: user.salt- });- } else {- await this.db.storage().deriveCryptoKey({- password,- salt: user.salt- });- }+ await this.setMasterKey(user, { password, usesFallback });this.db.eventManager.publish(EVENTS.userLoggedIn, user);} catch (e) {await this.tokenManager.saveToken(token);@@ -278,13 +293,47 @@ class UserManager {}setUser(user: User) {- return this.db.kv().write("user", user);+ return this.db.kv().writeUser(user);}getUser() {return this.db.kv().read("user");}+ /**+ * Stores the user before changing the master key, so the stored keys are+ * never older than the master key: in between, unwrapping them fails+ * instead of data getting encrypted with the wrong key.+ */+ private async setMasterKey(+ user: User,+ credentials: { password: string; usesFallback?: boolean }+ ) {+ // resolved before anything is stored so that a failed commit leaves+ // this device as it was+ const resolvedUser = user.pendingKeys+ ? (await this.resolvePendingKeys(user, credentials)) || user+ : user;++ await this.db.kv().write("masterKeyChangeStartedAt", Date.now());+ try {+ await this.setUser(resolvedUser);+ this.keyManager.clearCache();+ const key = { password: credentials.password, salt: resolvedUser.salt };+ if (credentials.usesFallback)+ await this.db.storage().deriveCryptoKeyFallback(key);+ else await this.db.storage().deriveCryptoKey(key);++ // stored after the master key, so a key wrapped with the old master key+ // is never sent with the new revision+ if (resolvedUser.revision === undefined)+ await this.db.kv().delete("masterKeyRevision");+ else await this.db.kv().write("masterKeyRevision", resolvedUser.revision);+ } finally {+ await this.db.kv().delete("masterKeyChangeStartedAt");+ }+ }+/*** @deprecated*/@@ -303,18 +352,45 @@ class UserManager {return true;}- private async updateUser(partial: Partial<User>) {- const user = await this.getUser();- if (!user) return;-+ /**+ * Adds the keys the user doesn't have yet. The returned user has another+ * device's keys where that device added them first.+ * @param revision revision at which the master key that wrapped the keys+ * was set+ */+ private async addUserKeys(keys: UserKeys, revision?: number): Promise<User> {const token = await this.tokenManager.getAccessToken();- await http.patch.json(- `${constants.API_HOST}${ENDPOINTS.user}`,- partial,+ return await http.post.json(+ `${constants.API_HOST}${ENDPOINTS.userKeys}${revisionQuery(revision)}`,+ keys,token);+ }++ private async getRemoteUser(): Promise<User | undefined> {+ const token = await this.tokenManager.getAccessToken();+ if (!token) return;+ return await http.get(`${constants.API_HOST}${ENDPOINTS.user}`, token);+ }- await this.setUser({ ...user, ...partial });+ private deriveMasterKey(+ salt: string,+ credentials: { password: string; usesFallback?: boolean }+ ) {+ const storage = this.db.storage();+ return credentials.usesFallback+ ? storage.generateCryptoKeyFallback(credentials.password, salt)+ : storage.generateCryptoKey(credentials.password, salt);+ }++ private canUnwrap(key: SerializedKey, cipher: Cipher<"base64">) {+ return this.db+ .storage()+ .decrypt(key, cipher)+ .then(+ () => true,+ () => false+ );}async deleteUser(password: string) {@@ -342,42 +418,82 @@ class UserManager {try {const token = await this.tokenManager.getAccessToken();if (!token) return;- const user = await http.get(+ const fetchedUser: User | undefined = await http.get(`${constants.API_HOST}${ENDPOINTS.user}`,token);- if (user) {- await this.setUser(user);- if (- oldUser &&- (oldUser.subscription.plan !== user.subscription.plan ||- oldUser.subscription.status !== user.subscription.status ||- oldUser.subscription.provider !== user.subscription.provider)- ) {- await this.tokenManager._refreshToken(true);- this.db.eventManager.publish(- EVENTS.userSubscriptionUpdated,- user.subscription- );- }- if (oldUser && !oldUser.isEmailConfirmed && user.isEmailConfirmed)- this.db.eventManager.publish(EVENTS.userEmailConfirmed);- this.db.eventManager.publish(EVENTS.userFetched, user);- return user;- } else {- return oldUser;+ if (!fetchedUser) return oldUser;++ await this.setUser(fetchedUser);+ // keeps a newer stored user, e.g. from a password change+ const user = await this.getUser();+ // only missing if the user logged out while the request was in flight+ if (!user) return;++ if (+ oldUser &&+ (oldUser.subscription.plan !== user.subscription.plan ||+ oldUser.subscription.status !== user.subscription.status ||+ oldUser.subscription.provider !== user.subscription.provider)+ ) {+ await this.tokenManager._refreshToken(true);+ this.db.eventManager.publish(+ EVENTS.userSubscriptionUpdated,+ user.subscription+ );}+ if (oldUser && !oldUser.isEmailConfirmed && user.isEmailConfirmed)+ this.db.eventManager.publish(EVENTS.userEmailConfirmed);+ this.db.eventManager.publish(EVENTS.userFetched, user);+ return user;} catch (e) {logger.error(e, "Error fetching user");- return oldUser;+ return await this.getUser().catch(() => oldUser);}}+ /**+ * Resolves the keys of a password change that failed midway, using the+ * password the user just logged in with: if it unwraps them, the password+ * was changed and they are committed, otherwise they are discarded.+ */+ private async resolvePendingKeys(+ user: User,+ credentials: { password: string; usesFallback?: boolean }+ ): Promise<User | undefined> {+ const verifier = anyWrappedKey(user.pendingKeys);+ if (!verifier) return;++ const masterKey = await this.deriveMasterKey(user.salt, credentials);+ const token = await this.tokenManager.getAccessToken();+ const query = revisionQuery(user.revision);+ if (await this.canUnwrap(masterKey, verifier))+ return await http.post.json(+ `${constants.API_HOST}${ENDPOINTS.commitPendingKeys}${query}`,+ null,+ token+ );++ // the current keys match this password, so discarding is only cleanup+ // that the next login can retry+ return await http.post+ .json(+ `${constants.API_HOST}${ENDPOINTS.discardPendingKeys}${query}`,+ null,+ token+ )+ .catch((e) => {+ logger.error(e, "Could not discard pending keys.");+ });+ }+changePassword(oldPassword: string, newPassword: string) {- return this.updatePassword("change", {- old_password: oldPassword,- new_password: newPassword- });+ return this.retryIfKeysChanged(() =>+ this.updatePassword("change", {+ old_password: oldPassword,+ new_password: newPassword+ })+ );}async changeMarketingConsent(enabled: boolean) {@@ -398,10 +514,33 @@ class UserManager {newPassword: string;encryptionKey: SerializedKey;}) {- return this.updatePassword("reset", {- new_password: options.newPassword,- encryptionKey: options.encryptionKey- });+ return this.retryIfKeysChanged(() =>+ this.updatePassword("reset", {+ new_password: options.newPassword,+ encryptionKey: options.encryptionKey+ })+ );+ }++ /**+ * The server rejects a password change made from an outdated copy of the+ * user (`keys_changed`) and keys wrapped with an old master key+ * (`password_changed`), so we fetch the user and try once more. A key+ * wrapped with this device's master key from before a password change on+ * another device is rejected again.+ */+ private async retryIfKeysChanged<T>(callback: () => Promise<T>) {+ try {+ return await callback();+ } catch (e) {+ if (+ !(e instanceof RequestError) ||+ (e.code !== "keys_changed" && e.code !== "password_changed")+ )+ throw e;+ await this.fetchUser();+ return await callback();+ }}async resetPasswordWithoutRecoveryKey(newPassword: string) {@@ -423,31 +562,18 @@ class UserManager {if (!(await this.resetUser())) throw new Error("Failed to reset user.");- await http.patch.json(- `${constants.API_HOST}/users/password/reset`,+ const updatedUser = await this.patchPassword(+ "reset",{newPassword: await this.db.storage().hash(newPassword, user.email.toLowerCase()),userKeys: updateUserPayload},- token+ token,+ newMasterKey);-- await this.db.storage().deriveCryptoKey({- password: newPassword,- salt: user.salt- });-- this.keyManager.clearCache();- await this.setUser({- ...user,- ...updateUserPayload,- attachmentsKey: undefined,- monographPasswordsKey: undefined,- inboxKeys: undefined,- legacyDataEncryptionKey: undefined- });+ await this.setMasterKey(updatedUser, { password: newPassword });return true;}@@ -481,18 +607,26 @@ class UserManager {logger.info("legacy DEK exists: ", {legacyDataEncryptionKey: !!legacyDataEncryptionKey});- if (legacyDataEncryptionKey)+ try {+ if (legacyDataEncryptionKey)+ keys.push({+ key: await this.keyManager.unwrapKey(+ legacyDataEncryptionKey,+ masterKey+ ),+ version: KEY_VERSION.LEGACY+ });keys.push({- key: await this.keyManager.unwrapKey(- legacyDataEncryptionKey,- masterKey- ),- version: KEY_VERSION.LEGACY+ key: await this.keyManager.unwrapKey(dataEncryptionKey, masterKey),+ version: KEY_VERSION.DEK});- keys.push({- key: await this.keyManager.unwrapKey(dataEncryptionKey, masterKey),- version: KEY_VERSION.DEK- });+ } catch (e) {+ // the stored keys are newer than the master key; unless the master key+ // is being changed, the user has to log in with the new password+ if (!(await this.isMasterKeyChanging()))+ this.db.eventManager.publish(EVENTS.userSessionExpired);+ throw e;+ }logger.info("Keys:", {keys: keys.length,keyVersions: keys.map((k) => k.version)@@ -500,6 +634,18 @@ class UserManager {return keys;}+ /**+ * Whether the master key is being changed, in this or another tab.+ */+ private async isMasterKeyChanging() {+ const startedAt = await this.db.kv().read("masterKeyChangeStartedAt");+ // a change that is older was interrupted, e.g. by the app being closed+ return (+ startedAt !== undefined &&+ Math.abs(Date.now() - startedAt) < MASTER_KEY_CHANGE_TIMEOUT+ );+ }+async getMasterKey(): Promise<SerializedKey | undefined> {const user = await this.getUser();if (!user) return;@@ -527,23 +673,36 @@ class UserManager {}): Promise<UnwrapKeyReturnType<KeyTypeFromId<TId>> | undefined> {try {- const masterKey = await this.getMasterKey();- if (!masterKey) return;-- const wrappedKey = await this.keyManager.get(id);-- if (!wrappedKey) {- const key = await config.generateKey();- await this.updateUser({- [id]: await this.keyManager.wrapKey(key, masterKey)- });- return key as UnwrapKeyReturnType<KeyTypeFromId<TId>>;- }+ return await this.retryIfKeysChanged(async () => {+ // read before the master key, so that a master key change in between+ // can only make the server reject the new key+ const revision = await this.db.kv().read("masterKeyRevision");+ const masterKey = await this.getMasterKey();+ if (!masterKey) return;++ let wrappedKey = (await this.getUser())?.[id] as+ | KeyTypeFromId<TId>+ | undefined;+ if (!wrappedKey) {+ const user = await this.addUserKeys(+ {+ [id]: await this.keyManager.wrapKey(+ await config.generateKey(),+ masterKey+ )+ },+ revision+ );+ await this.setUser(user);+ wrappedKey = user[id] as KeyTypeFromId<TId> | undefined;+ if (!wrappedKey) throw new Error("Key was not saved.");+ }- return (await this.keyManager.unwrapKey(- wrappedKey,- masterKey- )) as UnwrapKeyReturnType<KeyTypeFromId<TId>>;+ return (await this.keyManager.unwrapKey(+ wrappedKey,+ masterKey+ )) as UnwrapKeyReturnType<KeyTypeFromId<TId>>;+ });} catch (e) {logger.error(e, `Could not get ${config.errorContext}.`);if (e instanceof Error)@@ -584,32 +743,36 @@ class UserManager {async discardInboxKeys() {this.keyManager.clearCache();- const user = await this.getUser();- if (!user) return;-const token = await this.tokenManager.getAccessToken();- await http.patch.json(- `${constants.API_HOST}${ENDPOINTS.user}`,- { inboxKeys: { public: null, private: null } },- token+ if (!token) return;+ await this.setUser(+ await http.delete(`${constants.API_HOST}${ENDPOINTS.inboxKeys}`, token));-- await this.setUser({ ...user, inboxKeys: undefined });}async saveInboxKeys(keys: SerializedKeyPair) {- const userEncryptionKey = await this.getMasterKey();- if (!userEncryptionKey) return;-- const updatePayload = {- inboxKeys: {- public: keys.publicKey,- private: await this.db- .storage()- .encrypt(userEncryptionKey, keys.privateKey)- }- };- await this.updateUser(updatePayload);+ await this.retryIfKeysChanged(async () => {+ // read before the master key, see getUserKey+ const revision = await this.db.kv().read("masterKeyRevision");+ const userEncryptionKey = await this.getMasterKey();+ const token = await this.tokenManager.getAccessToken();+ if (!userEncryptionKey || !token) return;++ await this.setUser(+ await http.put.json(+ `${constants.API_HOST}${ENDPOINTS.inboxKeys}${revisionQuery(+ revision+ )}`,+ {+ public: keys.publicKey,+ private: await this.db+ .storage()+ .encrypt(userEncryptionKey, keys.privateKey)+ },+ token+ )+ );+ });this.keyManager.clearCache();}@@ -703,12 +866,7 @@ class UserManager {}for (const verifier of verifiers) {- const decryptedData = await this.db- .storage()- .decrypt(key, verifier)- .then(() => true)- .catch(() => false);- if (!decryptedData)+ if (!(await this.canUnwrap(key, verifier)))throw new Error("Your data cannot be decrypted using the provided encryption key.");@@ -727,7 +885,9 @@ class UserManager {if (!new_password) throw new Error("New password is required.");const token = await this.tokenManager.getAccessToken();- const user = await this.getUser();+ let user = await this.getUser();+ // stored by an older version, which can't tell if the keys are outdated+ if (user && user.revision === undefined) user = await this.fetchUser();if (!token || !user) throw new Error("You are not logged in.");const { email, salt } = user;@@ -804,27 +964,56 @@ class UserManager {}): null;- await http.patch.json(- `${constants.API_HOST}/users/password/${type}`,+ const updatedUser = await this.patchPassword(+ type,{oldPassword: oldPassword,newPassword: await this.db.storage().hash(new_password, email.toLowerCase()),- userKeys: updateUserPayload+ userKeys: updateUserPayload,+ // lets the server reject keys rewrapped from an outdated user+ revision: user.revision},- token+ token,+ newMasterKey);+ await this.setMasterKey(updatedUser, { password: new_password });- await this.db.storage().deriveCryptoKey({- password: new_password,- salt- });+ return true;+ }- this.keyManager.clearCache();- await this.setUser({ ...user, ...updateUserPayload });+ /**+ * Changes the password and returns the user with the rewrapped keys. A+ * failed request (e.g. a lost response) might still have changed the+ * password, so the user is fetched to check if its keys are rewrapped.+ */+ private async patchPassword(+ type: "change" | "reset",+ body: Record<string, unknown>,+ token: string,+ newMasterKey: SerializedKey+ ): Promise<User> {+ try {+ return await http.patch.json(+ `${constants.API_HOST}/users/password/${type}`,+ body,+ token+ );+ } catch (e) {+ // rejected before anything was changed+ if (+ e instanceof RequestError &&+ (e.code === "keys_changed" || e.code === "password_change_in_progress")+ )+ throw e;- return true;+ const user = await this.getRemoteUser().catch(() => undefined);+ const verifier = anyWrappedKey(user);+ if (!user || !verifier || !(await this.canUnwrap(newMasterKey, verifier)))+ throw e;+ return user;+ }}private async usesFallbackPWHash(password: string) {diff --git a/packages/core/src/database/kv.ts b/packages/core/src/database/kv.tsindex 7d9266d23..7fada4366 100644--- a/packages/core/src/database/kv.ts+++ b/packages/core/src/database/kv.ts@@ -17,6 +17,7 @@ You should have received a copy of the GNU General Public Licensealong with this program. If not, see <http://www.gnu.org/licenses/>.*/+import { sql } from "@streetwriters/kysely";import { LazyDatabaseAccessor, RawDatabaseSchema } from "./index.js";import { Token } from "../api/token-manager.js";import { User } from "../types.js";@@ -30,6 +31,11 @@ interface KV {deviceId: string;lastBackupTime: number;fullOfflineMode: boolean;+ masterKeyChangeStartedAt: number;+ /**+ * Revision of the user when the current master key was set.+ */+ masterKeyRevision: number;}export const KEYS: (keyof KV)[] = [@@ -75,6 +81,36 @@ export class KVStorage {);}+ /**+ * Writes the user unless the stored user (of the same account) has a newer+ * revision, so a user fetched before a change to its keys can never replace+ * the changed user.+ */+ async writeUser(user: User) {+ await this.db.then((db) =>+ db+ .insertInto("kv")+ .values({+ key: "user",+ value: JSON.stringify(user),+ dateModified: Date.now()+ })+ .onConflict((oc) =>+ oc+ .column("key")+ .doUpdateSet({+ value: sql`excluded.value`,+ dateModified: sql`excluded.dateModified`+ })+ .where(+ sql<boolean>`json_extract(kv.value, '$.id') is not json_extract(excluded.value, '$.id')+ or coalesce(json_extract(kv.value, '$.revision'), -1) <= coalesce(json_extract(excluded.value, '$.revision'), -1)`+ )+ )+ .execute()+ );+ }+async delete<T extends keyof KV>(key: T) {await this.db.then((db) =>db.deleteFrom("kv").where("key", "==", key).execute()diff --git a/packages/core/src/types.ts b/packages/core/src/types.tsindex c7e1c97d4..41f9b271a 100644--- a/packages/core/src/types.ts+++ b/packages/core/src/types.ts@@ -640,6 +640,14 @@ export type User = {inboxKeys?: { public: string; private: Cipher<"base64"> };dataEncryptionKey?: Cipher<"base64">;legacyDataEncryptionKey?: Cipher<"base64">;+ /**+ * Incremented by the server on every change to the user's keys.+ */+ revision?: number;+ /**+ * Keys of a password change that is in progress or failed midway.+ */+ pendingKeys?: UserKeys;marketingConsent?: boolean;storageUsed?: number;@@ -666,6 +674,17 @@ export type User = {};};+export type UserKeys = Partial<+ Pick<+ User,+ | "attachmentsKey"+ | "monographPasswordsKey"+ | "inboxKeys"+ | "dataEncryptionKey"+ | "legacyDataEncryptionKey"+ >+>;+export type Profile = {fullName?: string;profilePicture?: string;diff --git a/packages/core/src/utils/http.ts b/packages/core/src/utils/http.tsindex 75b8a1721..c344e3729 100644--- a/packages/core/src/utils/http.ts+++ b/packages/core/src/utils/http.ts@@ -61,9 +61,22 @@ post.json = function (url: string, data: JsonRequestBody, token?: string) {);};+const put = {+ json(url: string, data: JsonRequestBody, token?: string) {+ return bodyRequest(+ url,+ transformJson(data),+ token,+ "PUT",+ "application/json"+ );+ }+};+export default {get,post,+ put,delete: deleteRequest,patch};