Open-source and local-first Slack and Notion alternative that puts you in control of your data
TypeScript chatchat-applicationcrdteditorelectronknowledge-baselocal-firstnotionnotion-alternativerealtime-collaborationself-hostedslacksqliteteam-collaborationwikiyjs
Implement delta updates for documents
Hakan Shehu639b01c092ca
diff --git a/desktop/package-lock.json b/desktop/package-lock.jsonindex 940df84c..c79682c0 100644--- a/desktop/package-lock.json+++ b/desktop/package-lock.json@@ -64,6 +64,7 @@"clsx": "^2.1.1","cmdk": "^1.0.0","date-fns": "^3.6.0",+ "diff": "^7.0.0","electron-squirrel-startup": "^1.0.1","form-data": "^4.0.1","fractional-indexing-jittered": "^0.9.1",@@ -105,6 +106,7 @@"@electron-forge/plugin-vite": "^7.5.0","@electron/fuses": "^1.8.0","@types/better-sqlite3": "^7.6.11",+ "@types/diff": "^5.2.3","@types/is-hotkey": "^0.1.10","@types/lodash": "^4.17.12","@types/mime-types": "^2.1.4",@@ -3670,6 +3672,13 @@"@types/responselike": "^1.0.0"}},+ "node_modules/@types/diff": {+ "version": "5.2.3",+ "resolved": "https://registry.npmjs.org/@types/diff/-/diff-5.2.3.tgz",+ "integrity": "sha512-K0Oqlrq3kQMaO2RhfrNQX5trmt+XLyom88zS0u84nnIcLvFnRUMRRHmrGny5GSM+kNO9IZLARsdQHDzkhAgmrQ==",+ "dev": true,+ "license": "MIT"+ },"node_modules/@types/estree": {"version": "1.0.6","resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.6.tgz",@@ -6150,10 +6159,9 @@"license": "Apache-2.0"},"node_modules/diff": {- "version": "4.0.2",- "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz",- "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==",- "devOptional": true,+ "version": "7.0.0",+ "resolved": "https://registry.npmjs.org/diff/-/diff-7.0.0.tgz",+ "integrity": "sha512-PJWHUb1RFevKCwaFA9RlG5tCd+FO5iRh9A8HEtkmBH2Li03iJriB6m6JIN4rGz3K3JLawI7/veA1xzRKP6ISBw==","license": "BSD-3-Clause","engines": {"node": ">=0.3.1"@@ -13524,6 +13532,16 @@}}},+ "node_modules/ts-node/node_modules/diff": {+ "version": "4.0.2",+ "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz",+ "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==",+ "devOptional": true,+ "license": "BSD-3-Clause",+ "engines": {+ "node": ">=0.3.1"+ }+ },"node_modules/ts-pattern": {"version": "5.5.0","resolved": "https://registry.npmjs.org/ts-pattern/-/ts-pattern-5.5.0.tgz",diff --git a/desktop/package.json b/desktop/package.jsonindex 6a049683..f5d829d3 100644--- a/desktop/package.json+++ b/desktop/package.json@@ -23,6 +23,7 @@"@electron-forge/plugin-vite": "^7.5.0","@electron/fuses": "^1.8.0","@types/better-sqlite3": "^7.6.11",+ "@types/diff": "^5.2.3","@types/is-hotkey": "^0.1.10","@types/lodash": "^4.17.12","@types/mime-types": "^2.1.4",@@ -107,6 +108,7 @@"clsx": "^2.1.1","cmdk": "^1.0.0","date-fns": "^3.6.0",+ "diff": "^7.0.0","electron-squirrel-startup": "^1.0.1","form-data": "^4.0.1","fractional-indexing-jittered": "^0.9.1",diff --git a/desktop/src/lib/editor.ts b/desktop/src/lib/editor.tsnew file mode 100644index 00000000..032e6c10--- /dev/null+++ b/desktop/src/lib/editor.ts@@ -0,0 +1,370 @@+import { EditorNodeTypes } from '@/lib/constants';+import { generateId, getIdTypeFromNode } from '@/lib/id';+import { generateNodeIndex } from '@/lib/nodes';+import { compareString } from '@/lib/utils';+import { NodeBlock, NodeBlockContent } from '@/types/nodes';+import { JSONContent } from '@tiptap/core';+import { isEqual } from 'lodash';+import * as Y from 'yjs';+import { diffChars } from 'diff';++const leafBlockTypes = new Set([+ EditorNodeTypes.Paragraph,+ EditorNodeTypes.Heading1,+ EditorNodeTypes.Heading2,+ EditorNodeTypes.Heading3,+ EditorNodeTypes.HorizontalRule,+ EditorNodeTypes.CodeBlock,+]);++export const mapContentsToBlocks = (+ parentId: string,+ contents: JSONContent[],+ blocksMap: Map<string, NodeBlock>,+): NodeBlock[] => {+ const blocks: NodeBlock[] = [];+ mapAndPushContentsToBlocks(contents, parentId, blocks, blocksMap);+ validateBlocksIndexes(blocks);+ return blocks;+};++const mapAndPushContentsToBlocks = (+ contents: JSONContent[] | null | undefined,+ parentId: string,+ blocks: NodeBlock[],+ blocksMap: Map<string, NodeBlock>,+): void => {+ if (!contents) {+ return;+ }+ contents.map((content) => {+ mapAndPushContentToBlock(content, parentId, blocks, blocksMap);+ });+};++const mapAndPushContentToBlock = (+ content: JSONContent,+ parentId: string,+ blocks: NodeBlock[],+ blocksMap: Map<string, NodeBlock>,+): void => {+ const id = getIdFromContent(content);+ const index = blocksMap.get(id)?.index;+ const attrs =+ (content.attrs &&+ Object.entries(content.attrs).filter(([key]) => key !== 'id')) ??+ [];++ const isLeafBlock = leafBlockTypes.has(content.type);+ const blockContent = isLeafBlock+ ? mapContentsToNodeBlockContents(content.type, content.content)+ : null;++ blocks.push({+ id: id,+ index: index,+ attrs: attrs.length > 0 ? Object.fromEntries(attrs) : null,+ parentId: parentId,+ type: content.type,+ content: blockContent,+ });++ if (!isLeafBlock && content.content) {+ mapAndPushContentsToBlocks(content.content, id, blocks, blocksMap);+ }+};++const mapContentsToNodeBlockContents = (+ type: string,+ contents?: JSONContent[],+): NodeBlockContent[] | null => {+ if (!leafBlockTypes.has(type) || contents == null) {+ return null;+ }++ const nodeBlocks: NodeBlockContent[] = [];+ for (const content of contents) {+ nodeBlocks.push({+ type: content.type,+ text: content.text,+ marks:+ content.marks?.map((mark) => {+ return {+ type: mark.type,+ attrs: mark.attrs,+ };+ }) ?? null,+ });+ }+ return nodeBlocks;+};++export const mapBlocksToContents = (+ parentId: string,+ blocks: NodeBlock[],+): JSONContent[] => {+ const contents: JSONContent[] = [];+ const children = blocks+ .filter((block) => block.parentId === parentId)+ .sort((a, b) => compareString(a.index, b.index));++ for (const child of children) {+ contents.push(mapBlockToContent(child, blocks));+ }++ return contents;+};++const mapBlockToContent = (+ block: NodeBlock,+ blocks: NodeBlock[],+): JSONContent => {+ return {+ type: block.type,+ attrs: {+ id: block.id,+ ...block.attrs,+ },+ content: leafBlockTypes.has(block.type)+ ? mapNodeBlockContentsToContents(block.content)+ : mapBlocksToContents(block.id, blocks),+ };+};++const mapNodeBlockContentsToContents = (+ nodeBlocks: NodeBlockContent[] | null,+): JSONContent[] | null => {+ if (nodeBlocks == null) {+ return null;+ }+ const contents: JSONContent[] = [];+ for (const nodeBlock of nodeBlocks) {+ contents.push({+ type: nodeBlock.type,+ text: nodeBlock.text,+ marks:+ nodeBlock.marks?.map((mark) => {+ return {+ type: mark.type,+ attrs: mark.attrs,+ };+ }) ?? null,+ });+ }+ return contents;+};++const validateBlocksIndexes = (blocks: NodeBlock[]) => {+ //group by parentId+ const groupedBlocks: { [key: string]: NodeBlock[] } = {};+ for (const block of blocks) {+ if (!groupedBlocks[block.parentId]) {+ groupedBlocks[block.parentId] = [];+ }+ groupedBlocks[block.parentId].push(block);+ }+ for (const parentId in groupedBlocks) {+ const blocks = groupedBlocks[parentId];+ for (let i = 0; i < blocks.length; i++) {+ const currentIndex = blocks[i].index;+ const beforeIndex = i === 0 ? null : blocks[i - 1].index;+ // find the lowest index after the current node+ // we do this because sometimes nodes can be ordered in such a way that+ // the current node's index is higher than one of its siblings+ // after the next sibling+ // for example: 1, {current}, 4, 3+ let afterIndex = i === blocks.length - 1 ? null : blocks[i + 1].index;+ for (let j = i + 1; j < blocks.length; j++) {+ if (blocks[j].index < afterIndex) {+ afterIndex = blocks[j].index;+ break;+ }+ }+ // extra check to make sure that the beforeIndex is less than the afterIndex+ // because otherwise the fractional index library will throw an error+ if (afterIndex < beforeIndex) {+ afterIndex = generateNodeIndex(null, beforeIndex);+ } else if (beforeIndex === afterIndex) {+ afterIndex = generateNodeIndex(beforeIndex, null);+ }+ if (+ !currentIndex ||+ currentIndex <= beforeIndex ||+ currentIndex > afterIndex+ ) {+ blocks[i].index = generateNodeIndex(beforeIndex, afterIndex);+ }+ }+ }+};++const getIdFromContent = (content: JSONContent): string => {+ return content.attrs?.id ?? generateId(getIdTypeFromNode(content.type));+};++export const applyChangeToYDoc = (doc: Y.Doc, blocks: NodeBlock[]) => {+ const attributesMap = doc.getMap('attributes');+ if (!attributesMap.has('content')) {+ attributesMap.set('content', new Y.Map());+ }++ const contentMap = attributesMap.get('content') as Y.Map<any>;+ for (const block of blocks) {+ if (!contentMap.has(block.id)) {+ contentMap.set(block.id, new Y.Map());+ }++ const blockMap = contentMap.get(block.id) as Y.Map<any>;+ applyBlockChangesToYDoc(blockMap, block);+ }+};++const applyBlockChangesToYDoc = (blockMap: Y.Map<any>, block: NodeBlock) => {+ if (blockMap.get('id') !== block.id) {+ blockMap.set('id', block.id);+ }++ if (blockMap.get('type') !== block.type) {+ blockMap.set('type', block.type);+ }++ if (blockMap.get('index') !== block.index) {+ blockMap.set('index', block.index);+ }++ if (blockMap.get('parentId') !== block.parentId) {+ blockMap.set('parentId', block.parentId);+ }++ applyBlockAttrsChangesToYDoc(blockMap, block);+ applyBlockContentChangesToYDoc(blockMap, block);+};++const applyBlockAttrsChangesToYDoc = (+ blockMap: Y.Map<any>,+ block: NodeBlock,+) => {+ if (block.attrs === null) {+ if (blockMap.has('attrs')) {+ blockMap.delete('attrs');+ }++ return;+ }++ if (!blockMap.has('attrs')) {+ blockMap.set('attrs', new Y.Map());+ }++ const attrsMap = blockMap.get('attrs') as Y.Map<any>;+ for (const [key, value] of Object.entries(block.attrs)) {+ const existingValue = attrsMap.get(key);+ if (!isEqual(existingValue, value)) {+ attrsMap.set(key, value);+ }+ }+};++const applyBlockContentChangesToYDoc = (+ blockMap: Y.Map<any>,+ block: NodeBlock,+) => {+ if (block.content === null || block.content.length === 0) {+ if (blockMap.has('content')) {+ blockMap.delete('content');+ }++ return;+ }++ if (!blockMap.has('content')) {+ blockMap.set('content', new Y.Array());+ }++ const contentArray = blockMap.get('content') as Y.Array<any>;+ for (let i = 0; i < block.content.length; i++) {+ const blockContent = block.content[i];+ if (contentArray.length > i) {+ const blockContentMap = contentArray.get(i) as Y.Map<any>;+ applyBlockContentItemChangesToYDoc(blockContentMap, blockContent);+ } else {+ const blockContentMap = new Y.Map<any>();+ contentArray.insert(i, [blockContentMap]);+ applyBlockContentItemChangesToYDoc(blockContentMap, blockContent);+ }+ }+};++const applyBlockContentItemChangesToYDoc = (+ blockContentMap: Y.Map<any>,+ blockContent: NodeBlockContent,+) => {+ if (blockContentMap.get('type') !== blockContent.type) {+ blockContentMap.set('type', blockContent.type);+ }++ applyBlockContentTextChangesToYDoc(blockContentMap, blockContent);+ applyBlockContentMarksChangesToYDoc(blockContentMap, blockContent);+};++const applyBlockContentTextChangesToYDoc = (+ blockContentMap: Y.Map<any>,+ blockContent: NodeBlockContent,+) => {+ if (+ blockContent.text === null ||+ blockContent.text === undefined ||+ blockContent.text === ''+ ) {+ if (blockContentMap.has('text')) {+ blockContentMap.delete('text');+ }++ return;+ }++ if (!blockContentMap.has('text')) {+ blockContentMap.set('text', new Y.Text(blockContent.text));+ return;+ }++ const yText = blockContentMap.get('text') as Y.Text;+ const currentText = yText.toString();+ const newText = blockContent.text;++ if (currentText === newText) {+ return;+ }++ const diffs = diffChars(currentText, newText);+ let index = 0;+ for (const diff of diffs) {+ if (diff.added) {+ yText.insert(index, diff.value);+ index += diff.value.length;+ } else if (diff.removed) {+ yText.delete(index, diff.value.length);+ index -= diff.value.length;+ } else {+ index += diff.value.length;+ }+ }+};++const applyBlockContentMarksChangesToYDoc = (+ blockContentMap: Y.Map<any>,+ blockContent: NodeBlockContent,+) => {+ if (blockContent.marks === null || blockContent.marks.length === 0) {+ if (blockContentMap.has('marks')) {+ blockContentMap.delete('marks');+ }++ return;+ }++ const existingMarks = blockContentMap.get('marks');+ if (!isEqual(existingMarks, blockContent.marks)) {+ blockContentMap.set('marks', blockContent.marks);+ }+};diff --git a/desktop/src/main/handlers/mutations/document-save.ts b/desktop/src/main/handlers/mutations/document-save.tsindex cd49c3bd..5f54ddf6 100644--- a/desktop/src/main/handlers/mutations/document-save.ts+++ b/desktop/src/main/handlers/mutations/document-save.ts@@ -1,11 +1,12 @@import * as Y from 'yjs';import { fromUint8Array, toUint8Array } from 'js-base64';-import { mapNode } from '@/lib/nodes';import { databaseManager } from '@/main/data/database-manager';import { MutationHandler, MutationResult } from '@/operations/mutations';import { DocumentSaveMutationInput } from '@/operations/mutations/document-save';import { generateId, IdType } from '@/lib/id';import { LocalUpdateNodeChangeData } from '@/types/sync';+import { applyChangeToYDoc, mapContentsToBlocks } from '@/lib/editor';+import { LocalNodeAttributes, NodeBlock } from '@/types/nodes';export class DocumentSaveMutationHandlerimplements MutationHandler<DocumentSaveMutationInput>@@ -32,29 +33,53 @@ export class DocumentSaveMutationHandler};}- const node = mapNode(document);const versionId = generateId(IdType.Version);const updatedAt = new Date().toISOString();const updates: string[] = [];const doc = new Y.Doc({- guid: node.id,+ guid: document.id,});- Y.applyUpdate(doc, toUint8Array(node.state));+ Y.applyUpdate(doc, toUint8Array(document.state));doc.on('update', (update) => {updates.push(fromUint8Array(update));});- const attributesMap = doc.getMap('attributes');- attributesMap.set('content', input.content.content);+ const attributes = JSON.parse(document.attributes) as LocalNodeAttributes;+ const blocksMap = new Map<string, NodeBlock>();+ if (attributes.content) {+ for (const [key, value] of Object.entries(attributes.content)) {+ blocksMap.set(key, value);+ }+ }++ const blocks = mapContentsToBlocks(+ document.id,+ input.content.content,+ blocksMap,+ );++ doc.transact(() => {+ applyChangeToYDoc(doc, blocks);+ });++ if (updates.length === 0) {+ return {+ output: {+ success: true,+ },+ changes: [],+ };+ }- const attributes = JSON.stringify(attributesMap.toJSON());+ const attributesMap = doc.getMap('attributes');+ const attributesJson = JSON.stringify(attributesMap.toJSON());const encodedState = fromUint8Array(Y.encodeStateAsUpdate(doc));const changeData: LocalUpdateNodeChangeData = {type: 'node_update',- id: node.id,+ id: document.id,updatedAt: updatedAt,updatedBy: input.userId,versionId: versionId,@@ -65,7 +90,7 @@ export class DocumentSaveMutationHandlerawait trx.updateTable('nodes').set({- attributes: attributes,+ attributes: attributesJson,state: encodedState,updated_at: updatedAt,updated_by: input.userId,diff --git a/desktop/src/main/handlers/queries/document-get.ts b/desktop/src/main/handlers/queries/document-get.tsindex c45e50bd..eda7981b 100644--- a/desktop/src/main/handlers/queries/document-get.ts+++ b/desktop/src/main/handlers/queries/document-get.ts@@ -12,8 +12,9 @@ import {} from '@/operations/queries';import { SelectNode } from '@/main/data/workspace/schema';-import { mapNode } from '@/lib/nodes';import { MutationChange } from '@/operations/mutations';+import { LocalNodeAttributes } from '@/types/nodes';+import { mapBlocksToContents } from '@/lib/editor';export class DocumentGetQueryHandlerimplements QueryHandler<DocumentGetQueryInput>@@ -88,8 +89,10 @@ export class DocumentGetQueryHandler};}- const node = mapNode(document);- const contents = node.attributes?.content ?? [];+ const attributes = JSON.parse(document.attributes) as LocalNodeAttributes;+ const nodeBlocks = Object.values(attributes?.content ?? {});+ const contents = mapBlocksToContents(document.id, nodeBlocks);+if (!contents.length) {contents.push({type: 'paragraph',diff --git a/desktop/src/main/handlers/queries/message-list.ts b/desktop/src/main/handlers/queries/message-list.tsindex 7ab9a584..ac6f5d7a 100644--- a/desktop/src/main/handlers/queries/message-list.ts+++ b/desktop/src/main/handlers/queries/message-list.ts@@ -13,6 +13,7 @@ import { mapNode } from '@/lib/nodes';import { UserNode } from '@/types/users';import { compareString } from '@/lib/utils';import { isEqual } from 'lodash';+import { mapBlocksToContents } from '@/lib/editor';export class MessageListQueryHandlerimplements QueryHandler<MessageListQueryInput>@@ -159,10 +160,13 @@ export class MessageListQueryHandlerauthor: author ?? {id: messageNode.createdBy,name: 'Unknown User',- email: '[email protected]',+ email: '[email protected]',avatar: null,},- content: messageNode.attributes.content,+ content: mapBlocksToContents(+ messageNode.id,+ Object.values(messageNode.attributes.content ?? {}),+ ),reactionCounts,};diff --git a/desktop/src/types/editor.ts b/desktop/src/types/editor.tsindex 0fe7d497..c4d6e7e0 100644--- a/desktop/src/types/editor.ts+++ b/desktop/src/types/editor.ts@@ -1,5 +1,4 @@import { Editor, type Range } from '@tiptap/core';-import { LocalNodeAttributes } from '@/types/nodes';import { FC } from 'react';export type EditorCommandProps = {@@ -22,8 +21,3 @@ export type EditorCommand = {handler: (props: EditorCommandProps) => void | Promise<void>;disabled?: boolean;};--export type EditorNode = {- id: string;- attributes: LocalNodeAttributes;-};diff --git a/desktop/src/types/nodes.ts b/desktop/src/types/nodes.tsindex 9fa78ab8..e32a257c 100644--- a/desktop/src/types/nodes.ts+++ b/desktop/src/types/nodes.ts@@ -1,5 +1,3 @@-import { JSONContent } from '@tiptap/core';-export type LocalNode = {id: string;parentId: string | null;@@ -21,19 +19,28 @@ export type LocalNodeAttributes = {type: string;parentId?: string | null;index?: string | null;- content?: JSONContent[] | null;+ content?: Record<string, NodeBlock> | null;[key: string]: any;};export type NodeBlock = {+ id: string;+ type: string;+ index: string;+ parentId: string;+ content: NodeBlockContent[] | null;+ attrs: Record<string, any> | null;+};++export type NodeBlockContent = {type: string;text?: string | null;- marks?: NodeBlockMark[];+ marks?: NodeBlocContentkMark[];};-export type NodeBlockMark = {+export type NodeBlocContentkMark = {type: string;- attrs: any;+ attrs: Record<string, any>;};export type LocalNodeWithChildren = LocalNode & {@@ -60,7 +67,7 @@ export type ServerNodeAttributes = {type: string;parentId?: string | null;index?: string | null;- content?: NodeBlock[] | null;+ content?: NodeBlockContent[] | null;[key: string]: any;};