Skip to content

Open-source and local-first Slack and Notion alternative that puts you in control of your data

Implement delta updates for documents

Hakan Shehu639b01c092ca
diff --git a/desktop/package-lock.json b/desktop/package-lock.json
index 940df84c..c79682c0 100644
--- a/desktop/package-lock.json
+++ b/desktop/package-lock.json
@@ -64,6 +64,7 @@
"clsx": "^2.1.1",
"cmdk": "^1.0.0",
"date-fns": "^3.6.0",
+ "diff": "^7.0.0",
"electron-squirrel-startup": "^1.0.1",
"form-data": "^4.0.1",
"fractional-indexing-jittered": "^0.9.1",
@@ -105,6 +106,7 @@
"@electron-forge/plugin-vite": "^7.5.0",
"@electron/fuses": "^1.8.0",
"@types/better-sqlite3": "^7.6.11",
+ "@types/diff": "^5.2.3",
"@types/is-hotkey": "^0.1.10",
"@types/lodash": "^4.17.12",
"@types/mime-types": "^2.1.4",
@@ -3670,6 +3672,13 @@
"@types/responselike": "^1.0.0"
}
},
+ "node_modules/@types/diff": {
+ "version": "5.2.3",
+ "resolved": "https://registry.npmjs.org/@types/diff/-/diff-5.2.3.tgz",
+ "integrity": "sha512-K0Oqlrq3kQMaO2RhfrNQX5trmt+XLyom88zS0u84nnIcLvFnRUMRRHmrGny5GSM+kNO9IZLARsdQHDzkhAgmrQ==",
+ "dev": true,
+ "license": "MIT"
+ },
"node_modules/@types/estree": {
"version": "1.0.6",
"resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.6.tgz",
@@ -6150,10 +6159,9 @@
"license": "Apache-2.0"
},
"node_modules/diff": {
- "version": "4.0.2",
- "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz",
- "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==",
- "devOptional": true,
+ "version": "7.0.0",
+ "resolved": "https://registry.npmjs.org/diff/-/diff-7.0.0.tgz",
+ "integrity": "sha512-PJWHUb1RFevKCwaFA9RlG5tCd+FO5iRh9A8HEtkmBH2Li03iJriB6m6JIN4rGz3K3JLawI7/veA1xzRKP6ISBw==",
"license": "BSD-3-Clause",
"engines": {
"node": ">=0.3.1"
@@ -13524,6 +13532,16 @@
}
}
},
+ "node_modules/ts-node/node_modules/diff": {
+ "version": "4.0.2",
+ "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz",
+ "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==",
+ "devOptional": true,
+ "license": "BSD-3-Clause",
+ "engines": {
+ "node": ">=0.3.1"
+ }
+ },
"node_modules/ts-pattern": {
"version": "5.5.0",
"resolved": "https://registry.npmjs.org/ts-pattern/-/ts-pattern-5.5.0.tgz",
diff --git a/desktop/package.json b/desktop/package.json
index 6a049683..f5d829d3 100644
--- a/desktop/package.json
+++ b/desktop/package.json
@@ -23,6 +23,7 @@
"@electron-forge/plugin-vite": "^7.5.0",
"@electron/fuses": "^1.8.0",
"@types/better-sqlite3": "^7.6.11",
+ "@types/diff": "^5.2.3",
"@types/is-hotkey": "^0.1.10",
"@types/lodash": "^4.17.12",
"@types/mime-types": "^2.1.4",
@@ -107,6 +108,7 @@
"clsx": "^2.1.1",
"cmdk": "^1.0.0",
"date-fns": "^3.6.0",
+ "diff": "^7.0.0",
"electron-squirrel-startup": "^1.0.1",
"form-data": "^4.0.1",
"fractional-indexing-jittered": "^0.9.1",
diff --git a/desktop/src/lib/editor.ts b/desktop/src/lib/editor.ts
new file mode 100644
index 00000000..032e6c10
--- /dev/null
+++ b/desktop/src/lib/editor.ts
@@ -0,0 +1,370 @@
+import { EditorNodeTypes } from '@/lib/constants';
+import { generateId, getIdTypeFromNode } from '@/lib/id';
+import { generateNodeIndex } from '@/lib/nodes';
+import { compareString } from '@/lib/utils';
+import { NodeBlock, NodeBlockContent } from '@/types/nodes';
+import { JSONContent } from '@tiptap/core';
+import { isEqual } from 'lodash';
+import * as Y from 'yjs';
+import { diffChars } from 'diff';
+
+const leafBlockTypes = new Set([
+ EditorNodeTypes.Paragraph,
+ EditorNodeTypes.Heading1,
+ EditorNodeTypes.Heading2,
+ EditorNodeTypes.Heading3,
+ EditorNodeTypes.HorizontalRule,
+ EditorNodeTypes.CodeBlock,
+]);
+
+export const mapContentsToBlocks = (
+ parentId: string,
+ contents: JSONContent[],
+ blocksMap: Map<string, NodeBlock>,
+): NodeBlock[] => {
+ const blocks: NodeBlock[] = [];
+ mapAndPushContentsToBlocks(contents, parentId, blocks, blocksMap);
+ validateBlocksIndexes(blocks);
+ return blocks;
+};
+
+const mapAndPushContentsToBlocks = (
+ contents: JSONContent[] | null | undefined,
+ parentId: string,
+ blocks: NodeBlock[],
+ blocksMap: Map<string, NodeBlock>,
+): void => {
+ if (!contents) {
+ return;
+ }
+ contents.map((content) => {
+ mapAndPushContentToBlock(content, parentId, blocks, blocksMap);
+ });
+};
+
+const mapAndPushContentToBlock = (
+ content: JSONContent,
+ parentId: string,
+ blocks: NodeBlock[],
+ blocksMap: Map<string, NodeBlock>,
+): void => {
+ const id = getIdFromContent(content);
+ const index = blocksMap.get(id)?.index;
+ const attrs =
+ (content.attrs &&
+ Object.entries(content.attrs).filter(([key]) => key !== 'id')) ??
+ [];
+
+ const isLeafBlock = leafBlockTypes.has(content.type);
+ const blockContent = isLeafBlock
+ ? mapContentsToNodeBlockContents(content.type, content.content)
+ : null;
+
+ blocks.push({
+ id: id,
+ index: index,
+ attrs: attrs.length > 0 ? Object.fromEntries(attrs) : null,
+ parentId: parentId,
+ type: content.type,
+ content: blockContent,
+ });
+
+ if (!isLeafBlock && content.content) {
+ mapAndPushContentsToBlocks(content.content, id, blocks, blocksMap);
+ }
+};
+
+const mapContentsToNodeBlockContents = (
+ type: string,
+ contents?: JSONContent[],
+): NodeBlockContent[] | null => {
+ if (!leafBlockTypes.has(type) || contents == null) {
+ return null;
+ }
+
+ const nodeBlocks: NodeBlockContent[] = [];
+ for (const content of contents) {
+ nodeBlocks.push({
+ type: content.type,
+ text: content.text,
+ marks:
+ content.marks?.map((mark) => {
+ return {
+ type: mark.type,
+ attrs: mark.attrs,
+ };
+ }) ?? null,
+ });
+ }
+ return nodeBlocks;
+};
+
+export const mapBlocksToContents = (
+ parentId: string,
+ blocks: NodeBlock[],
+): JSONContent[] => {
+ const contents: JSONContent[] = [];
+ const children = blocks
+ .filter((block) => block.parentId === parentId)
+ .sort((a, b) => compareString(a.index, b.index));
+
+ for (const child of children) {
+ contents.push(mapBlockToContent(child, blocks));
+ }
+
+ return contents;
+};
+
+const mapBlockToContent = (
+ block: NodeBlock,
+ blocks: NodeBlock[],
+): JSONContent => {
+ return {
+ type: block.type,
+ attrs: {
+ id: block.id,
+ ...block.attrs,
+ },
+ content: leafBlockTypes.has(block.type)
+ ? mapNodeBlockContentsToContents(block.content)
+ : mapBlocksToContents(block.id, blocks),
+ };
+};
+
+const mapNodeBlockContentsToContents = (
+ nodeBlocks: NodeBlockContent[] | null,
+): JSONContent[] | null => {
+ if (nodeBlocks == null) {
+ return null;
+ }
+ const contents: JSONContent[] = [];
+ for (const nodeBlock of nodeBlocks) {
+ contents.push({
+ type: nodeBlock.type,
+ text: nodeBlock.text,
+ marks:
+ nodeBlock.marks?.map((mark) => {
+ return {
+ type: mark.type,
+ attrs: mark.attrs,
+ };
+ }) ?? null,
+ });
+ }
+ return contents;
+};
+
+const validateBlocksIndexes = (blocks: NodeBlock[]) => {
+ //group by parentId
+ const groupedBlocks: { [key: string]: NodeBlock[] } = {};
+ for (const block of blocks) {
+ if (!groupedBlocks[block.parentId]) {
+ groupedBlocks[block.parentId] = [];
+ }
+ groupedBlocks[block.parentId].push(block);
+ }
+ for (const parentId in groupedBlocks) {
+ const blocks = groupedBlocks[parentId];
+ for (let i = 0; i < blocks.length; i++) {
+ const currentIndex = blocks[i].index;
+ const beforeIndex = i === 0 ? null : blocks[i - 1].index;
+ // find the lowest index after the current node
+ // we do this because sometimes nodes can be ordered in such a way that
+ // the current node's index is higher than one of its siblings
+ // after the next sibling
+ // for example: 1, {current}, 4, 3
+ let afterIndex = i === blocks.length - 1 ? null : blocks[i + 1].index;
+ for (let j = i + 1; j < blocks.length; j++) {
+ if (blocks[j].index < afterIndex) {
+ afterIndex = blocks[j].index;
+ break;
+ }
+ }
+ // extra check to make sure that the beforeIndex is less than the afterIndex
+ // because otherwise the fractional index library will throw an error
+ if (afterIndex < beforeIndex) {
+ afterIndex = generateNodeIndex(null, beforeIndex);
+ } else if (beforeIndex === afterIndex) {
+ afterIndex = generateNodeIndex(beforeIndex, null);
+ }
+ if (
+ !currentIndex ||
+ currentIndex <= beforeIndex ||
+ currentIndex > afterIndex
+ ) {
+ blocks[i].index = generateNodeIndex(beforeIndex, afterIndex);
+ }
+ }
+ }
+};
+
+const getIdFromContent = (content: JSONContent): string => {
+ return content.attrs?.id ?? generateId(getIdTypeFromNode(content.type));
+};
+
+export const applyChangeToYDoc = (doc: Y.Doc, blocks: NodeBlock[]) => {
+ const attributesMap = doc.getMap('attributes');
+ if (!attributesMap.has('content')) {
+ attributesMap.set('content', new Y.Map());
+ }
+
+ const contentMap = attributesMap.get('content') as Y.Map<any>;
+ for (const block of blocks) {
+ if (!contentMap.has(block.id)) {
+ contentMap.set(block.id, new Y.Map());
+ }
+
+ const blockMap = contentMap.get(block.id) as Y.Map<any>;
+ applyBlockChangesToYDoc(blockMap, block);
+ }
+};
+
+const applyBlockChangesToYDoc = (blockMap: Y.Map<any>, block: NodeBlock) => {
+ if (blockMap.get('id') !== block.id) {
+ blockMap.set('id', block.id);
+ }
+
+ if (blockMap.get('type') !== block.type) {
+ blockMap.set('type', block.type);
+ }
+
+ if (blockMap.get('index') !== block.index) {
+ blockMap.set('index', block.index);
+ }
+
+ if (blockMap.get('parentId') !== block.parentId) {
+ blockMap.set('parentId', block.parentId);
+ }
+
+ applyBlockAttrsChangesToYDoc(blockMap, block);
+ applyBlockContentChangesToYDoc(blockMap, block);
+};
+
+const applyBlockAttrsChangesToYDoc = (
+ blockMap: Y.Map<any>,
+ block: NodeBlock,
+) => {
+ if (block.attrs === null) {
+ if (blockMap.has('attrs')) {
+ blockMap.delete('attrs');
+ }
+
+ return;
+ }
+
+ if (!blockMap.has('attrs')) {
+ blockMap.set('attrs', new Y.Map());
+ }
+
+ const attrsMap = blockMap.get('attrs') as Y.Map<any>;
+ for (const [key, value] of Object.entries(block.attrs)) {
+ const existingValue = attrsMap.get(key);
+ if (!isEqual(existingValue, value)) {
+ attrsMap.set(key, value);
+ }
+ }
+};
+
+const applyBlockContentChangesToYDoc = (
+ blockMap: Y.Map<any>,
+ block: NodeBlock,
+) => {
+ if (block.content === null || block.content.length === 0) {
+ if (blockMap.has('content')) {
+ blockMap.delete('content');
+ }
+
+ return;
+ }
+
+ if (!blockMap.has('content')) {
+ blockMap.set('content', new Y.Array());
+ }
+
+ const contentArray = blockMap.get('content') as Y.Array<any>;
+ for (let i = 0; i < block.content.length; i++) {
+ const blockContent = block.content[i];
+ if (contentArray.length > i) {
+ const blockContentMap = contentArray.get(i) as Y.Map<any>;
+ applyBlockContentItemChangesToYDoc(blockContentMap, blockContent);
+ } else {
+ const blockContentMap = new Y.Map<any>();
+ contentArray.insert(i, [blockContentMap]);
+ applyBlockContentItemChangesToYDoc(blockContentMap, blockContent);
+ }
+ }
+};
+
+const applyBlockContentItemChangesToYDoc = (
+ blockContentMap: Y.Map<any>,
+ blockContent: NodeBlockContent,
+) => {
+ if (blockContentMap.get('type') !== blockContent.type) {
+ blockContentMap.set('type', blockContent.type);
+ }
+
+ applyBlockContentTextChangesToYDoc(blockContentMap, blockContent);
+ applyBlockContentMarksChangesToYDoc(blockContentMap, blockContent);
+};
+
+const applyBlockContentTextChangesToYDoc = (
+ blockContentMap: Y.Map<any>,
+ blockContent: NodeBlockContent,
+) => {
+ if (
+ blockContent.text === null ||
+ blockContent.text === undefined ||
+ blockContent.text === ''
+ ) {
+ if (blockContentMap.has('text')) {
+ blockContentMap.delete('text');
+ }
+
+ return;
+ }
+
+ if (!blockContentMap.has('text')) {
+ blockContentMap.set('text', new Y.Text(blockContent.text));
+ return;
+ }
+
+ const yText = blockContentMap.get('text') as Y.Text;
+ const currentText = yText.toString();
+ const newText = blockContent.text;
+
+ if (currentText === newText) {
+ return;
+ }
+
+ const diffs = diffChars(currentText, newText);
+ let index = 0;
+ for (const diff of diffs) {
+ if (diff.added) {
+ yText.insert(index, diff.value);
+ index += diff.value.length;
+ } else if (diff.removed) {
+ yText.delete(index, diff.value.length);
+ index -= diff.value.length;
+ } else {
+ index += diff.value.length;
+ }
+ }
+};
+
+const applyBlockContentMarksChangesToYDoc = (
+ blockContentMap: Y.Map<any>,
+ blockContent: NodeBlockContent,
+) => {
+ if (blockContent.marks === null || blockContent.marks.length === 0) {
+ if (blockContentMap.has('marks')) {
+ blockContentMap.delete('marks');
+ }
+
+ return;
+ }
+
+ const existingMarks = blockContentMap.get('marks');
+ if (!isEqual(existingMarks, blockContent.marks)) {
+ blockContentMap.set('marks', blockContent.marks);
+ }
+};
diff --git a/desktop/src/main/handlers/mutations/document-save.ts b/desktop/src/main/handlers/mutations/document-save.ts
index cd49c3bd..5f54ddf6 100644
--- a/desktop/src/main/handlers/mutations/document-save.ts
+++ b/desktop/src/main/handlers/mutations/document-save.ts
@@ -1,11 +1,12 @@
import * as Y from 'yjs';
import { fromUint8Array, toUint8Array } from 'js-base64';
-import { mapNode } from '@/lib/nodes';
import { databaseManager } from '@/main/data/database-manager';
import { MutationHandler, MutationResult } from '@/operations/mutations';
import { DocumentSaveMutationInput } from '@/operations/mutations/document-save';
import { generateId, IdType } from '@/lib/id';
import { LocalUpdateNodeChangeData } from '@/types/sync';
+import { applyChangeToYDoc, mapContentsToBlocks } from '@/lib/editor';
+import { LocalNodeAttributes, NodeBlock } from '@/types/nodes';
export class DocumentSaveMutationHandler
implements MutationHandler<DocumentSaveMutationInput>
@@ -32,29 +33,53 @@ export class DocumentSaveMutationHandler
};
}
- const node = mapNode(document);
const versionId = generateId(IdType.Version);
const updatedAt = new Date().toISOString();
const updates: string[] = [];
const doc = new Y.Doc({
- guid: node.id,
+ guid: document.id,
});
- Y.applyUpdate(doc, toUint8Array(node.state));
+ Y.applyUpdate(doc, toUint8Array(document.state));
doc.on('update', (update) => {
updates.push(fromUint8Array(update));
});
- const attributesMap = doc.getMap('attributes');
- attributesMap.set('content', input.content.content);
+ const attributes = JSON.parse(document.attributes) as LocalNodeAttributes;
+ const blocksMap = new Map<string, NodeBlock>();
+ if (attributes.content) {
+ for (const [key, value] of Object.entries(attributes.content)) {
+ blocksMap.set(key, value);
+ }
+ }
+
+ const blocks = mapContentsToBlocks(
+ document.id,
+ input.content.content,
+ blocksMap,
+ );
+
+ doc.transact(() => {
+ applyChangeToYDoc(doc, blocks);
+ });
+
+ if (updates.length === 0) {
+ return {
+ output: {
+ success: true,
+ },
+ changes: [],
+ };
+ }
- const attributes = JSON.stringify(attributesMap.toJSON());
+ const attributesMap = doc.getMap('attributes');
+ const attributesJson = JSON.stringify(attributesMap.toJSON());
const encodedState = fromUint8Array(Y.encodeStateAsUpdate(doc));
const changeData: LocalUpdateNodeChangeData = {
type: 'node_update',
- id: node.id,
+ id: document.id,
updatedAt: updatedAt,
updatedBy: input.userId,
versionId: versionId,
@@ -65,7 +90,7 @@ export class DocumentSaveMutationHandler
await trx
.updateTable('nodes')
.set({
- attributes: attributes,
+ attributes: attributesJson,
state: encodedState,
updated_at: updatedAt,
updated_by: input.userId,
diff --git a/desktop/src/main/handlers/queries/document-get.ts b/desktop/src/main/handlers/queries/document-get.ts
index c45e50bd..eda7981b 100644
--- a/desktop/src/main/handlers/queries/document-get.ts
+++ b/desktop/src/main/handlers/queries/document-get.ts
@@ -12,8 +12,9 @@ import {
} from '@/operations/queries';
import { SelectNode } from '@/main/data/workspace/schema';
-import { mapNode } from '@/lib/nodes';
import { MutationChange } from '@/operations/mutations';
+import { LocalNodeAttributes } from '@/types/nodes';
+import { mapBlocksToContents } from '@/lib/editor';
export class DocumentGetQueryHandler
implements QueryHandler<DocumentGetQueryInput>
@@ -88,8 +89,10 @@ export class DocumentGetQueryHandler
};
}
- const node = mapNode(document);
- const contents = node.attributes?.content ?? [];
+ const attributes = JSON.parse(document.attributes) as LocalNodeAttributes;
+ const nodeBlocks = Object.values(attributes?.content ?? {});
+ const contents = mapBlocksToContents(document.id, nodeBlocks);
+
if (!contents.length) {
contents.push({
type: 'paragraph',
diff --git a/desktop/src/main/handlers/queries/message-list.ts b/desktop/src/main/handlers/queries/message-list.ts
index 7ab9a584..ac6f5d7a 100644
--- a/desktop/src/main/handlers/queries/message-list.ts
+++ b/desktop/src/main/handlers/queries/message-list.ts
@@ -13,6 +13,7 @@ import { mapNode } from '@/lib/nodes';
import { UserNode } from '@/types/users';
import { compareString } from '@/lib/utils';
import { isEqual } from 'lodash';
+import { mapBlocksToContents } from '@/lib/editor';
export class MessageListQueryHandler
implements QueryHandler<MessageListQueryInput>
@@ -159,10 +160,13 @@ export class MessageListQueryHandler
author: author ?? {
id: messageNode.createdBy,
name: 'Unknown User',
- email: '[email protected]',
+ email: '[email protected]',
avatar: null,
},
- content: messageNode.attributes.content,
+ content: mapBlocksToContents(
+ messageNode.id,
+ Object.values(messageNode.attributes.content ?? {}),
+ ),
reactionCounts,
};
diff --git a/desktop/src/types/editor.ts b/desktop/src/types/editor.ts
index 0fe7d497..c4d6e7e0 100644
--- a/desktop/src/types/editor.ts
+++ b/desktop/src/types/editor.ts
@@ -1,5 +1,4 @@
import { Editor, type Range } from '@tiptap/core';
-import { LocalNodeAttributes } from '@/types/nodes';
import { FC } from 'react';
export type EditorCommandProps = {
@@ -22,8 +21,3 @@ export type EditorCommand = {
handler: (props: EditorCommandProps) => void | Promise<void>;
disabled?: boolean;
};
-
-export type EditorNode = {
- id: string;
- attributes: LocalNodeAttributes;
-};
diff --git a/desktop/src/types/nodes.ts b/desktop/src/types/nodes.ts
index 9fa78ab8..e32a257c 100644
--- a/desktop/src/types/nodes.ts
+++ b/desktop/src/types/nodes.ts
@@ -1,5 +1,3 @@
-import { JSONContent } from '@tiptap/core';
-
export type LocalNode = {
id: string;
parentId: string | null;
@@ -21,19 +19,28 @@ export type LocalNodeAttributes = {
type: string;
parentId?: string | null;
index?: string | null;
- content?: JSONContent[] | null;
+ content?: Record<string, NodeBlock> | null;
[key: string]: any;
};
export type NodeBlock = {
+ id: string;
+ type: string;
+ index: string;
+ parentId: string;
+ content: NodeBlockContent[] | null;
+ attrs: Record<string, any> | null;
+};
+
+export type NodeBlockContent = {
type: string;
text?: string | null;
- marks?: NodeBlockMark[];
+ marks?: NodeBlocContentkMark[];
};
-export type NodeBlockMark = {
+export type NodeBlocContentkMark = {
type: string;
- attrs: any;
+ attrs: Record<string, any>;
};
export type LocalNodeWithChildren = LocalNode & {
@@ -60,7 +67,7 @@ export type ServerNodeAttributes = {
type: string;
parentId?: string | null;
index?: string | null;
- content?: NodeBlock[] | null;
+ content?: NodeBlockContent[] | null;
[key: string]: any;
};