Skip to content

Open-source and local-first Slack and Notion alternative that puts you in control of your data

Implement and improve document operations

Hakan Shehubd8b8518f394
diff --git a/desktop/src/renderer/editor/mappers.ts b/desktop/src/lib/editor.ts
similarity index 99%
rename from desktop/src/renderer/editor/mappers.ts
rename to desktop/src/lib/editor.ts
index 210ac554..f228a375 100644
--- a/desktop/src/renderer/editor/mappers.ts
+++ b/desktop/src/lib/editor.ts
@@ -1,5 +1,5 @@
import { EditorNodeTypes, NodeTypes } from '@/lib/constants';
-import { generateId, getIdTypeFromNode, IdType } from '@/lib/id';
+import { generateId, getIdTypeFromNode } from '@/lib/id';
import { generateNodeIndex } from '@/lib/nodes';
import { compareString } from '@/lib/utils';
import { EditorNode } from '@/types/editor';
diff --git a/desktop/src/renderer/editor/observer.ts b/desktop/src/main/handlers/mutations/document-save.ts
similarity index 56%
rename from desktop/src/renderer/editor/observer.ts
rename to desktop/src/main/handlers/mutations/document-save.ts
index fc43e890..55fd370c 100644
--- a/desktop/src/renderer/editor/observer.ts
+++ b/desktop/src/main/handlers/mutations/document-save.ts
@@ -1,83 +1,60 @@
-import { WorkspaceDatabaseSchema } from '@/main/data/workspace/schema';
-import { generateId, IdType } from '@/lib/id';
-import { LocalNode } from '@/types/nodes';
-import { Workspace } from '@/types/workspaces';
-import { Editor, JSONContent } from '@tiptap/core';
-import { CompiledQuery, Kysely } from 'kysely';
-import { debounce, isEqual } from 'lodash';
-import {
- mapContentsToEditorNodes,
- mapNodesToContents,
-} from '@/renderer/editor/mappers';
-import { EditorNode } from '@/types/editor';
import * as Y from 'yjs';
+import { CompiledQuery, sql } from 'kysely';
import { fromUint8Array, toUint8Array } from 'js-base64';
-
-export class EditorObserver {
- private readonly workspace: Workspace;
- private readonly database: Kysely<WorkspaceDatabaseSchema>;
- private readonly rootNodeId: string;
- private readonly nodesMap: Map<string, LocalNode>;
- private editorContent: JSONContent;
-
- private readonly onEditorUpdateDebounced: () => void;
- private readonly onNodesChangeDebounced: (
- newNodesMap: Map<string, LocalNode>,
- editor: Editor,
- ) => void;
-
- constructor(
- workspace: Workspace,
- database: Kysely<WorkspaceDatabaseSchema>,
- rootNodeId: string,
- nodesMap: Map<string, LocalNode>,
- ) {
- this.workspace = workspace;
- this.database = database;
- this.rootNodeId = rootNodeId;
- this.nodesMap = nodesMap;
- this.editorContent = this.buildEditorContent();
- this.onEditorUpdateDebounced = debounce(
- this.checkEditorContentChanges,
- 500,
+import { isEqual } from 'lodash';
+import { NodeTypes } from '@/lib/constants';
+import { mapNode } from '@/lib/nodes';
+import { databaseManager } from '@/main/data/database-manager';
+import { EditorNode } from '@/types/editor';
+import { SelectNode } from '@/main/data/workspace/schema';
+import {
+ MutationChange,
+ MutationHandler,
+ MutationResult,
+} from '@/operations/mutations';
+import { DocumentSaveMutationInput } from '@/operations/mutations/document-save';
+import { LocalNode } from '@/types/nodes';
+import { generateId, IdType } from '@/lib/id';
+import { mapContentsToEditorNodes } from '@/lib/editor';
+
+export class DocumentSaveMutationHandler
+ implements MutationHandler<DocumentSaveMutationInput>
+{
+ async handleMutation(
+ input: DocumentSaveMutationInput,
+ ): Promise<MutationResult<DocumentSaveMutationInput>> {
+ const workspaceDatabase = await databaseManager.getWorkspaceDatabase(
+ input.userId,
);
- this.onNodesChangeDebounced = debounce(this.checkNodesChanges, 500);
- }
- public getEditorContent(): JSONContent {
- return this.editorContent;
- }
+ const query = sql<SelectNode>`
+ WITH RECURSIVE document_nodes AS (
+ SELECT *
+ FROM nodes
+ WHERE parent_id = ${input.documentId}
+
+ UNION ALL
+
+ SELECT child.*
+ FROM nodes child
+ INNER JOIN document_nodes parent ON child.parent_id = parent.id
+ WHERE parent.type NOT IN (${NodeTypes.Page})
+ )
+ SELECT *
+ FROM document_nodes
+ `.compile(workspaceDatabase);
+
+ const result = await workspaceDatabase.executeQuery(query);
+ const nodes = result.rows;
+ const map = new Map<string, LocalNode>();
+ nodes.forEach((node) => {
+ map.set(node.id, mapNode(node));
+ });
- public onEditorUpdate(editor: Editor) {
- this.editorContent = editor.getJSON();
- this.onEditorUpdateDebounced();
- }
-
- public onNodesChange(nodes: Map<string, LocalNode>, editor: Editor) {
- this.onNodesChangeDebounced(nodes, editor);
- }
-
- private buildEditorContent(): JSONContent {
- const nodesArray = Array.from(this.nodesMap.values());
- const contents = mapNodesToContents(this.rootNodeId, nodesArray);
-
- if (!contents.length) {
- contents.push({
- type: 'paragraph',
- });
- }
-
- return {
- type: 'doc',
- content: contents,
- };
- }
-
- private async checkEditorContentChanges() {
const editorNodes = mapContentsToEditorNodes(
- this.editorContent.content,
- this.rootNodeId,
- this.nodesMap,
+ input.content.content,
+ input.documentId,
+ map,
);
const editorNodesMap = new Map<string, EditorNode>();
@@ -86,8 +63,8 @@ export class EditorObserver {
}
const allNodeIds = new Set([
- ...this.nodesMap.keys(),
- ...editorNodesMap.keys(),
+ ...map.keys(),
+ ...editorNodes.map((node) => node.id),
]);
const insertQueries: CompiledQuery[] = [];
@@ -95,7 +72,7 @@ export class EditorObserver {
const deleteQueries: CompiledQuery[] = [];
for (const nodeId of allNodeIds) {
- const existingNode = this.nodesMap.get(nodeId);
+ const existingNode = map.get(nodeId);
const editorNode = editorNodesMap.get(nodeId);
if (!existingNode) {
@@ -124,7 +101,7 @@ export class EditorObserver {
attributes: editorNode.attributes,
state: state,
createdAt: new Date().toISOString(),
- createdBy: this.workspace.userId,
+ createdBy: input.userId,
versionId: generateId(IdType.Version),
updatedAt: null,
updatedBy: null,
@@ -133,28 +110,26 @@ export class EditorObserver {
serverVersionId: null,
};
- const insertNodeQuery = this.database
+ const insertNodeQuery = workspaceDatabase
.insertInto('nodes')
.values({
id: editorNode.id,
attributes: JSON.stringify(attributes),
state: state,
created_at: new Date().toISOString(),
- created_by: this.workspace.userId,
+ created_by: input.userId,
version_id: generateId(IdType.Version),
})
.compile();
insertQueries.push(insertNodeQuery);
- this.nodesMap.set(newNode.id, newNode);
} else if (!editorNode) {
- const query = this.database
+ const query = workspaceDatabase
.deleteFrom('nodes')
.where('id', '=', existingNode.id)
.compile();
deleteQueries.push(query);
- this.nodesMap.delete(existingNode.id);
} else {
if (!isEqual(existingNode.attributes, editorNode.attributes)) {
const doc = new Y.Doc({
@@ -224,10 +199,10 @@ export class EditorObserver {
existingNode.index = editorNode.attributes.index;
existingNode.attributes = editorNode.attributes;
existingNode.updatedAt = new Date().toISOString();
- existingNode.updatedBy = this.workspace.userId;
+ existingNode.updatedBy = input.userId;
existingNode.versionId = generateId(IdType.Version);
- const query = this.database
+ const query = workspaceDatabase
.updateTable('nodes')
.set({
attributes: JSON.stringify(attributes),
@@ -244,66 +219,42 @@ export class EditorObserver {
}
}
- if (
+ const hasChanges =
insertQueries.length > 0 ||
updateQueries.length > 0 ||
- deleteQueries.length > 0
- ) {
- const queries = [...insertQueries, ...updateQueries, ...deleteQueries];
- // await window.neuron.executeWorkspaceMutation(
- // this.workspace.accountId,
- // this.workspace.id,
- // queries,
- // );
- }
- }
+ deleteQueries.length > 0;
- private checkNodesChanges(
- newNodesMap: Map<string, LocalNode>,
- editor: Editor,
- ) {
- const allNodeIds = new Set([
- ...this.nodesMap.keys(),
- ...newNodesMap.keys(),
- ]);
- let hasChanges = false;
+ if (hasChanges) {
+ await workspaceDatabase.transaction().execute(async (trx) => {
+ for (const query of insertQueries) {
+ await trx.executeQuery(query);
+ }
- for (const nodeId of allNodeIds) {
- const oldNode = this.nodesMap.get(nodeId);
- const newNode = newNodesMap.get(nodeId);
-
- if (!oldNode) {
- this.nodesMap.set(newNode.id, newNode);
- hasChanges = true;
- } else if (!newNode) {
- this.nodesMap.delete(oldNode.id);
- hasChanges = true;
- } else if (
- !isEqual(oldNode, newNode) &&
- oldNode.updatedAt !== null &&
- newNode.updatedAt !== null
- ) {
- const oldUpdatedAt = new Date(oldNode.updatedAt);
- const newUpdatedAt = new Date(newNode.updatedAt);
- if (oldUpdatedAt < newUpdatedAt) {
- this.nodesMap.set(newNode.id, newNode);
- hasChanges = true;
+ for (const query of updateQueries) {
+ await trx.executeQuery(query);
}
- }
- }
- if (hasChanges) {
- this.editorContent = this.buildEditorContent();
- const selection = editor.state.selection;
- if (selection.$anchor != null && selection.$head != null) {
- editor
- .chain()
- .setContent(this.editorContent)
- .setTextSelection(selection)
- .run();
- } else {
- editor.chain().setContent(this.editorContent).run();
- }
+ for (const query of deleteQueries) {
+ await trx.executeQuery(query);
+ }
+ });
}
+
+ const changes: MutationChange[] = hasChanges
+ ? [
+ {
+ type: 'workspace',
+ table: 'nodes',
+ userId: input.userId,
+ },
+ ]
+ : [];
+
+ return {
+ output: {
+ success: true,
+ },
+ changes: changes,
+ };
}
}
diff --git a/desktop/src/main/handlers/mutations/index.ts b/desktop/src/main/handlers/mutations/index.ts
index 52c132a9..997c2675 100644
--- a/desktop/src/main/handlers/mutations/index.ts
+++ b/desktop/src/main/handlers/mutations/index.ts
@@ -29,6 +29,7 @@ import { WorkspaceUpdateMutationHandler } from '@/main/handlers/mutations/worksp
import { NodeSyncMutationHandler } from '@/main/handlers/mutations/node-sync';
import { NodeReactionSyncMutationHandler } from '@/main/handlers/mutations/node-reaction-sync';
import { NodeCollaboratorSyncMutationHandler } from '@/main/handlers/mutations/node-collaborator-sync';
+import { DocumentSaveMutationHandler } from '@/main/handlers/mutations/document-save';
type MutationHandlerMap = {
[K in keyof MutationMap]: MutationHandler<MutationMap[K]['input']>;
@@ -66,4 +67,5 @@ export const mutationHandlerMap: MutationHandlerMap = {
node_sync: new NodeSyncMutationHandler(),
node_reaction_sync: new NodeReactionSyncMutationHandler(),
node_collaborator_sync: new NodeCollaboratorSyncMutationHandler(),
+ document_save: new DocumentSaveMutationHandler(),
};
diff --git a/desktop/src/main/handlers/mutations/message-create.ts b/desktop/src/main/handlers/mutations/message-create.ts
index be07259b..5ffe6858 100644
--- a/desktop/src/main/handlers/mutations/message-create.ts
+++ b/desktop/src/main/handlers/mutations/message-create.ts
@@ -1,4 +1,4 @@
-import { mapContentsToEditorNodes } from '@/renderer/editor/mappers';
+import { mapContentsToEditorNodes } from '@/lib/editor';
import { databaseManager } from '@/main/data/database-manager';
import { NodeTypes } from '@/lib/constants';
import { generateId, IdType } from '@/lib/id';
diff --git a/desktop/src/main/handlers/queries/document-get.ts b/desktop/src/main/handlers/queries/document-get.ts
index 234cb03a..61a2cec0 100644
--- a/desktop/src/main/handlers/queries/document-get.ts
+++ b/desktop/src/main/handlers/queries/document-get.ts
@@ -1,17 +1,22 @@
-import { DocumentGetQueryInput } from '@/operations/queries/document-get';
+import { sql } from 'kysely';
+import { sha256 } from 'js-sha256';
+import { isEqual } from 'lodash';
+import {
+ DocumentGetQueryInput,
+ DocumentGetQueryOutput,
+} from '@/operations/queries/document-get';
import { databaseManager } from '@/main/data/database-manager';
import {
ChangeCheckResult,
QueryHandler,
QueryResult,
} from '@/operations/queries';
-import { sql } from 'kysely';
+
import { SelectNode } from '@/main/data/workspace/schema';
import { mapNode } from '@/lib/nodes';
import { NodeTypes } from '@/lib/constants';
-import { LocalNode } from '@/types/nodes';
import { MutationChange } from '@/operations/mutations';
-import { isEqual } from 'lodash';
+import { mapNodesToContents } from '@/lib/editor';
export class DocumentGetQueryHandler
implements QueryHandler<DocumentGetQueryInput>
@@ -21,9 +26,7 @@ export class DocumentGetQueryHandler
): Promise<QueryResult<DocumentGetQueryInput>> {
const rows = await this.fetchNodes(input);
return {
- output: {
- nodes: this.buildMap(rows),
- },
+ output: this.buildOutput(input.documentId, rows),
state: {
rows,
},
@@ -58,9 +61,7 @@ export class DocumentGetQueryHandler
return {
hasChanges: true,
result: {
- output: {
- nodes: this.buildMap(rows),
- },
+ output: this.buildOutput(input.documentId, rows),
state: {
rows,
},
@@ -79,7 +80,7 @@ export class DocumentGetQueryHandler
WITH RECURSIVE document_nodes AS (
SELECT *
FROM nodes
- WHERE parent_id = ${input.nodeId}
+ WHERE parent_id = ${input.documentId}
UNION ALL
@@ -96,11 +97,28 @@ export class DocumentGetQueryHandler
return result.rows;
}
- private buildMap = (rows: SelectNode[]): Map<string, LocalNode> => {
- const map = new Map<string, LocalNode>();
- rows.forEach((row) => {
- map.set(row.id, mapNode(row));
- });
- return map;
- };
+ private buildOutput(
+ documentId: string,
+ rows: SelectNode[],
+ ): DocumentGetQueryOutput {
+ const nodes = rows.map(mapNode);
+ const contents = mapNodesToContents(documentId, nodes);
+ if (!contents.length) {
+ contents.push({
+ type: 'paragraph',
+ });
+ }
+
+ const content = {
+ type: 'doc',
+ content: contents,
+ };
+
+ const hash = sha256(JSON.stringify(content));
+
+ return {
+ content,
+ hash,
+ };
+ }
}
diff --git a/desktop/src/operations/mutations/document-save.ts b/desktop/src/operations/mutations/document-save.ts
new file mode 100644
index 00000000..f1056681
--- /dev/null
+++ b/desktop/src/operations/mutations/document-save.ts
@@ -0,0 +1,21 @@
+import { JSONContent } from '@tiptap/core';
+
+export type DocumentSaveMutationInput = {
+ type: 'document_save';
+ userId: string;
+ documentId: string;
+ content: JSONContent;
+};
+
+export type DocumentSaveMutationOutput = {
+ success: boolean;
+};
+
+declare module '@/operations/mutations' {
+ interface MutationMap {
+ document_save: {
+ input: DocumentSaveMutationInput;
+ output: DocumentSaveMutationOutput;
+ };
+ }
+}
diff --git a/desktop/src/operations/queries/document-get.ts b/desktop/src/operations/queries/document-get.ts
index 50ef00ab..34f052f0 100644
--- a/desktop/src/operations/queries/document-get.ts
+++ b/desktop/src/operations/queries/document-get.ts
@@ -1,13 +1,14 @@
-import { LocalNode } from '@/types/nodes';
+import { JSONContent } from '@tiptap/core';
export type DocumentGetQueryInput = {
type: 'document_get';
- nodeId: string;
+ documentId: string;
userId: string;
};
export type DocumentGetQueryOutput = {
- nodes: Map<string, LocalNode>;
+ content: JSONContent;
+ hash: string;
};
declare module '@/operations/queries' {
diff --git a/desktop/src/renderer/components/documents/document-editor.tsx b/desktop/src/renderer/components/documents/document-editor.tsx
index 1260e495..a6610fc8 100644
--- a/desktop/src/renderer/components/documents/document-editor.tsx
+++ b/desktop/src/renderer/components/documents/document-editor.tsx
@@ -1,6 +1,7 @@
import React from 'react';
import '@/renderer/styles/editor.css';
-import { useEditor, EditorContent } from '@tiptap/react';
+import { debounce } from 'lodash';
+import { useEditor, EditorContent, JSONContent } from '@tiptap/react';
import {
ParagraphCommand,
BlockquoteCommand,
@@ -47,22 +48,40 @@ import {
DeleteControlExtension,
DropcursorExtension,
} from '@/renderer/editor/extensions';
-
import { EditorBubbleMenu } from '@/renderer/editor/menu/bubble-menu';
-import { LocalNode } from '@/types/nodes';
import { useWorkspace } from '@/renderer/contexts/workspace';
-import { EditorObserver } from '@/renderer/editor/observer';
+import { useMutation } from '@/renderer/hooks/use-mutation';
interface DocumentEditorProps {
- nodeId: string;
- nodes: Map<string, LocalNode>;
+ documentId: string;
+ content: JSONContent;
+ hash: string;
}
-export const DocumentEditor = ({ nodeId, nodes }: DocumentEditorProps) => {
+export const DocumentEditor = ({
+ documentId,
+ content,
+ hash,
+}: DocumentEditorProps) => {
const workspace = useWorkspace();
- const observer = React.useMemo<EditorObserver>(
- () => new EditorObserver(workspace, workspace as any, nodeId, nodes),
- [nodeId],
+ const { mutate } = useMutation();
+
+ const hasPendingChanges = React.useRef(false);
+ const hashRef = React.useRef(hash);
+ const debouncedSave = React.useMemo(
+ () =>
+ debounce((content: JSONContent) => {
+ hasPendingChanges.current = false;
+ mutate({
+ input: {
+ type: 'document_save',
+ documentId: documentId,
+ userId: workspace.userId,
+ content,
+ },
+ });
+ }, 500),
+ [mutate, documentId, workspace.userId],
);
const editor = useEditor(
@@ -124,21 +143,37 @@ export const DocumentEditor = ({ nodeId, nodes }: DocumentEditorProps) => {
spellCheck: 'false',
},
},
- content: observer.getEditorContent(),
+ content: content,
shouldRerenderOnTransaction: false,
autofocus: 'start',
onUpdate: async ({ editor, transaction }) => {
if (transaction.docChanged) {
- observer.onEditorUpdate(editor);
+ hasPendingChanges.current = true;
+ debouncedSave(editor.getJSON());
}
},
},
- [nodeId],
+ [documentId],
);
React.useEffect(() => {
- observer.onNodesChange(nodes, editor);
- }, [nodes]);
+ if (hasPendingChanges.current) {
+ return;
+ }
+
+ if (hashRef.current === hash) {
+ return;
+ }
+
+ const selection = editor.state.selection;
+ if (selection.$anchor != null && selection.$head != null) {
+ editor.chain().setContent(content).setTextSelection(selection).run();
+ } else {
+ editor.chain().setContent(content).run();
+ }
+
+ hashRef.current = hash;
+ }, [content, hash]);
return (
<div className="min-h-[500px]">
diff --git a/desktop/src/renderer/components/documents/document.tsx b/desktop/src/renderer/components/documents/document.tsx
index f2a8b7f4..c4cc30d6 100644
--- a/desktop/src/renderer/components/documents/document.tsx
+++ b/desktop/src/renderer/components/documents/document.tsx
@@ -11,13 +11,20 @@ export const Document = ({ nodeId }: DocumentProps) => {
const workspace = useWorkspace();
const { data, isPending } = useQuery({
type: 'document_get',
- nodeId,
+ documentId: nodeId,
userId: workspace.userId,
});
- if (isPending) {
+ if (isPending || !data) {
return null;
}
- return <DocumentEditor key={nodeId} nodeId={nodeId} nodes={data?.nodes} />;
+ return (
+ <DocumentEditor
+ key={nodeId}
+ documentId={nodeId}
+ content={data.content}
+ hash={data.hash}
+ />
+ );
};