Open-source and local-first Slack and Notion alternative that puts you in control of your data
TypeScript chatchat-applicationcrdteditorelectronknowledge-baselocal-firstnotionnotion-alternativerealtime-collaborationself-hostedslacksqliteteam-collaborationwikiyjs
Implement and improve document operations
Hakan Shehubd8b8518f394
diff --git a/desktop/src/renderer/editor/mappers.ts b/desktop/src/lib/editor.tssimilarity index 99%rename from desktop/src/renderer/editor/mappers.tsrename to desktop/src/lib/editor.tsindex 210ac554..f228a375 100644--- a/desktop/src/renderer/editor/mappers.ts+++ b/desktop/src/lib/editor.ts@@ -1,5 +1,5 @@import { EditorNodeTypes, NodeTypes } from '@/lib/constants';-import { generateId, getIdTypeFromNode, IdType } from '@/lib/id';+import { generateId, getIdTypeFromNode } from '@/lib/id';import { generateNodeIndex } from '@/lib/nodes';import { compareString } from '@/lib/utils';import { EditorNode } from '@/types/editor';diff --git a/desktop/src/renderer/editor/observer.ts b/desktop/src/main/handlers/mutations/document-save.tssimilarity index 56%rename from desktop/src/renderer/editor/observer.tsrename to desktop/src/main/handlers/mutations/document-save.tsindex fc43e890..55fd370c 100644--- a/desktop/src/renderer/editor/observer.ts+++ b/desktop/src/main/handlers/mutations/document-save.ts@@ -1,83 +1,60 @@-import { WorkspaceDatabaseSchema } from '@/main/data/workspace/schema';-import { generateId, IdType } from '@/lib/id';-import { LocalNode } from '@/types/nodes';-import { Workspace } from '@/types/workspaces';-import { Editor, JSONContent } from '@tiptap/core';-import { CompiledQuery, Kysely } from 'kysely';-import { debounce, isEqual } from 'lodash';-import {- mapContentsToEditorNodes,- mapNodesToContents,-} from '@/renderer/editor/mappers';-import { EditorNode } from '@/types/editor';import * as Y from 'yjs';+import { CompiledQuery, sql } from 'kysely';import { fromUint8Array, toUint8Array } from 'js-base64';--export class EditorObserver {- private readonly workspace: Workspace;- private readonly database: Kysely<WorkspaceDatabaseSchema>;- private readonly rootNodeId: string;- private readonly nodesMap: Map<string, LocalNode>;- private editorContent: JSONContent;-- private readonly onEditorUpdateDebounced: () => void;- private readonly onNodesChangeDebounced: (- newNodesMap: Map<string, LocalNode>,- editor: Editor,- ) => void;-- constructor(- workspace: Workspace,- database: Kysely<WorkspaceDatabaseSchema>,- rootNodeId: string,- nodesMap: Map<string, LocalNode>,- ) {- this.workspace = workspace;- this.database = database;- this.rootNodeId = rootNodeId;- this.nodesMap = nodesMap;- this.editorContent = this.buildEditorContent();- this.onEditorUpdateDebounced = debounce(- this.checkEditorContentChanges,- 500,+import { isEqual } from 'lodash';+import { NodeTypes } from '@/lib/constants';+import { mapNode } from '@/lib/nodes';+import { databaseManager } from '@/main/data/database-manager';+import { EditorNode } from '@/types/editor';+import { SelectNode } from '@/main/data/workspace/schema';+import {+ MutationChange,+ MutationHandler,+ MutationResult,+} from '@/operations/mutations';+import { DocumentSaveMutationInput } from '@/operations/mutations/document-save';+import { LocalNode } from '@/types/nodes';+import { generateId, IdType } from '@/lib/id';+import { mapContentsToEditorNodes } from '@/lib/editor';++export class DocumentSaveMutationHandler+ implements MutationHandler<DocumentSaveMutationInput>+{+ async handleMutation(+ input: DocumentSaveMutationInput,+ ): Promise<MutationResult<DocumentSaveMutationInput>> {+ const workspaceDatabase = await databaseManager.getWorkspaceDatabase(+ input.userId,);- this.onNodesChangeDebounced = debounce(this.checkNodesChanges, 500);- }- public getEditorContent(): JSONContent {- return this.editorContent;- }+ const query = sql<SelectNode>`+ WITH RECURSIVE document_nodes AS (+ SELECT *+ FROM nodes+ WHERE parent_id = ${input.documentId}++ UNION ALL++ SELECT child.*+ FROM nodes child+ INNER JOIN document_nodes parent ON child.parent_id = parent.id+ WHERE parent.type NOT IN (${NodeTypes.Page})+ )+ SELECT *+ FROM document_nodes+ `.compile(workspaceDatabase);++ const result = await workspaceDatabase.executeQuery(query);+ const nodes = result.rows;+ const map = new Map<string, LocalNode>();+ nodes.forEach((node) => {+ map.set(node.id, mapNode(node));+ });- public onEditorUpdate(editor: Editor) {- this.editorContent = editor.getJSON();- this.onEditorUpdateDebounced();- }-- public onNodesChange(nodes: Map<string, LocalNode>, editor: Editor) {- this.onNodesChangeDebounced(nodes, editor);- }-- private buildEditorContent(): JSONContent {- const nodesArray = Array.from(this.nodesMap.values());- const contents = mapNodesToContents(this.rootNodeId, nodesArray);-- if (!contents.length) {- contents.push({- type: 'paragraph',- });- }-- return {- type: 'doc',- content: contents,- };- }-- private async checkEditorContentChanges() {const editorNodes = mapContentsToEditorNodes(- this.editorContent.content,- this.rootNodeId,- this.nodesMap,+ input.content.content,+ input.documentId,+ map,);const editorNodesMap = new Map<string, EditorNode>();@@ -86,8 +63,8 @@ export class EditorObserver {}const allNodeIds = new Set([- ...this.nodesMap.keys(),- ...editorNodesMap.keys(),+ ...map.keys(),+ ...editorNodes.map((node) => node.id),]);const insertQueries: CompiledQuery[] = [];@@ -95,7 +72,7 @@ export class EditorObserver {const deleteQueries: CompiledQuery[] = [];for (const nodeId of allNodeIds) {- const existingNode = this.nodesMap.get(nodeId);+ const existingNode = map.get(nodeId);const editorNode = editorNodesMap.get(nodeId);if (!existingNode) {@@ -124,7 +101,7 @@ export class EditorObserver {attributes: editorNode.attributes,state: state,createdAt: new Date().toISOString(),- createdBy: this.workspace.userId,+ createdBy: input.userId,versionId: generateId(IdType.Version),updatedAt: null,updatedBy: null,@@ -133,28 +110,26 @@ export class EditorObserver {serverVersionId: null,};- const insertNodeQuery = this.database+ const insertNodeQuery = workspaceDatabase.insertInto('nodes').values({id: editorNode.id,attributes: JSON.stringify(attributes),state: state,created_at: new Date().toISOString(),- created_by: this.workspace.userId,+ created_by: input.userId,version_id: generateId(IdType.Version),}).compile();insertQueries.push(insertNodeQuery);- this.nodesMap.set(newNode.id, newNode);} else if (!editorNode) {- const query = this.database+ const query = workspaceDatabase.deleteFrom('nodes').where('id', '=', existingNode.id).compile();deleteQueries.push(query);- this.nodesMap.delete(existingNode.id);} else {if (!isEqual(existingNode.attributes, editorNode.attributes)) {const doc = new Y.Doc({@@ -224,10 +199,10 @@ export class EditorObserver {existingNode.index = editorNode.attributes.index;existingNode.attributes = editorNode.attributes;existingNode.updatedAt = new Date().toISOString();- existingNode.updatedBy = this.workspace.userId;+ existingNode.updatedBy = input.userId;existingNode.versionId = generateId(IdType.Version);- const query = this.database+ const query = workspaceDatabase.updateTable('nodes').set({attributes: JSON.stringify(attributes),@@ -244,66 +219,42 @@ export class EditorObserver {}}- if (+ const hasChanges =insertQueries.length > 0 ||updateQueries.length > 0 ||- deleteQueries.length > 0- ) {- const queries = [...insertQueries, ...updateQueries, ...deleteQueries];- // await window.neuron.executeWorkspaceMutation(- // this.workspace.accountId,- // this.workspace.id,- // queries,- // );- }- }+ deleteQueries.length > 0;- private checkNodesChanges(- newNodesMap: Map<string, LocalNode>,- editor: Editor,- ) {- const allNodeIds = new Set([- ...this.nodesMap.keys(),- ...newNodesMap.keys(),- ]);- let hasChanges = false;+ if (hasChanges) {+ await workspaceDatabase.transaction().execute(async (trx) => {+ for (const query of insertQueries) {+ await trx.executeQuery(query);+ }- for (const nodeId of allNodeIds) {- const oldNode = this.nodesMap.get(nodeId);- const newNode = newNodesMap.get(nodeId);-- if (!oldNode) {- this.nodesMap.set(newNode.id, newNode);- hasChanges = true;- } else if (!newNode) {- this.nodesMap.delete(oldNode.id);- hasChanges = true;- } else if (- !isEqual(oldNode, newNode) &&- oldNode.updatedAt !== null &&- newNode.updatedAt !== null- ) {- const oldUpdatedAt = new Date(oldNode.updatedAt);- const newUpdatedAt = new Date(newNode.updatedAt);- if (oldUpdatedAt < newUpdatedAt) {- this.nodesMap.set(newNode.id, newNode);- hasChanges = true;+ for (const query of updateQueries) {+ await trx.executeQuery(query);}- }- }- if (hasChanges) {- this.editorContent = this.buildEditorContent();- const selection = editor.state.selection;- if (selection.$anchor != null && selection.$head != null) {- editor- .chain()- .setContent(this.editorContent)- .setTextSelection(selection)- .run();- } else {- editor.chain().setContent(this.editorContent).run();- }+ for (const query of deleteQueries) {+ await trx.executeQuery(query);+ }+ });}++ const changes: MutationChange[] = hasChanges+ ? [+ {+ type: 'workspace',+ table: 'nodes',+ userId: input.userId,+ },+ ]+ : [];++ return {+ output: {+ success: true,+ },+ changes: changes,+ };}}diff --git a/desktop/src/main/handlers/mutations/index.ts b/desktop/src/main/handlers/mutations/index.tsindex 52c132a9..997c2675 100644--- a/desktop/src/main/handlers/mutations/index.ts+++ b/desktop/src/main/handlers/mutations/index.ts@@ -29,6 +29,7 @@ import { WorkspaceUpdateMutationHandler } from '@/main/handlers/mutations/workspimport { NodeSyncMutationHandler } from '@/main/handlers/mutations/node-sync';import { NodeReactionSyncMutationHandler } from '@/main/handlers/mutations/node-reaction-sync';import { NodeCollaboratorSyncMutationHandler } from '@/main/handlers/mutations/node-collaborator-sync';+import { DocumentSaveMutationHandler } from '@/main/handlers/mutations/document-save';type MutationHandlerMap = {[K in keyof MutationMap]: MutationHandler<MutationMap[K]['input']>;@@ -66,4 +67,5 @@ export const mutationHandlerMap: MutationHandlerMap = {node_sync: new NodeSyncMutationHandler(),node_reaction_sync: new NodeReactionSyncMutationHandler(),node_collaborator_sync: new NodeCollaboratorSyncMutationHandler(),+ document_save: new DocumentSaveMutationHandler(),};diff --git a/desktop/src/main/handlers/mutations/message-create.ts b/desktop/src/main/handlers/mutations/message-create.tsindex be07259b..5ffe6858 100644--- a/desktop/src/main/handlers/mutations/message-create.ts+++ b/desktop/src/main/handlers/mutations/message-create.ts@@ -1,4 +1,4 @@-import { mapContentsToEditorNodes } from '@/renderer/editor/mappers';+import { mapContentsToEditorNodes } from '@/lib/editor';import { databaseManager } from '@/main/data/database-manager';import { NodeTypes } from '@/lib/constants';import { generateId, IdType } from '@/lib/id';diff --git a/desktop/src/main/handlers/queries/document-get.ts b/desktop/src/main/handlers/queries/document-get.tsindex 234cb03a..61a2cec0 100644--- a/desktop/src/main/handlers/queries/document-get.ts+++ b/desktop/src/main/handlers/queries/document-get.ts@@ -1,17 +1,22 @@-import { DocumentGetQueryInput } from '@/operations/queries/document-get';+import { sql } from 'kysely';+import { sha256 } from 'js-sha256';+import { isEqual } from 'lodash';+import {+ DocumentGetQueryInput,+ DocumentGetQueryOutput,+} from '@/operations/queries/document-get';import { databaseManager } from '@/main/data/database-manager';import {ChangeCheckResult,QueryHandler,QueryResult,} from '@/operations/queries';-import { sql } from 'kysely';+import { SelectNode } from '@/main/data/workspace/schema';import { mapNode } from '@/lib/nodes';import { NodeTypes } from '@/lib/constants';-import { LocalNode } from '@/types/nodes';import { MutationChange } from '@/operations/mutations';-import { isEqual } from 'lodash';+import { mapNodesToContents } from '@/lib/editor';export class DocumentGetQueryHandlerimplements QueryHandler<DocumentGetQueryInput>@@ -21,9 +26,7 @@ export class DocumentGetQueryHandler): Promise<QueryResult<DocumentGetQueryInput>> {const rows = await this.fetchNodes(input);return {- output: {- nodes: this.buildMap(rows),- },+ output: this.buildOutput(input.documentId, rows),state: {rows,},@@ -58,9 +61,7 @@ export class DocumentGetQueryHandlerreturn {hasChanges: true,result: {- output: {- nodes: this.buildMap(rows),- },+ output: this.buildOutput(input.documentId, rows),state: {rows,},@@ -79,7 +80,7 @@ export class DocumentGetQueryHandlerWITH RECURSIVE document_nodes AS (SELECT *FROM nodes- WHERE parent_id = ${input.nodeId}+ WHERE parent_id = ${input.documentId}UNION ALL@@ -96,11 +97,28 @@ export class DocumentGetQueryHandlerreturn result.rows;}- private buildMap = (rows: SelectNode[]): Map<string, LocalNode> => {- const map = new Map<string, LocalNode>();- rows.forEach((row) => {- map.set(row.id, mapNode(row));- });- return map;- };+ private buildOutput(+ documentId: string,+ rows: SelectNode[],+ ): DocumentGetQueryOutput {+ const nodes = rows.map(mapNode);+ const contents = mapNodesToContents(documentId, nodes);+ if (!contents.length) {+ contents.push({+ type: 'paragraph',+ });+ }++ const content = {+ type: 'doc',+ content: contents,+ };++ const hash = sha256(JSON.stringify(content));++ return {+ content,+ hash,+ };+ }}diff --git a/desktop/src/operations/mutations/document-save.ts b/desktop/src/operations/mutations/document-save.tsnew file mode 100644index 00000000..f1056681--- /dev/null+++ b/desktop/src/operations/mutations/document-save.ts@@ -0,0 +1,21 @@+import { JSONContent } from '@tiptap/core';++export type DocumentSaveMutationInput = {+ type: 'document_save';+ userId: string;+ documentId: string;+ content: JSONContent;+};++export type DocumentSaveMutationOutput = {+ success: boolean;+};++declare module '@/operations/mutations' {+ interface MutationMap {+ document_save: {+ input: DocumentSaveMutationInput;+ output: DocumentSaveMutationOutput;+ };+ }+}diff --git a/desktop/src/operations/queries/document-get.ts b/desktop/src/operations/queries/document-get.tsindex 50ef00ab..34f052f0 100644--- a/desktop/src/operations/queries/document-get.ts+++ b/desktop/src/operations/queries/document-get.ts@@ -1,13 +1,14 @@-import { LocalNode } from '@/types/nodes';+import { JSONContent } from '@tiptap/core';export type DocumentGetQueryInput = {type: 'document_get';- nodeId: string;+ documentId: string;userId: string;};export type DocumentGetQueryOutput = {- nodes: Map<string, LocalNode>;+ content: JSONContent;+ hash: string;};declare module '@/operations/queries' {diff --git a/desktop/src/renderer/components/documents/document-editor.tsx b/desktop/src/renderer/components/documents/document-editor.tsxindex 1260e495..a6610fc8 100644--- a/desktop/src/renderer/components/documents/document-editor.tsx+++ b/desktop/src/renderer/components/documents/document-editor.tsx@@ -1,6 +1,7 @@import React from 'react';import '@/renderer/styles/editor.css';-import { useEditor, EditorContent } from '@tiptap/react';+import { debounce } from 'lodash';+import { useEditor, EditorContent, JSONContent } from '@tiptap/react';import {ParagraphCommand,BlockquoteCommand,@@ -47,22 +48,40 @@ import {DeleteControlExtension,DropcursorExtension,} from '@/renderer/editor/extensions';-import { EditorBubbleMenu } from '@/renderer/editor/menu/bubble-menu';-import { LocalNode } from '@/types/nodes';import { useWorkspace } from '@/renderer/contexts/workspace';-import { EditorObserver } from '@/renderer/editor/observer';+import { useMutation } from '@/renderer/hooks/use-mutation';interface DocumentEditorProps {- nodeId: string;- nodes: Map<string, LocalNode>;+ documentId: string;+ content: JSONContent;+ hash: string;}-export const DocumentEditor = ({ nodeId, nodes }: DocumentEditorProps) => {+export const DocumentEditor = ({+ documentId,+ content,+ hash,+}: DocumentEditorProps) => {const workspace = useWorkspace();- const observer = React.useMemo<EditorObserver>(- () => new EditorObserver(workspace, workspace as any, nodeId, nodes),- [nodeId],+ const { mutate } = useMutation();++ const hasPendingChanges = React.useRef(false);+ const hashRef = React.useRef(hash);+ const debouncedSave = React.useMemo(+ () =>+ debounce((content: JSONContent) => {+ hasPendingChanges.current = false;+ mutate({+ input: {+ type: 'document_save',+ documentId: documentId,+ userId: workspace.userId,+ content,+ },+ });+ }, 500),+ [mutate, documentId, workspace.userId],);const editor = useEditor(@@ -124,21 +143,37 @@ export const DocumentEditor = ({ nodeId, nodes }: DocumentEditorProps) => {spellCheck: 'false',},},- content: observer.getEditorContent(),+ content: content,shouldRerenderOnTransaction: false,autofocus: 'start',onUpdate: async ({ editor, transaction }) => {if (transaction.docChanged) {- observer.onEditorUpdate(editor);+ hasPendingChanges.current = true;+ debouncedSave(editor.getJSON());}},},- [nodeId],+ [documentId],);React.useEffect(() => {- observer.onNodesChange(nodes, editor);- }, [nodes]);+ if (hasPendingChanges.current) {+ return;+ }++ if (hashRef.current === hash) {+ return;+ }++ const selection = editor.state.selection;+ if (selection.$anchor != null && selection.$head != null) {+ editor.chain().setContent(content).setTextSelection(selection).run();+ } else {+ editor.chain().setContent(content).run();+ }++ hashRef.current = hash;+ }, [content, hash]);return (<div className="min-h-[500px]">diff --git a/desktop/src/renderer/components/documents/document.tsx b/desktop/src/renderer/components/documents/document.tsxindex f2a8b7f4..c4cc30d6 100644--- a/desktop/src/renderer/components/documents/document.tsx+++ b/desktop/src/renderer/components/documents/document.tsx@@ -11,13 +11,20 @@ export const Document = ({ nodeId }: DocumentProps) => {const workspace = useWorkspace();const { data, isPending } = useQuery({type: 'document_get',- nodeId,+ documentId: nodeId,userId: workspace.userId,});- if (isPending) {+ if (isPending || !data) {return null;}- return <DocumentEditor key={nodeId} nodeId={nodeId} nodes={data?.nodes} />;+ return (+ <DocumentEditor+ key={nodeId}+ documentId={nodeId}+ content={data.content}+ hash={data.hash}+ />+ );};