Open-source and local-first Slack and Notion alternative that puts you in control of your data
TypeScript chatchat-applicationcrdteditorelectronknowledge-baselocal-firstnotionnotion-alternativerealtime-collaborationself-hostedslacksqliteteam-collaborationwikiyjs
Store node state as byte array in Postgres
Hakan Shehue469885706c9
diff --git a/apps/server/src/data/migrations.ts b/apps/server/src/data/migrations.tsindex 38de4a5b..7d86ab47 100644--- a/apps/server/src/data/migrations.ts+++ b/apps/server/src/data/migrations.ts@@ -101,22 +101,20 @@ const createNodesTable: Migration = {.addColumn('id', 'varchar(30)', (col) => col.notNull().primaryKey()).addColumn('workspace_id', 'varchar(30)', (col) => col.notNull()).addColumn('type', 'varchar(30)', (col) =>- col.generatedAlwaysAs(sql`(attributes->>'type')::VARCHAR(30)`).stored(),+ col.generatedAlwaysAs(sql`(attributes->>'type')::VARCHAR(30)`).stored()).addColumn('parent_id', 'varchar(30)', (col) =>col.generatedAlwaysAs(sql`(attributes->>'parentId')::VARCHAR(30)`).stored().references('nodes.id')- .onDelete('cascade'),+ .onDelete('cascade')).addColumn('index', 'varchar(30)', (col) =>- col- .generatedAlwaysAs(sql`(attributes->>'index')::VARCHAR(30)`)- .stored(),+ col.generatedAlwaysAs(sql`(attributes->>'index')::VARCHAR(30)`).stored()).addColumn('attributes', 'jsonb', (col) => col.notNull())- .addColumn('state', 'text', (col) => col.notNull())+ .addColumn('state', 'bytea', (col) => col.notNull()).addColumn('created_at', 'timestamptz', (col) => col.notNull()).addColumn('updated_at', 'timestamptz').addColumn('created_by', 'varchar(30)', (col) => col.notNull())@@ -136,10 +134,10 @@ const createNodePathsTable: Migration = {await db.schema.createTable('node_paths').addColumn('ancestor_id', 'varchar(30)', (col) =>- col.notNull().references('nodes.id').onDelete('cascade'),+ col.notNull().references('nodes.id').onDelete('cascade')).addColumn('descendant_id', 'varchar(30)', (col) =>- col.notNull().references('nodes.id').onDelete('cascade'),+ col.notNull().references('nodes.id').onDelete('cascade')).addColumn('workspace_id', 'varchar(30)', (col) => col.notNull()).addColumn('level', 'integer', (col) => col.notNull())@@ -215,7 +213,7 @@ const createUserNodesTable: Migration = {.addColumn('last_seen_version_id', 'varchar(30)').addColumn('last_seen_at', 'timestamptz').addColumn('mentions_count', 'integer', (col) =>- col.notNull().defaultTo(0),+ col.notNull().defaultTo(0)).addColumn('attributes', 'jsonb').addColumn('created_at', 'timestamptz', (col) => col.notNull())diff --git a/apps/server/src/data/schema.ts b/apps/server/src/data/schema.tsindex 59e76d6a..7547ad91 100644--- a/apps/server/src/data/schema.ts+++ b/apps/server/src/data/schema.ts@@ -92,7 +92,7 @@ interface NodeTable {string | null,string | null>;- state: ColumnType<string, string, string>;+ state: ColumnType<Uint8Array, Uint8Array, Uint8Array>;created_at: ColumnType<Date, Date, never>;updated_at: ColumnType<Date | null, Date | null, Date>;created_by: ColumnType<string, string, never>;diff --git a/apps/server/src/lib/nodes.ts b/apps/server/src/lib/nodes.tsindex 53d9f136..919d40f7 100644--- a/apps/server/src/lib/nodes.ts+++ b/apps/server/src/lib/nodes.ts@@ -1,10 +1,7 @@import { database } from '@/data/database';import { SelectNode } from '@/data/schema';-import {- NodeCollaborator,- ServerNode,- ServerNodeAttributes,-} from '@/types/nodes';+import { NodeCollaborator, ServerNode } from '@/types/nodes';+import { fromUint8Array } from 'js-base64';export const mapNode = (node: SelectNode): ServerNode => {return {@@ -14,7 +11,7 @@ export const mapNode = (node: SelectNode): ServerNode => {type: node.type,index: node.index,attributes: node.attributes,- state: node.state,+ state: fromUint8Array(node.state),createdAt: node.created_at,createdBy: node.created_by,versionId: node.version_id,@@ -36,7 +33,7 @@ export const fetchNode = async (nodeId: string): Promise<SelectNode | null> => {};export const fetchNodeAncestors = async (- nodeId: string,+ nodeId: string): Promise<SelectNode[]> => {const result = await database.selectFrom('nodes')@@ -50,7 +47,7 @@ export const fetchNodeAncestors = async (};export const fetchNodeDescendants = async (- nodeId: string,+ nodeId: string): Promise<string[]> => {const result = await database.selectFrom('node_paths')@@ -63,7 +60,7 @@ export const fetchNodeDescendants = async (};export const fetchNodeCollaborators = async (- nodeId: string,+ nodeId: string): Promise<NodeCollaborator[]> => {const ancestors = await fetchNodeAncestors(nodeId);const collaboratorsMap = new Map<string, string>();@@ -83,13 +80,13 @@ export const fetchNodeCollaborators = async (nodeId: nodeId,collaboratorId: collaboratorId,role: role,- }),+ }));};export const fetchNodeRole = async (nodeId: string,- collaboratorId: string,+ collaboratorId: string): Promise<string | null> => {const ancestors = await fetchNodeAncestors(nodeId);if (ancestors.length === 0) {@@ -100,7 +97,7 @@ export const fetchNodeRole = async (};export const fetchWorkspaceUsers = async (- workspaceId: string,+ workspaceId: string): Promise<string[]> => {const result = await database.selectFrom('workspace_users')@@ -113,7 +110,7 @@ export const fetchWorkspaceUsers = async (export const extractNodeRole = (ancestors: SelectNode[],- collaboratorId: string,+ collaboratorId: string): string | null => {let role: string | null = null;for (const ancestor of ancestors) {diff --git a/apps/server/src/lib/workspaces.ts b/apps/server/src/lib/workspaces.tsindex 151632cb..99303823 100644--- a/apps/server/src/lib/workspaces.ts+++ b/apps/server/src/lib/workspaces.ts@@ -81,7 +81,7 @@ const buildUserNodeCreate = (});const attributes = JSON.stringify(attributesMap.toJSON());- const state = fromUint8Array(Y.encodeStateAsUpdate(doc));+ const state = Y.encodeStateAsUpdate(doc);return {id,@@ -118,7 +118,7 @@ const buildSpaceNodeCreate = (});const attributes = JSON.stringify(attributesMap.toJSON());- const state = fromUint8Array(Y.encodeStateAsUpdate(doc));+ const state = Y.encodeStateAsUpdate(doc);return {id,@@ -149,7 +149,7 @@ const buildPageNodeCreate = (});const attributes = JSON.stringify(attributesMap.toJSON());- const state = fromUint8Array(Y.encodeStateAsUpdate(doc));+ const state = Y.encodeStateAsUpdate(doc);return {id,@@ -180,7 +180,7 @@ const buildChannelNodeCreate = (});const attributes = JSON.stringify(attributesMap.toJSON());- const state = fromUint8Array(Y.encodeStateAsUpdate(doc));+ const state = Y.encodeStateAsUpdate(doc);return {id,diff --git a/apps/server/src/routes/accounts.ts b/apps/server/src/routes/accounts.tsindex 659376ca..1d9da9e0 100644--- a/apps/server/src/routes/accounts.ts+++ b/apps/server/src/routes/accounts.ts@@ -20,7 +20,6 @@ import { generateToken } from '@/lib/tokens';import { mapNode } from '@/lib/nodes';import { enqueueTask } from '@/queues/tasks';import * as Y from 'yjs';-import { fromUint8Array, toUint8Array } from 'js-base64';import { CompiledQuery } from 'kysely';import { ServerNodeAttributes } from '@/types/nodes';import { NodeUpdatedEvent } from '@/types/events';@@ -320,7 +319,7 @@ accountsRouter.put(}const doc = new Y.Doc({ guid: user.id });- Y.applyUpdate(doc, toUint8Array(user.state));+ Y.applyUpdate(doc, user.state);const attributesMap = doc.getMap('attributes');if (name != input.name) {@@ -333,7 +332,7 @@ accountsRouter.put(const attributes = attributesMap.toJSON() as ServerNodeAttributes;const attributesJson = JSON.stringify(attributes);- const encodedState = fromUint8Array(Y.encodeStateAsUpdate(doc));+ const state = Y.encodeStateAsUpdate(doc);const updatedAt = new Date();const versionId = generateId(IdType.Version);@@ -343,7 +342,7 @@ accountsRouter.put(.updateTable('nodes').set({attributes: attributesJson,- state: encodedState,+ state: state,updated_at: updatedAt,updated_by: user.id,version_id: versionId,diff --git a/apps/server/src/routes/sync.ts b/apps/server/src/routes/sync.tsindex 96cb62fc..d1e43398 100644--- a/apps/server/src/routes/sync.ts+++ b/apps/server/src/routes/sync.ts@@ -142,7 +142,7 @@ const handleCreateNodeChange = async (id: changeData.id,attributes: JSON.stringify(attributes),workspace_id: workspaceUser.workspace_id,- state: changeData.state,+ state: toUint8Array(changeData.state),created_at: new Date(changeData.createdAt),created_by: changeData.createdBy,version_id: changeData.versionId,@@ -190,7 +190,7 @@ const handleUpdateNodeChange = async (}const doc = new Y.Doc({ guid: changeData.id });- Y.applyUpdate(doc, toUint8Array(existingNode.state));+ Y.applyUpdate(doc, existingNode.state);for (const update of changeData.updates) {Y.applyUpdate(doc, toUint8Array(update));@@ -199,7 +199,7 @@ const handleUpdateNodeChange = async (const attributesMap = doc.getMap('attributes');const attributes = attributesMap.toJSON() as ServerNodeAttributes;const attributesJson = JSON.stringify(attributes);- const encodedState = fromUint8Array(Y.encodeStateAsUpdate(doc));+ const state = Y.encodeStateAsUpdate(doc);const validator = getValidator(existingNode.type);if (!validator) {@@ -224,7 +224,7 @@ const handleUpdateNodeChange = async (.updateTable('nodes').set({attributes: attributesJson,- state: encodedState,+ state: state,updated_at: new Date(changeData.updatedAt),updated_by: changeData.updatedBy,version_id: changeData.versionId,diff --git a/apps/server/src/routes/workspaces.ts b/apps/server/src/routes/workspaces.tsindex 0c748204..208f2032 100644--- a/apps/server/src/routes/workspaces.ts+++ b/apps/server/src/routes/workspaces.ts@@ -80,7 +80,7 @@ workspacesRouter.post(});const userAttributes = JSON.stringify(userAttributesMap.toJSON());- const userState = fromUint8Array(Y.encodeStateAsUpdate(userDoc));+ const userState = Y.encodeStateAsUpdate(userDoc);await database.transaction().execute(async (trx) => {await trx@@ -154,7 +154,7 @@ workspacesRouter.post(workspaceId: workspaceId,type: 'user',attributes: JSON.parse(userAttributes),- state: userState,+ state: fromUint8Array(userState),createdAt: new Date(),createdBy: account.id,versionId: userVersionId,@@ -629,7 +629,7 @@ workspacesRouter.post(});const userAttributes = JSON.stringify(userAttributesMap.toJSON());- const userState = fromUint8Array(Y.encodeStateAsUpdate(userDoc));+ const userState = Y.encodeStateAsUpdate(userDoc);workspaceUsersToCreate.push({id: userId,@@ -646,7 +646,7 @@ workspacesRouter.post(id: userId,type: 'user',attributes: JSON.parse(userAttributes),- state: userState,+ state: fromUint8Array(userState),createdAt: new Date(),createdBy: workspaceUser.id,serverCreatedAt: new Date(),@@ -791,7 +791,7 @@ workspacesRouter.put(}const userDoc = new Y.Doc({ guid: user.id });- Y.applyUpdate(userDoc, toUint8Array(user.state));+ Y.applyUpdate(userDoc, user.state);const userUpdates: string[] = [];userDoc.on('update', (update) => {@@ -802,7 +802,7 @@ workspacesRouter.put(userAttributesMap.set('role', input.role);const userAttributes = JSON.stringify(userAttributesMap.toJSON());- const encodedState = fromUint8Array(Y.encodeStateAsUpdate(userDoc));+ const state = Y.encodeStateAsUpdate(userDoc);const updatedAt = new Date();const userNode: ServerNode = {@@ -812,7 +812,7 @@ workspacesRouter.put(index: null,parentId: null,attributes: JSON.parse(userAttributes),- state: encodedState,+ state: fromUint8Array(state),createdAt: user.created_at,createdBy: user.created_by,serverCreatedAt: user.server_created_at,@@ -838,7 +838,7 @@ workspacesRouter.put(.updateTable('nodes').set({attributes: userAttributes,- state: encodedState,+ state: state,server_updated_at: updatedAt,updated_at: updatedAt,updated_by: currentWorkspaceUser.id,diff --git a/apps/server/src/services/synapse.ts b/apps/server/src/services/synapse.tsindex 39b48080..eb23ea6c 100644--- a/apps/server/src/services/synapse.ts+++ b/apps/server/src/services/synapse.ts@@ -11,6 +11,7 @@ import {} from '@/types/synapse';import { getIdType, IdType } from '@colanode/core';import { MessageInput } from '@/types/messages';+import { fromUint8Array } from 'js-base64';interface SynapseConnection {accountId: string;@@ -302,7 +303,7 @@ class SynapseService {type: 'server_node_sync',id: node.id,workspaceId: data.workspaceId,- state: node.state!,+ state: fromUint8Array(node.state),createdAt: node.created_at.toISOString(),createdBy: node.created_by,updatedAt: node.updated_at?.toISOString() ?? null,@@ -446,7 +447,7 @@ class SynapseService {type: 'server_node_sync',id: row.id,workspaceId: row.workspace_id,- state: row.state!,+ state: fromUint8Array(row.state!),createdAt: row.created_at!.toISOString(),createdBy: row.created_by!,updatedAt: row.updated_at?.toISOString() ?? null,diff --git a/apps/server/src/validators/space-validator.ts b/apps/server/src/validators/space-validator.tsindex a3694570..d6a2378f 100644--- a/apps/server/src/validators/space-validator.ts+++ b/apps/server/src/validators/space-validator.ts@@ -1,5 +1,5 @@import { SelectWorkspaceUser } from '@/data/schema';-import { hasAdminAccess, hasEditorAccess, NodeRoles } from '@/lib/constants';+import { hasAdminAccess, hasEditorAccess } from '@/lib/constants';import { ServerNode, ServerNodeAttributes } from '@/types/nodes';import { Validator } from '@/types/validators';import { WorkspaceRole } from '@/types/workspaces';@@ -7,7 +7,7 @@ import { WorkspaceRole } from '@/types/workspaces';export class SpaceValidator implements Validator {async canCreate(workspaceUser: SelectWorkspaceUser,- attributes: ServerNodeAttributes,+ attributes: ServerNodeAttributes): Promise<boolean> {if (workspaceUser.role === WorkspaceRole.Viewer) {return false;@@ -22,7 +22,7 @@ export class SpaceValidator implements Validator {async canUpdate(workspaceUser: SelectWorkspaceUser,node: ServerNode,- attributes: ServerNodeAttributes,+ attributes: ServerNodeAttributes): Promise<boolean> {const collaborators = attributes.collaborators ?? {};const role = collaborators[workspaceUser.id];@@ -35,7 +35,7 @@ export class SpaceValidator implements Validator {async canDelete(workspaceUser: SelectWorkspaceUser,- node: ServerNode,+ node: ServerNode): Promise<boolean> {const collaborators = node.attributes.collaborators ?? {};const role = collaborators[workspaceUser.id];