Open-source and local-first Slack and Notion alternative that puts you in control of your data
TypeScript chatchat-applicationcrdteditorelectronknowledge-baselocal-firstnotionnotion-alternativerealtime-collaborationself-hostedslacksqliteteam-collaborationwikiyjs
Introduce redis and websockets for real time update changes
Hakan Shehu54c374458384
diff --git a/desktop/src/electron/database/global/migrations.ts b/desktop/src/data/databases/global/migrations.tssimilarity index 96%rename from desktop/src/electron/database/global/migrations.tsrename to desktop/src/data/databases/global/migrations.tsindex 44fed7cd..cbf5b6cf 100644--- a/desktop/src/electron/database/global/migrations.ts+++ b/desktop/src/data/databases/global/migrations.ts@@ -10,6 +10,8 @@ const createAccountsTable: Migration = {.addColumn('email', 'text', (col) => col.notNull()).addColumn('avatar', 'text').addColumn('token', 'text', (col) => col.notNull())+ .addColumn('last_update_id', 'text')+ .addColumn('last_update_at', 'text').execute();},down: async (db) => {diff --git a/desktop/src/electron/database/workspace/migrations.ts b/desktop/src/data/databases/workspace/migrations.tssimilarity index 92%rename from desktop/src/electron/database/workspace/migrations.tsrename to desktop/src/data/databases/workspace/migrations.tsindex 93f2e88e..387478a1 100644--- a/desktop/src/electron/database/workspace/migrations.ts+++ b/desktop/src/data/databases/workspace/migrations.ts@@ -18,6 +18,8 @@ const createNodesTable: Migration = {.addColumn('created_by', 'text', (col) => col.notNull()).addColumn('updated_by', 'text').addColumn('version_id', 'text', (col) => col.notNull())+ .addColumn('server_created_at', 'text')+ .addColumn('server_updated_at', 'text').addColumn('state', 'text').execute();},diff --git a/server/package-lock.json b/server/package-lock.jsonindex 3c35bb55..cf4219f3 100644--- a/server/package-lock.json+++ b/server/package-lock.json@@ -18,13 +18,16 @@"jsonwebtoken": "^9.0.2","kafkajs": "^2.2.4","postgres": "^3.4.4",- "ulid": "^2.3.0"+ "redis": "^4.7.0",+ "ulid": "^2.3.0",+ "ws": "^8.18.0"},"devDependencies": {"@types/bcrypt": "^5.0.2","@types/express": "^4.17.21","@types/jsonwebtoken": "^9.0.6","@types/node": "^22.0.0",+ "@types/ws": "^8.5.12","concurrently": "^8.2.2","nodemon": "^3.1.4","prettier": "^3.3.3",@@ -176,6 +179,65 @@"@prisma/debug": "5.17.0"}},+ "node_modules/@redis/bloom": {+ "version": "1.2.0",+ "resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-1.2.0.tgz",+ "integrity": "sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==",+ "license": "MIT",+ "peerDependencies": {+ "@redis/client": "^1.0.0"+ }+ },+ "node_modules/@redis/client": {+ "version": "1.6.0",+ "resolved": "https://registry.npmjs.org/@redis/client/-/client-1.6.0.tgz",+ "integrity": "sha512-aR0uffYI700OEEH4gYnitAnv3vzVGXCFvYfdpu/CJKvk4pHfLPEy/JSZyrpQ+15WhXe1yJRXLtfQ84s4mEXnPg==",+ "license": "MIT",+ "dependencies": {+ "cluster-key-slot": "1.1.2",+ "generic-pool": "3.9.0",+ "yallist": "4.0.0"+ },+ "engines": {+ "node": ">=14"+ }+ },+ "node_modules/@redis/graph": {+ "version": "1.1.1",+ "resolved": "https://registry.npmjs.org/@redis/graph/-/graph-1.1.1.tgz",+ "integrity": "sha512-FEMTcTHZozZciLRl6GiiIB4zGm5z5F3F6a6FZCyrfxdKOhFlGkiAqlexWMBzCi4DcRoyiOsuLfW+cjlGWyExOw==",+ "license": "MIT",+ "peerDependencies": {+ "@redis/client": "^1.0.0"+ }+ },+ "node_modules/@redis/json": {+ "version": "1.0.7",+ "resolved": "https://registry.npmjs.org/@redis/json/-/json-1.0.7.tgz",+ "integrity": "sha512-6UyXfjVaTBTJtKNG4/9Z8PSpKE6XgSyEb8iwaqDcy+uKrd/DGYHTWkUdnQDyzm727V7p21WUMhsqz5oy65kPcQ==",+ "license": "MIT",+ "peerDependencies": {+ "@redis/client": "^1.0.0"+ }+ },+ "node_modules/@redis/search": {+ "version": "1.2.0",+ "resolved": "https://registry.npmjs.org/@redis/search/-/search-1.2.0.tgz",+ "integrity": "sha512-tYoDBbtqOVigEDMAcTGsRlMycIIjwMCgD8eR2t0NANeQmgK/lvxNAvYyb6bZDD4frHRhIHkJu2TBRvB0ERkOmw==",+ "license": "MIT",+ "peerDependencies": {+ "@redis/client": "^1.0.0"+ }+ },+ "node_modules/@redis/time-series": {+ "version": "1.1.0",+ "resolved": "https://registry.npmjs.org/@redis/time-series/-/time-series-1.1.0.tgz",+ "integrity": "sha512-c1Q99M5ljsIuc4YdaCwfUEXsofakb9c8+Zse2qxTadu8TalLXuAESzLvFAvNVbkmSlvlzIQOLpBCmWI9wTOt+g==",+ "license": "MIT",+ "peerDependencies": {+ "@redis/client": "^1.0.0"+ }+ },"node_modules/@tsconfig/node10": {"version": "1.0.11","resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.11.tgz",@@ -340,6 +402,16 @@"@types/send": "*"}},+ "node_modules/@types/ws": {+ "version": "8.5.12",+ "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.5.12.tgz",+ "integrity": "sha512-3tPRkv1EtkDpzlgyKyI8pGsGZAGPEaXeu0DOj5DI25Ja91bdAYddYHbADRYVrZMRbfW+1l5YwXVDKohDJNQxkQ==",+ "dev": true,+ "license": "MIT",+ "dependencies": {+ "@types/node": "*"+ }+ },"node_modules/abbrev": {"version": "1.1.1","resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz",@@ -712,6 +784,15 @@"node": ">=12"}},+ "node_modules/cluster-key-slot": {+ "version": "1.1.2",+ "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz",+ "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==",+ "license": "Apache-2.0",+ "engines": {+ "node": ">=0.10.0"+ }+ },"node_modules/color-convert": {"version": "2.0.1","resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",@@ -1247,6 +1328,15 @@"node": ">=10"}},+ "node_modules/generic-pool": {+ "version": "3.9.0",+ "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.9.0.tgz",+ "integrity": "sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==",+ "license": "MIT",+ "engines": {+ "node": ">= 4"+ }+ },"node_modules/get-caller-file": {"version": "2.0.5","resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz",@@ -2164,6 +2254,23 @@"node": ">=8.10.0"}},+ "node_modules/redis": {+ "version": "4.7.0",+ "resolved": "https://registry.npmjs.org/redis/-/redis-4.7.0.tgz",+ "integrity": "sha512-zvmkHEAdGMn+hMRXuMBtu4Vo5P6rHQjLoHftu+lBqq8ZTA3RCVC/WzD790bkKKiNFp7d5/9PcSD19fJyyRvOdQ==",+ "license": "MIT",+ "workspaces": [+ "./packages/*"+ ],+ "dependencies": {+ "@redis/bloom": "1.2.0",+ "@redis/client": "1.6.0",+ "@redis/graph": "1.1.1",+ "@redis/json": "1.0.7",+ "@redis/search": "1.2.0",+ "@redis/time-series": "1.1.0"+ }+ },"node_modules/regenerator-runtime": {"version": "0.14.1","resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.14.1.tgz",@@ -2708,6 +2815,27 @@"integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==","license": "ISC"},+ "node_modules/ws": {+ "version": "8.18.0",+ "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.0.tgz",+ "integrity": "sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==",+ "license": "MIT",+ "engines": {+ "node": ">=10.0.0"+ },+ "peerDependencies": {+ "bufferutil": "^4.0.1",+ "utf-8-validate": ">=5.0.2"+ },+ "peerDependenciesMeta": {+ "bufferutil": {+ "optional": true+ },+ "utf-8-validate": {+ "optional": true+ }+ }+ },"node_modules/y18n": {"version": "5.0.8","resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz",diff --git a/server/package.json b/server/package.jsonindex 08ce725f..a20e7444 100644--- a/server/package.json+++ b/server/package.json@@ -17,6 +17,7 @@"@types/express": "^4.17.21","@types/jsonwebtoken": "^9.0.6","@types/node": "^22.0.0",+ "@types/ws": "^8.5.12","concurrently": "^8.2.2","nodemon": "^3.1.4","prettier": "^3.3.3",@@ -35,6 +36,8 @@"jsonwebtoken": "^9.0.2","kafkajs": "^2.2.4","postgres": "^3.4.4",- "ulid": "^2.3.0"+ "redis": "^4.7.0",+ "ulid": "^2.3.0",+ "ws": "^8.18.0"}}diff --git a/server/prisma/migrations/20240805122702_add_account_devices/migration.sql b/server/prisma/migrations/20240805122702_add_account_devices/migration.sqldeleted file mode 100644index 391b6929..00000000--- a/server/prisma/migrations/20240805122702_add_account_devices/migration.sql+++ /dev/null@@ -1,15 +0,0 @@--- CreateTable-CREATE TABLE "account_devices" (- "id" VARCHAR(30) NOT NULL,- "account_id" VARCHAR(30) NOT NULL,- "type" INTEGER NOT NULL,- "version" VARCHAR(30) NOT NULL,- "platform" VARCHAR(30),- "cpu" VARCHAR(30),- "hostname" VARCHAR(30),- "created_at" TIMESTAMPTZ(6) NOT NULL,- "last_online_at" TIMESTAMPTZ(6),- "last_active_at" TIMESTAMPTZ(6),-- CONSTRAINT "account_devices_pkey" PRIMARY KEY ("id")-);diff --git a/server/prisma/migrations/20240805101219_init_db/migration.sql b/server/prisma/migrations/20240808134858_init_db/migration.sqlsimilarity index 71%rename from server/prisma/migrations/20240805101219_init_db/migration.sqlrename to server/prisma/migrations/20240808134858_init_db/migration.sqlindex f1c4177e..fd217f17 100644--- a/server/prisma/migrations/20240805101219_init_db/migration.sql+++ b/server/prisma/migrations/20240808134858_init_db/migration.sql@@ -61,11 +61,41 @@ CREATE TABLE "nodes" ("updated_at" TIMESTAMPTZ(6),"updated_by" VARCHAR(30),"version_id" VARCHAR(30) NOT NULL,+ "server_created_at" TIMESTAMPTZ(6) NOT NULL,+ "server_updated_at" TIMESTAMPTZ(6),"state" TEXT,CONSTRAINT "nodes_pkey" PRIMARY KEY ("id"));+-- CreateTable+CREATE TABLE "account_devices" (+ "id" VARCHAR(30) NOT NULL,+ "account_id" VARCHAR(30) NOT NULL,+ "type" INTEGER NOT NULL,+ "version" VARCHAR(30) NOT NULL,+ "platform" VARCHAR(30),+ "cpu" VARCHAR(30),+ "hostname" VARCHAR(30),+ "created_at" TIMESTAMPTZ(6) NOT NULL,+ "last_online_at" TIMESTAMPTZ(6),+ "last_active_at" TIMESTAMPTZ(6),++ CONSTRAINT "account_devices_pkey" PRIMARY KEY ("id")+);++-- CreateTable+CREATE TABLE "updates" (+ "id" VARCHAR(30) NOT NULL,+ "workspace_id" VARCHAR(30) NOT NULL,+ "type" VARCHAR(30) NOT NULL,+ "content" JSONB,+ "created_at" TIMESTAMPTZ(6) NOT NULL,+ "devices" TEXT[],++ CONSTRAINT "updates_pkey" PRIMARY KEY ("id")+);+-- CreateIndexCREATE UNIQUE INDEX "IX_accounts_email" ON "accounts"("email");diff --git a/server/prisma/schema.prisma b/server/prisma/schema.prismaindex fb6d8f86..6734c09f 100644--- a/server/prisma/schema.prisma+++ b/server/prisma/schema.prisma@@ -55,19 +55,21 @@ model workspaceAccounts {}model nodes {- id String @id @db.VarChar(30)- workspaceId String @db.VarChar(30) @map("workspace_id")- parentId String? @db.VarChar(30) @map("parent_id")- type String @db.VarChar(30)- index String? @db.VarChar(30)- attrs Json?- content Json?- createdAt DateTime @db.Timestamptz(6) @map("created_at")- createdBy String @db.VarChar(30) @map("created_by")- updatedAt DateTime? @db.Timestamptz(6) @map("updated_at")- updatedBy String? @db.VarChar(30) @map("updated_by")- versionId String @db.VarChar(30) @map("version_id")- state String? @db.Text()+ id String @id @db.VarChar(30)+ workspaceId String @db.VarChar(30) @map("workspace_id")+ parentId String? @db.VarChar(30) @map("parent_id")+ type String @db.VarChar(30)+ index String? @db.VarChar(30)+ attrs Json?+ content Json?+ createdAt DateTime @db.Timestamptz(6) @map("created_at")+ createdBy String @db.VarChar(30) @map("created_by")+ updatedAt DateTime? @db.Timestamptz(6) @map("updated_at")+ updatedBy String? @db.VarChar(30) @map("updated_by")+ versionId String @db.VarChar(30) @map("version_id")+ serverCreatedAt DateTime @db.Timestamptz(6) @map("server_created_at")+ serverUpdatedAt DateTime? @db.Timestamptz(6) @map("server_updated_at")+ state String? @db.Text()parent nodes? @relation("node_parent_child", fields: [parentId], references: [id], onDelete: Cascade)children nodes[] @relation("node_parent_child")@@ -77,7 +79,7 @@ model nodes {model accountDevices {id String @id @db.VarChar(30)- account_id String @db.VarChar(30)+ accountId String @db.VarChar(30) @map("account_id")type Intversion String @db.VarChar(30) @map("version")platform String? @db.VarChar(30)@@ -89,3 +91,14 @@ model accountDevices {@@map("account_devices")}++model updates {+ id String @id @db.VarChar(30)+ workspaceId String @db.VarChar(30) @map("workspace_id")+ type String @db.VarChar(30)+ content Json?+ createdAt DateTime @db.Timestamptz(6) @map("created_at")+ devices String[]++ @@map("updates")+}diff --git a/server/src/consumers/node-changes.ts b/server/src/consumers/node-changes.tsnew file mode 100644index 00000000..0cb7e93c--- /dev/null+++ b/server/src/consumers/node-changes.ts@@ -0,0 +1,90 @@+import { kafka, TOPIC_NAMES, CONSUMER_IDS } from '@/data/kafka';+import { ChangeMessage, NodeChangeData } from '@/types/changes';+import { prisma } from '@/data/prisma';+import { NeuronId } from '@/lib/id';+import { Node } from '@/types/nodes';++export const initNodeChangesConsumer = async () => {+ const consumer = kafka.consumer({ groupId: CONSUMER_IDS.NODE_CHANGES });++ await consumer.connect();+ await consumer.subscribe({ topic: TOPIC_NAMES.NODE_CHANGES });++ await consumer.run({+ eachMessage: async ({ message }) => {+ if (!message || !message.value) {+ return;+ }++ const change = JSON.parse(+ message.value.toString(),+ ) as ChangeMessage<NodeChangeData>;++ await handleNodeChange(change);+ },+ });+};++const handleNodeChange = async (change: ChangeMessage<NodeChangeData>) => {+ const changeData = change.after;+ if (!changeData) {+ return;+ }++ const workspaceAccounts = await prisma.workspaceAccounts.findMany({+ where: {+ workspaceId: changeData.workspace_id,+ },+ });++ if (workspaceAccounts.length === 0) {+ return;+ }++ const accountIds = workspaceAccounts.map((account) => account.accountId);+ const accountDevices = await prisma.accountDevices.findMany({+ where: {+ accountId: {+ in: accountIds,+ },+ },+ });++ if (accountDevices.length === 0) {+ return;+ }++ const deviceIds = accountDevices.map((device) => device.id);+ const node: Node = {+ id: changeData.id,+ workspaceId: changeData.workspace_id,+ parentId: changeData.parent_id,+ type: changeData.type,+ index: changeData.index,+ attrs: changeData.attrs ? JSON.parse(changeData.attrs) : null,+ content: changeData.content ? JSON.parse(changeData.content) : null,+ createdAt: new Date(changeData.created_at),+ createdBy: changeData.created_by,+ updatedAt: changeData.updated_at ? new Date(changeData.updated_at) : null,+ updatedBy: changeData.updated_by,+ versionId: changeData.version_id,+ serverCreatedAt: new Date(changeData.server_created_at),+ serverUpdatedAt: changeData.server_updated_at+ ? new Date(changeData.server_updated_at)+ : null,+ state: changeData.state,+ };++ await prisma.updates.create({+ data: {+ id: NeuronId.generate(NeuronId.Type.Update),+ workspaceId: changeData.workspace_id,+ devices: deviceIds,+ type: 'node_sync',+ content: JSON.stringify(node),+ createdAt: new Date(),+ },+ });+};++const handelNodeDelete = async (id: string) => {};diff --git a/server/src/consumers/transactions.ts b/server/src/consumers/transactions.tsindex 3b0cdd16..9b048e61 100644--- a/server/src/consumers/transactions.ts+++ b/server/src/consumers/transactions.ts@@ -1,4 +1,4 @@-import { kafka, TOPIC_NAMES, KAFKA_CONSUMER_GROUP } from '@/data/kafka';+import { kafka, TOPIC_NAMES, CONSUMER_IDS } from '@/data/kafka';import {CreateNodeTransactionInput,Transaction,@@ -8,7 +8,7 @@ import { prisma } from '@/data/prisma';import { Prisma } from '@prisma/client';export const initTransactionsConsumer = async () => {- const consumer = kafka.consumer({ groupId: KAFKA_CONSUMER_GROUP });+ const consumer = kafka.consumer({ groupId: CONSUMER_IDS.TRANSACTIONS });await consumer.connect();await consumer.subscribe({ topic: TOPIC_NAMES.TRANSACTIONS });@@ -55,6 +55,7 @@ const handleCreateNodeTransaction = async (transaction: Transaction) => {createdAt: input.createdAt,createdBy: input.createdBy,versionId: input.versionId,+ serverCreatedAt: new Date(),};if (input.attrs !== undefined && input.attrs !== null) {@@ -82,6 +83,7 @@ const handleCreateNodesTransaction = async (transaction: Transaction) => {createdAt: node.createdAt,createdBy: node.createdBy,versionId: node.versionId,+ serverCreatedAt: new Date(),};if (node.attrs !== undefined && node.attrs !== null) {@@ -118,6 +120,7 @@ const handleUpdateNodeTransaction = async (transaction: Transaction) => {updatedAt: input.updatedAt,updatedBy: input.updatedBy,versionId: input.versionId,+ serverUpdatedAt: new Date(),};if (input.attrs !== undefined) {diff --git a/server/src/consumers/update-changes.ts b/server/src/consumers/update-changes.tsnew file mode 100644index 00000000..4a6e1da8--- /dev/null+++ b/server/src/consumers/update-changes.ts@@ -0,0 +1,46 @@+import { kafka, TOPIC_NAMES, CONSUMER_IDS } from '@/data/kafka';+import { ChangeMessage, UpdateChangeData } from '@/types/changes';+import { redis, CHANNEL_NAMES } from '@/data/redis';+import { Update } from '@/types/updates';++export const initUpdateChangesConsumer = async () => {+ const consumer = kafka.consumer({ groupId: CONSUMER_IDS.UPDATE_CHANGES });++ await consumer.connect();+ await consumer.subscribe({ topic: TOPIC_NAMES.UPDATE_CHANGES });++ await consumer.run({+ eachMessage: async ({ message }) => {+ if (!message || !message.value) {+ return;+ }++ const change = JSON.parse(+ message.value.toString(),+ ) as ChangeMessage<UpdateChangeData>;++ await handleUpdateChange(change);+ },+ });+};++const handleUpdateChange = async (change: ChangeMessage<UpdateChangeData>) => {+ const changeData = change.after;+ if (!changeData) {+ return;+ }++ const devices = changeData.devices;+ for (const deviceId of devices) {+ const update: Update = {+ id: changeData.id,+ deviceId: deviceId,+ type: changeData.type,+ content: changeData.content,+ workspaceId: changeData.workspace_id,+ createdAt: changeData.created_at,+ };++ await redis.publish(CHANNEL_NAMES.UPDATES, JSON.stringify(update));+ }+};diff --git a/server/src/consumers/updates.ts b/server/src/consumers/updates.tsnew file mode 100644index 00000000..6fba3353--- /dev/null+++ b/server/src/consumers/updates.ts@@ -0,0 +1,19 @@+import { redis, CHANNEL_NAMES } from '@/data/redis';+import { sockets } from '@/lib/sockets';+import { Update } from '@/types/updates';++export const initUpdatesSubscriber = async () => {+ const subscriber = redis.duplicate();+ await subscriber.connect();+ await subscriber.subscribe(CHANNEL_NAMES.UPDATES, handleMessage);+};++const handleMessage = async (_: string, message: string) => {+ const update = JSON.parse(message) as Update;+ const socket = sockets.getSocket(update.deviceId);+ if (!socket) {+ return;+ }++ socket.send(JSON.stringify(update));+};diff --git a/server/src/data/kafka.ts b/server/src/data/kafka.tsindex b2308f73..41982211 100644--- a/server/src/data/kafka.ts+++ b/server/src/data/kafka.ts@@ -4,10 +4,6 @@ const KAFKA_CLIENT_ID = process.env.KAFKA_CLIENT_ID ?? 'neuron';const KAFKA_BROKERS = process.env.KAFKA_BROKERS ?? '';const KAFKA_USERNAME = process.env.KAFKA_USERNAME;const KAFKA_PASSWORD = process.env.KAFKA_PASSWORD;-const KAFKA_TRANSACTIONS_TOPIC_NAME =- process.env.KAFKA_TRANSACTIONS_TOPIC_NAME ?? 'neuron_transactions';-export const KAFKA_CONSUMER_GROUP =- process.env.KAFKA_CONSUMER_GROUP ?? 'neuron';export const kafka = new Kafka({clientId: KAFKA_CLIENT_ID,@@ -25,7 +21,24 @@ export const kafka = new Kafka({export const producer = kafka.producer();export const TOPIC_NAMES = {- TRANSACTIONS: KAFKA_TRANSACTIONS_TOPIC_NAME,+ TRANSACTIONS:+ process.env.KAFKA_TRANSACTIONS_TOPIC_NAME ?? 'neuron_transactions',+ NODE_CHANGES:+ process.env.KAFKA_NODE_CHANGES_TOPIC_NAME ?? 'neuron_node_changes',+ UPDATE_CHANGES:+ process.env.KAFKA_UPDATE_CHANGES_TOPIC_NAME ?? 'neuron_update_changes',+};++export const CONSUMER_IDS = {+ TRANSACTIONS:+ process.env.KAFKA_TRANSACTIONS_CONSUMER_ID ??+ 'neuron_transactions_consumer',+ NODE_CHANGES:+ process.env.KAFKA_NODE_CHANGES_CONSUMER_ID ??+ 'neuron_node_changes_consumer',+ UPDATE_CHANGES:+ process.env.KAFKA_UPDATE_CHANGES_CONSUMER_ID ??+ 'neuron_update_changes_consumer',};const connectProducer = async () => {diff --git a/server/src/data/redis.ts b/server/src/data/redis.tsnew file mode 100644index 00000000..e0873f47--- /dev/null+++ b/server/src/data/redis.ts@@ -0,0 +1,18 @@+import { createClient } from 'redis';++const REDIS_URL = process.env.REDIS_URL || '';+export const redis = createClient({+ url: REDIS_URL,+});++export const initRedis = async () => {+ await redis.connect();++ redis.on('error', (err) => {+ console.error('Redis client error:', err);+ });+};++export const CHANNEL_NAMES = {+ UPDATES: process.env.REDIS_UPDATES_CHANNEL_NAME || 'neuron_updates',+};diff --git a/server/src/index.ts b/server/src/index.tsindex d8599bf2..49f06386 100644--- a/server/src/index.ts+++ b/server/src/index.ts@@ -1,7 +1,24 @@import { initApi } from '@/api';+import { initRedis } from '@/data/redis';import { initTransactionsConsumer } from '@/consumers/transactions';+import { initNodeChangesConsumer } from '@/consumers/node-changes';+import { initUpdateChangesConsumer } from '@/consumers/update-changes';+import { initUpdatesSubscriber } from '@/consumers/updates';initApi();initTransactionsConsumer().then(() => {console.log('Transactions consumer started');});+initNodeChangesConsumer().then(() => {+ console.log('Node changes consumer started');+});+initUpdateChangesConsumer().then(() => {+ console.log('Update changes consumer started');+});++initRedis().then(() => {+ console.log('Redis initialized');+ // initUpdatesSubscriber().then(() => {+ // console.log('Updates subscriber started');+ // });+});diff --git a/server/src/lib/id.ts b/server/src/lib/id.tsindex c6b7e412..36d0b2f3 100644--- a/server/src/lib/id.ts+++ b/server/src/lib/id.ts@@ -14,6 +14,7 @@ enum IdType {Node = 'nd',Message = 'ms',Device = 'dv',+ Update = 'up',}export class NeuronId {@@ -47,6 +48,12 @@ export class NeuronId {return IdType.Page;case 'channel':return IdType.Channel;+ case 'message':+ return IdType.Message;+ case 'device':+ return IdType.Device;+ case 'update':+ return IdType.Update;default:return IdType.Node;}diff --git a/server/src/lib/jwt.ts b/server/src/lib/jwt.tsnew file mode 100644index 00000000..0e6f7a11--- /dev/null+++ b/server/src/lib/jwt.ts@@ -0,0 +1,34 @@+import jwt from 'jsonwebtoken';++const JwtSecretKey = process.env.JWT_SECRET ?? '';+const JwtAudience = process.env.JWT_AUDIENCE ?? '';+const JwtIssuer = process.env.JWT_ISSUER ?? '';++export type JwtPayload = {+ id: string;+ name: string;+ email: string;+};++export const createJwtToken = (payload: JwtPayload): string => {+ const signOptions: jwt.SignOptions = {+ issuer: JwtIssuer,+ audience: JwtAudience,+ subject: payload.id,+ };++ return jwt.sign(payload, JwtSecretKey, signOptions);+};++export const verifyJwtToken = (token: string): JwtPayload | null => {+ try {+ const decoded = jwt.verify(token, JwtSecretKey, {+ issuer: JwtIssuer,+ audience: JwtAudience,+ });++ return decoded as JwtPayload;+ } catch (err) {+ return null;+ }+};diff --git a/server/src/lib/sockets.ts b/server/src/lib/sockets.tsnew file mode 100644index 00000000..31cdeaee--- /dev/null+++ b/server/src/lib/sockets.ts@@ -0,0 +1,19 @@+import { WebSocket } from 'ws';++class SocketConnections {+ private sockets: Map<string, WebSocket> = new Map();++ public addSocket(deviceId: string, ws: WebSocket) {+ this.sockets.set(deviceId, ws);+ }++ public removeSocket(deviceId: string) {+ this.sockets.delete(deviceId);+ }++ public getSocket(deviceId: string): WebSocket | undefined {+ return this.sockets.get(deviceId);+ }+}++export const sockets = new SocketConnections();diff --git a/server/src/middlewares/auth.ts b/server/src/middlewares/auth.tsindex 8e0b80aa..df37e187 100644--- a/server/src/middlewares/auth.ts+++ b/server/src/middlewares/auth.ts@@ -1,14 +1,10 @@-import jwt from 'jsonwebtoken';import {ApiError,NeuronNextFunction,NeuronRequest,NeuronResponse,} from '@/types/api';--const JwtSecretKey = process.env.JWT_SECRET ?? '';-const JwtAudience = process.env.JWT_AUDIENCE ?? '';-const JwtIssuer = process.env.JWT_ISSUER ?? '';+import { verifyJwtToken } from '@/lib/jwt';export const authMiddleware = (req: NeuronRequest,@@ -24,18 +20,14 @@ export const authMiddleware = (});}- try {- const decoded = jwt.verify(token, JwtSecretKey, {- issuer: JwtIssuer,- audience: JwtAudience,- });-- req.accountId = decoded.sub as string;- next();- } catch (err) {- res.status(400).json({+ const payload = verifyJwtToken(token);+ if (!payload) {+ return res.status(400).json({code: ApiError.Unauthorized,message: 'Invalid Token',});}++ req.accountId = payload.id as string;+ next();};diff --git a/server/src/types/changes.ts b/server/src/types/changes.tsnew file mode 100644index 00000000..77e0f478--- /dev/null+++ b/server/src/types/changes.ts@@ -0,0 +1,53 @@+export type ChangeMessage<T> = {+ before?: T | null;+ after: T;+ source: ChangeSource;+ op: string;+ ts_ms: number;+ ts_ns: number;+ ts_us: number;+ transaction: any;+};++type ChangeSource = {+ version: string;+ connector: string;+ name: string;+ ts_ms: number;+ snapshot: string;+ db: string;+ sequence: string;+ ts_us: number;+ ts_ns: number;+ schema: string;+ table: string;+ txId: number;+ lsn: number;+};++export type NodeChangeData = {+ id: string;+ workspace_id: string;+ parent_id?: string | null;+ type: string;+ index: string | null;+ attrs?: string | null;+ content?: string | null;+ created_at: string;+ created_by: string;+ updated_at?: string | null;+ updated_by?: string | null;+ version_id: string;+ server_created_at: string;+ server_updated_at?: string | null;+ state: string | null;+};++export type UpdateChangeData = {+ id: string;+ workspace_id: string;+ devices: string[];+ type: string;+ content: string;+ created_at: string;+};diff --git a/server/src/types/updates.ts b/server/src/types/updates.tsnew file mode 100644index 00000000..5795f05a--- /dev/null+++ b/server/src/types/updates.ts@@ -0,0 +1,8 @@+export type Update = {+ id: string;+ workspaceId: string;+ deviceId: string;+ type: string;+ content: string;+ createdAt: string;+};