Open-source and local-first Slack and Notion alternative that puts you in control of your data
TypeScript chatchat-applicationcrdteditorelectronknowledge-baselocal-firstnotionnotion-alternativerealtime-collaborationself-hostedslacksqliteteam-collaborationwikiyjs
Improve clean entry data job
Hakan Shehu8d23d1849180
diff --git a/apps/server/src/jobs/clean-entry-data.ts b/apps/server/src/jobs/clean-entry-data.tsindex b4e07392..deb9f318 100644--- a/apps/server/src/jobs/clean-entry-data.ts+++ b/apps/server/src/jobs/clean-entry-data.ts@@ -1,12 +1,16 @@+import { DeleteObjectCommand } from '@aws-sdk/client-s3';import { generateId, IdType } from '@colanode/core';-// import { DeleteObjectCommand } from '@aws-sdk/client-s3';import { database } from '@/data/database';-import { CreateEntryTransaction } from '@/data/schema';+import {+ CreateEntryTransaction,+ CreateFileTombstone,+ CreateMessageTombstone,+} from '@/data/schema';import { JobHandler } from '@/types/jobs';-// import { filesStorage, BUCKET_NAMES } from '@/data/storage';import { eventBus } from '@/lib/event-bus';import { createLogger } from '@/lib/logger';+import { BUCKET_NAMES, filesStorage } from '@/data/storage';const BATCH_SIZE = 100;@@ -51,24 +55,21 @@ export const cleanEntryDataHandler: JobHandler<CleanEntryDataInput> = async (return;}+ const userId = deleteTransaction.created_by;+const parentIds = [input.entryId];while (parentIds.length > 0) {const tempParentIds = parentIds.splice(0, BATCH_SIZE);- const deletedEntryIds = await deleteChildren(- tempParentIds,- input.workspaceId,- deleteTransaction.created_by- );+ const deletedEntryIds = await deleteChildren(tempParentIds, userId);++ await deleteMessages(tempParentIds, userId);+ await deleteFiles(tempParentIds, userId);parentIds.push(...deletedEntryIds);}};-const deleteChildren = async (- parentIds: string[],- workspaceId: string,- userId: string-) => {+const deleteChildren = async (parentIds: string[], userId: string) => {const deletedEntryIds: string[] = [];let hasMore = true;while (hasMore) {@@ -87,33 +88,19 @@ const deleteChildren = async (break;}- // const fileIds: string[] = descendants- // .filter((d) => d.type === 'file')- // .map((d) => d.id);-- // const uploads: SelectUpload[] =- // fileIds.length > 0- // ? await database- // .selectFrom('uploads')- // .selectAll()- // .where('node_id', 'in', fileIds)- // .execute()- // : [];-const entryIds: string[] = descendants.map((d) => d.id);const transactionsToCreate: CreateEntryTransaction[] = descendants.map((descendant) => ({id: generateId(IdType.Transaction),entry_id: descendant.id,root_id: descendant.root_id,- workspace_id: workspaceId,+ workspace_id: descendant.workspace_id,operation: 'delete',created_at: new Date(),created_by: userId,server_created_at: new Date(),}));- // const uploadsToDelete: string[] = uploads.map((u) => u.node_id);await database.transaction().execute(async (trx) => {await trx@@ -131,13 +118,6 @@ const deleteChildren = async (throw new Error('Failed to create transactions');}- // if (uploadsToDelete.length > 0) {- // await trx- // .deleteFrom('uploads')- // .where('node_id', 'in', uploadsToDelete)- // .execute();- // }-await trx.deleteFrom('entries').where('id', 'in', entryIds).execute();await trx.updateTable('collaborations')@@ -149,26 +129,13 @@ const deleteChildren = async (.execute();});- // for (const upload of uploads) {- // const command = new DeleteObjectCommand({- // Bucket: BUCKET_NAMES.FILES,- // Key: upload.path,- // });-- // logger.trace(- // `Deleting file as a descendant of ${parentIds}: ${upload.path}`- // );-- // await filesStorage.send(command);- // }-for (const entry of descendants) {eventBus.publish({type: 'entry_deleted',entryId: entry.id,entryType: entry.type,rootId: entry.root_id,- workspaceId: workspaceId,+ workspaceId: entry.workspace_id,});deletedEntryIds.push(entry.id);@@ -183,3 +150,137 @@ const deleteChildren = async (return deletedEntryIds;};++const deleteMessages = async (entryIds: string[], userId: string) => {+ let hasMore = true;+ while (hasMore) {+ try {+ const messages = await database+ .selectFrom('messages')+ .selectAll()+ .where('entry_id', 'in', entryIds)+ .orderBy('id', 'asc')+ .limit(BATCH_SIZE)+ .execute();++ if (messages.length === 0) {+ hasMore = false;+ break;+ }++ const messageIds: string[] = messages.map((m) => m.id);+ const messageTombstonesToCreate: CreateMessageTombstone[] = messages.map(+ (message) => ({+ id: message.id,+ root_id: message.root_id,+ workspace_id: message.workspace_id,+ deleted_at: new Date(),+ deleted_by: userId,+ })+ );++ await database.transaction().execute(async (trx) => {+ await trx+ .deleteFrom('messages')+ .where('id', 'in', messageIds)+ .execute();++ await trx+ .deleteFrom('message_reactions')+ .where('message_id', 'in', messageIds)+ .execute();++ await trx+ .deleteFrom('message_interactions')+ .where('message_id', 'in', messageIds)+ .execute();++ await trx+ .insertInto('message_tombstones')+ .values(messageTombstonesToCreate)+ .execute();+ });++ for (const message of messages) {+ eventBus.publish({+ type: 'message_deleted',+ messageId: message.id,+ rootId: message.root_id,+ workspaceId: message.workspace_id,+ });+ }++ hasMore = messages.length === BATCH_SIZE;+ } catch (error) {+ logger.error(`Error deleting messages for ${entryIds}: ${error}`);+ hasMore = false;+ }+ }+};++const deleteFiles = async (entryIds: string[], userId: string) => {+ let hasMore = true;+ while (hasMore) {+ try {+ const files = await database+ .selectFrom('files')+ .selectAll()+ .where('entry_id', 'in', entryIds)+ .orderBy('id', 'asc')+ .limit(BATCH_SIZE)+ .execute();++ if (files.length === 0) {+ hasMore = false;+ break;+ }++ const fileIds: string[] = files.map((m) => m.id);+ const fileTombstonesToCreate: CreateFileTombstone[] = files.map(+ (file) => ({+ id: file.id,+ root_id: file.root_id,+ workspace_id: file.workspace_id,+ deleted_at: new Date(),+ deleted_by: userId,+ })+ );++ await database.transaction().execute(async (trx) => {+ await trx.deleteFrom('files').where('id', 'in', fileIds).execute();++ await trx+ .deleteFrom('file_interactions')+ .where('file_id', 'in', fileIds)+ .execute();++ await trx+ .insertInto('file_tombstones')+ .values(fileTombstonesToCreate)+ .execute();+ });++ for (const file of files) {+ eventBus.publish({+ type: 'file_deleted',+ fileId: file.id,+ rootId: file.root_id,+ workspaceId: file.workspace_id,+ });++ const path = `files/${file.workspace_id}/${file.id}${file.extension}`;+ const command = new DeleteObjectCommand({+ Bucket: BUCKET_NAMES.FILES,+ Key: path,+ });++ await filesStorage.send(command);+ }++ hasMore = files.length === BATCH_SIZE;+ } catch (error) {+ logger.error(`Error deleting files for ${entryIds}: ${error}`);+ hasMore = false;+ }+ }+};