High performance self-hosted photo and video management solution.
TypeScript backup-toolfluttergoogle-photosgoogle-photos-alternativejavascriptmobile-appnestjsnodejsphoto-galleryphotosphotos-managementself-hostedsveltesveltekittypescriptvideos
refactor: sync service (#19225)
Jason Rasmussen35280b94cc3f
diff --git a/server/src/services/sync.service.ts b/server/src/services/sync.service.tsindex 4705fc8e1f..46937a570f 100644--- a/server/src/services/sync.service.ts+++ b/server/src/services/sync.service.ts@@ -12,6 +12,8 @@ import {AssetFullSyncDto,SyncAckDeleteDto,SyncAckSetDto,+ SyncAssetV1,+ SyncItem,SyncStreamDto,} from 'src/dtos/sync.dto';import { AssetVisibility, DatabaseAction, EntityType, Permission, SyncEntityType, SyncRequestType } from 'src/enum';@@ -20,7 +22,35 @@ import { SyncAck } from 'src/types';import { getMyPartnerIds } from 'src/utils/asset.util';import { hexOrBufferToBase64 } from 'src/utils/bytes';import { setIsEqual } from 'src/utils/set';-import { fromAck, serialize, toAck } from 'src/utils/sync';+import { fromAck, serialize, SerializeOptions, toAck } from 'src/utils/sync';++type CheckpointMap = Partial<Record<SyncEntityType, SyncAck>>;+type AssetLike = Omit<SyncAssetV1, 'checksum' | 'thumbhash'> & {+ checksum: Buffer<ArrayBufferLike>;+ thumbhash: Buffer<ArrayBufferLike> | null;+};++const COMPLETE_ID = 'complete';++const mapSyncAssetV1 = ({ checksum, thumbhash, ...data }: AssetLike): SyncAssetV1 => ({+ ...data,+ checksum: hexOrBufferToBase64(checksum),+ thumbhash: thumbhash ? hexOrBufferToBase64(thumbhash) : null,+});++const isEntityBackfillComplete = (entity: { createId: string }, checkpoint: SyncAck | undefined): boolean =>+ entity.createId === checkpoint?.updateId && checkpoint.extraId === COMPLETE_ID;++const getStartId = (entity: { createId: string }, checkpoint: SyncAck | undefined): string | undefined =>+ checkpoint?.updateId === entity.createId ? checkpoint?.extraId : undefined;++const send = <T extends keyof SyncItem, D extends SyncItem[T]>(response: Writable, item: SerializeOptions<T, D>) => {+ response.write(serialize(item));+};++const sendEntityBackfillCompleteAck = (response: Writable, ackType: SyncEntityType, id: string) => {+ send(response, { type: SyncEntityType.SyncAckV1, data: {}, ackType, ids: [id, COMPLETE_ID] });+};const FULL_SYNC = { needsFullSync: true, deleted: [], upserted: [] };export const SYNC_TYPES_ORDER = [@@ -89,266 +119,48 @@ export class SyncService extends BaseService {}const checkpoints = await this.syncRepository.getCheckpoints(sessionId);- const checkpointMap: Partial<Record<SyncEntityType, SyncAck>> = Object.fromEntries(- checkpoints.map(({ type, ack }) => [type, fromAck(ack)]),- );+ const checkpointMap: CheckpointMap = Object.fromEntries(checkpoints.map(({ type, ack }) => [type, fromAck(ack)]));for (const type of SYNC_TYPES_ORDER.filter((type) => dto.types.includes(type))) {switch (type) {case SyncRequestType.UsersV1: {- const deletes = this.syncRepository.getUserDeletes(checkpointMap[SyncEntityType.UserDeleteV1]);- for await (const { id, ...data } of deletes) {- response.write(serialize({ type: SyncEntityType.UserDeleteV1, ids: [id], data }));- }-- const upserts = this.syncRepository.getUserUpserts(checkpointMap[SyncEntityType.UserV1]);- for await (const { updateId, ...data } of upserts) {- response.write(serialize({ type: SyncEntityType.UserV1, ids: [updateId], data }));- }-+ await this.syncUsersV1(response, checkpointMap);break;}case SyncRequestType.PartnersV1: {- const deletes = this.syncRepository.getPartnerDeletes(- auth.user.id,- checkpointMap[SyncEntityType.PartnerDeleteV1],- );- for await (const { id, ...data } of deletes) {- response.write(serialize({ type: SyncEntityType.PartnerDeleteV1, ids: [id], data }));- }-- const upserts = this.syncRepository.getPartnerUpserts(auth.user.id, checkpointMap[SyncEntityType.PartnerV1]);- for await (const { updateId, ...data } of upserts) {- response.write(serialize({ type: SyncEntityType.PartnerV1, ids: [updateId], data }));- }-+ await this.syncPartnersV1(response, checkpointMap, auth);break;}case SyncRequestType.AssetsV1: {- const deletes = this.syncRepository.getAssetDeletes(- auth.user.id,- checkpointMap[SyncEntityType.AssetDeleteV1],- );- for await (const { id, ...data } of deletes) {- response.write(serialize({ type: SyncEntityType.AssetDeleteV1, ids: [id], data }));- }-- const upserts = this.syncRepository.getAssetUpserts(auth.user.id, checkpointMap[SyncEntityType.AssetV1]);- for await (const { updateId, checksum, thumbhash, ...data } of upserts) {- response.write(- serialize({- type: SyncEntityType.AssetV1,- ids: [updateId],- data: {- ...data,- checksum: hexOrBufferToBase64(checksum),- thumbhash: thumbhash ? hexOrBufferToBase64(thumbhash) : null,- },- }),- );- }-+ await this.syncAssetsV1(response, checkpointMap, auth);break;}case SyncRequestType.PartnerAssetsV1: {- const deletes = this.syncRepository.getPartnerAssetDeletes(- auth.user.id,- checkpointMap[SyncEntityType.PartnerAssetDeleteV1],- );- for await (const { id, ...data } of deletes) {- response.write(serialize({ type: SyncEntityType.PartnerAssetDeleteV1, ids: [id], data }));- }-- const checkpoint = checkpointMap[SyncEntityType.PartnerAssetBackfillV1];- const partnerAssetCheckpoint = checkpointMap[SyncEntityType.PartnerAssetV1];-- const partners = await this.syncRepository.getPartnerBackfill(auth.user.id, checkpoint?.updateId);-- if (partnerAssetCheckpoint) {- for (const partner of partners) {- if (partner.createId === checkpoint?.updateId && checkpoint.extraId === 'complete') {- continue;- }- const partnerCheckpoint = checkpoint?.updateId === partner.createId ? checkpoint?.extraId : undefined;- const backfill = this.syncRepository.getPartnerAssetsBackfill(- partner.sharedById,- partnerCheckpoint,- partnerAssetCheckpoint.updateId,- );-- for await (const { updateId, checksum, thumbhash, ...data } of backfill) {- response.write(- serialize({- type: SyncEntityType.PartnerAssetBackfillV1,- ids: [updateId],- data: {- ...data,- checksum: hexOrBufferToBase64(checksum),- thumbhash: thumbhash ? hexOrBufferToBase64(thumbhash) : null,- },- }),- );- }- response.write(- serialize({- type: SyncEntityType.SyncAckV1,- data: {},- ackType: SyncEntityType.PartnerAssetBackfillV1,- ids: [partner.sharedById, 'complete'],- }),- );- }- } else if (partners.length > 0) {- await this.syncRepository.upsertCheckpoints([- {- type: SyncEntityType.PartnerAssetBackfillV1,- sessionId,- ack: toAck({- type: SyncEntityType.PartnerAssetBackfillV1,- updateId: partners.at(-1)!.createId,- extraId: 'complete',- }),- },- ]);- }-- const upserts = this.syncRepository.getPartnerAssetsUpserts(- auth.user.id,- checkpointMap[SyncEntityType.PartnerAssetV1],- );- for await (const { updateId, checksum, thumbhash, ...data } of upserts) {- response.write(- serialize({- type: SyncEntityType.PartnerAssetV1,- ids: [updateId],- data: {- ...data,- checksum: hexOrBufferToBase64(checksum),- thumbhash: thumbhash ? hexOrBufferToBase64(thumbhash) : null,- },- }),- );- }+ await this.syncPartnerAssetsV1(response, checkpointMap, auth, sessionId);break;}case SyncRequestType.AssetExifsV1: {- const upserts = this.syncRepository.getAssetExifsUpserts(- auth.user.id,- checkpointMap[SyncEntityType.AssetExifV1],- );- for await (const { updateId, ...data } of upserts) {- response.write(serialize({ type: SyncEntityType.AssetExifV1, ids: [updateId], data }));- }-+ await this.syncAssetExifsV1(response, checkpointMap, auth);break;}case SyncRequestType.PartnerAssetExifsV1: {- const checkpoint = checkpointMap[SyncEntityType.PartnerAssetExifBackfillV1];- const partnerAssetCheckpoint = checkpointMap[SyncEntityType.PartnerAssetExifV1];-- const partners = await this.syncRepository.getPartnerBackfill(auth.user.id, checkpoint?.updateId);-- if (partnerAssetCheckpoint) {- for (const partner of partners) {- if (partner.createId === checkpoint?.updateId && checkpoint.extraId === 'complete') {- continue;- }- const partnerCheckpoint = checkpoint?.updateId === partner.createId ? checkpoint?.extraId : undefined;- const backfill = this.syncRepository.getPartnerAssetExifsBackfill(- partner.sharedById,- partnerCheckpoint,- partnerAssetCheckpoint.updateId,- );-- for await (const { updateId, ...data } of backfill) {- response.write(- serialize({- type: SyncEntityType.PartnerAssetExifBackfillV1,- ids: [updateId],- data,- }),- );- }- response.write(- serialize({- type: SyncEntityType.SyncAckV1,- data: {},- ackType: SyncEntityType.PartnerAssetExifBackfillV1,- ids: [partner.sharedById, 'complete'],- }),- );- }- } else if (partners.length > 0) {- await this.syncRepository.upsertCheckpoints([- {- type: SyncEntityType.PartnerAssetExifBackfillV1,- sessionId,- ack: toAck({- type: SyncEntityType.PartnerAssetExifBackfillV1,- updateId: partners.at(-1)!.createId,- extraId: 'complete',- }),- },- ]);- }-- const upserts = this.syncRepository.getPartnerAssetExifsUpserts(- auth.user.id,- checkpointMap[SyncEntityType.PartnerAssetExifV1],- );- for await (const { updateId, ...data } of upserts) {- response.write(- serialize({- type: SyncEntityType.PartnerAssetExifV1,- ids: [updateId],- data,- }),- );- }-+ await this.syncPartnerAssetExifsV1(response, checkpointMap, auth, sessionId);break;}case SyncRequestType.AlbumsV1: {- const deletes = this.syncRepository.getAlbumDeletes(- auth.user.id,- checkpointMap[SyncEntityType.AlbumDeleteV1],- );- for await (const { id, ...data } of deletes) {- response.write(serialize({ type: SyncEntityType.AlbumDeleteV1, ids: [id], data }));- }-- const upserts = this.syncRepository.getAlbumUpserts(auth.user.id, checkpointMap[SyncEntityType.AlbumV1]);- for await (const { updateId, ...data } of upserts) {- response.write(serialize({ type: SyncEntityType.AlbumV1, ids: [updateId], data }));- }-+ await this.syncAlbumsV1(response, checkpointMap, auth);break;}case SyncRequestType.AlbumUsersV1: {- const deletes = this.syncRepository.getAlbumUserDeletes(- auth.user.id,- checkpointMap[SyncEntityType.AlbumUserDeleteV1],- );- for await (const { id, ...data } of deletes) {- response.write(serialize({ type: SyncEntityType.AlbumUserDeleteV1, ids: [id], data }));- }-- const upserts = this.syncRepository.getAlbumUserUpserts(- auth.user.id,- checkpointMap[SyncEntityType.AlbumUserV1],- );- for await (const { updateId, ...data } of upserts) {- response.write(serialize({ type: SyncEntityType.AlbumUserV1, ids: [updateId], data }));- }-+ await this.syncAlbumUsersV1(response, checkpointMap, auth);break;}@@ -362,6 +174,192 @@ export class SyncService extends BaseService {response.end();}+ private async syncUsersV1(response: Writable, checkpointMap: CheckpointMap) {+ const deletes = this.syncRepository.getUserDeletes(checkpointMap[SyncEntityType.UserDeleteV1]);+ for await (const { id, ...data } of deletes) {+ send(response, { type: SyncEntityType.UserDeleteV1, ids: [id], data });+ }++ const upserts = this.syncRepository.getUserUpserts(checkpointMap[SyncEntityType.UserV1]);+ for await (const { updateId, ...data } of upserts) {+ send(response, { type: SyncEntityType.UserV1, ids: [updateId], data });+ }+ }++ private async syncPartnersV1(response: Writable, checkpointMap: CheckpointMap, auth: AuthDto) {+ const deletes = this.syncRepository.getPartnerDeletes(auth.user.id, checkpointMap[SyncEntityType.PartnerDeleteV1]);+ for await (const { id, ...data } of deletes) {+ send(response, { type: SyncEntityType.PartnerDeleteV1, ids: [id], data });+ }++ const upserts = this.syncRepository.getPartnerUpserts(auth.user.id, checkpointMap[SyncEntityType.PartnerV1]);+ for await (const { updateId, ...data } of upserts) {+ send(response, { type: SyncEntityType.PartnerV1, ids: [updateId], data });+ }+ }++ private async syncAssetsV1(response: Writable, checkpointMap: CheckpointMap, auth: AuthDto) {+ const deletes = this.syncRepository.getAssetDeletes(auth.user.id, checkpointMap[SyncEntityType.AssetDeleteV1]);+ for await (const { id, ...data } of deletes) {+ send(response, { type: SyncEntityType.AssetDeleteV1, ids: [id], data });+ }++ const upserts = this.syncRepository.getAssetUpserts(auth.user.id, checkpointMap[SyncEntityType.AssetV1]);+ for await (const { updateId, ...data } of upserts) {+ send(response, { type: SyncEntityType.AssetV1, ids: [updateId], data: mapSyncAssetV1(data) });+ }+ }++ private async syncPartnerAssetsV1(+ response: Writable,+ checkpointMap: CheckpointMap,+ auth: AuthDto,+ sessionId: string,+ ) {+ const backfillType = SyncEntityType.PartnerAssetBackfillV1;+ const upsertType = SyncEntityType.PartnerAssetV1;+ const deleteType = SyncEntityType.PartnerAssetDeleteV1;++ const backfillCheckpoint = checkpointMap[backfillType];+ const upsertCheckpoint = checkpointMap[upsertType];++ const deletes = this.syncRepository.getPartnerAssetDeletes(auth.user.id, checkpointMap[deleteType]);++ for await (const { id, ...data } of deletes) {+ send(response, { type: deleteType, ids: [id], data });+ }++ const partners = await this.syncRepository.getPartnerBackfill(auth.user.id, backfillCheckpoint?.updateId);++ if (upsertCheckpoint) {+ const endId = upsertCheckpoint.updateId;++ for (const partner of partners) {+ if (isEntityBackfillComplete(partner, backfillCheckpoint)) {+ continue;+ }++ const startId = getStartId(partner, backfillCheckpoint);+ const backfill = this.syncRepository.getPartnerAssetsBackfill(partner.sharedById, startId, endId);++ for await (const { updateId, ...data } of backfill) {+ send(response, {+ type: backfillType,+ ids: [updateId],+ data: mapSyncAssetV1(data),+ });+ }++ sendEntityBackfillCompleteAck(response, backfillType, partner.sharedById);+ }+ } else if (partners.length > 0) {+ await this.upsertBackfillCheckpoint({+ type: backfillType,+ sessionId,+ createId: partners.at(-1)!.createId,+ });+ }++ const upserts = this.syncRepository.getPartnerAssetsUpserts(auth.user.id, checkpointMap[upsertType]);+ for await (const { updateId, ...data } of upserts) {+ send(response, { type: upsertType, ids: [updateId], data: mapSyncAssetV1(data) });+ }+ }++ private async syncAssetExifsV1(response: Writable, checkpointMap: CheckpointMap, auth: AuthDto) {+ const upserts = this.syncRepository.getAssetExifsUpserts(auth.user.id, checkpointMap[SyncEntityType.AssetExifV1]);+ for await (const { updateId, ...data } of upserts) {+ send(response, { type: SyncEntityType.AssetExifV1, ids: [updateId], data });+ }+ }++ private async syncPartnerAssetExifsV1(+ response: Writable,+ checkpointMap: CheckpointMap,+ auth: AuthDto,+ sessionId: string,+ ) {+ const backfillType = SyncEntityType.PartnerAssetExifBackfillV1;+ const upsertType = SyncEntityType.PartnerAssetExifV1;++ const backfillCheckpoint = checkpointMap[backfillType];+ const upsertCheckpoint = checkpointMap[upsertType];++ const partners = await this.syncRepository.getPartnerBackfill(auth.user.id, backfillCheckpoint?.updateId);++ if (upsertCheckpoint) {+ const endId = upsertCheckpoint.updateId;++ for (const partner of partners) {+ if (isEntityBackfillComplete(partner, backfillCheckpoint)) {+ continue;+ }++ const startId = getStartId(partner, backfillCheckpoint);+ const backfill = this.syncRepository.getPartnerAssetExifsBackfill(partner.sharedById, startId, endId);++ for await (const { updateId, ...data } of backfill) {+ send(response, { type: backfillType, ids: [updateId], data });+ }++ sendEntityBackfillCompleteAck(response, backfillType, partner.sharedById);+ }+ } else if (partners.length > 0) {+ await this.upsertBackfillCheckpoint({+ type: backfillType,+ sessionId,+ createId: partners.at(-1)!.createId,+ });+ }++ const upserts = this.syncRepository.getPartnerAssetExifsUpserts(auth.user.id, checkpointMap[upsertType]);+ for await (const { updateId, ...data } of upserts) {+ send(response, { type: upsertType, ids: [updateId], data });+ }+ }++ private async syncAlbumsV1(response: Writable, checkpointMap: CheckpointMap, auth: AuthDto) {+ const deletes = this.syncRepository.getAlbumDeletes(auth.user.id, checkpointMap[SyncEntityType.AlbumDeleteV1]);+ for await (const { id, ...data } of deletes) {+ send(response, { type: SyncEntityType.AlbumDeleteV1, ids: [id], data });+ }++ const upserts = this.syncRepository.getAlbumUpserts(auth.user.id, checkpointMap[SyncEntityType.AlbumV1]);+ for await (const { updateId, ...data } of upserts) {+ send(response, { type: SyncEntityType.AlbumV1, ids: [updateId], data });+ }+ }++ private async syncAlbumUsersV1(response: Writable, checkpointMap: CheckpointMap, auth: AuthDto) {+ const deletes = this.syncRepository.getAlbumUserDeletes(+ auth.user.id,+ checkpointMap[SyncEntityType.AlbumUserDeleteV1],+ );+ for await (const { id, ...data } of deletes) {+ send(response, { type: SyncEntityType.AlbumUserDeleteV1, ids: [id], data });+ }++ const upserts = this.syncRepository.getAlbumUserUpserts(auth.user.id, checkpointMap[SyncEntityType.AlbumUserV1]);+ for await (const { updateId, ...data } of upserts) {+ send(response, { type: SyncEntityType.AlbumUserV1, ids: [updateId], data });+ }+ }++ private async upsertBackfillCheckpoint(item: { type: SyncEntityType; sessionId: string; createId: string }) {+ const { type, sessionId, createId } = item;+ await this.syncRepository.upsertCheckpoints([+ {+ type,+ sessionId,+ ack: toAck({+ type,+ updateId: createId,+ extraId: COMPLETE_ID,+ }),+ },+ ]);+ }+async getFullSync(auth: AuthDto, dto: AssetFullSyncDto): Promise<AssetResponseDto[]> {// mobile implementation is faster if this is a single idconst userId = dto.userId || auth.user.id;diff --git a/server/src/utils/sync.ts b/server/src/utils/sync.tsindex 893c94dfcd..82222708af 100644--- a/server/src/utils/sync.ts+++ b/server/src/utils/sync.ts@@ -18,14 +18,17 @@ export const toAck = ({ type, updateId, extraId }: SyncAck) =>export const mapJsonLine = (object: unknown) => JSON.stringify(object) + '\n';+export type SerializeOptions<T extends keyof SyncItem, D extends SyncItem[T]> = {+ type: T;+ data: Exact<SyncItem[T], D>;+ ids: [string] | [string, string];+ ackType?: SyncEntityType;+};+export const serialize = <T extends keyof SyncItem, D extends SyncItem[T]>({type,data,ids,ackType,-}: {- type: T;- data: Exact<SyncItem[T], D>;- ids: [string] | [string, string];- ackType?: SyncEntityType;-}) => mapJsonLine({ type, data, ack: toAck({ type: ackType ?? type, updateId: ids[0], extraId: ids[1] }) });+}: SerializeOptions<T, D>) =>+ mapJsonLine({ type, data, ack: toAck({ type: ackType ?? type, updateId: ids[0], extraId: ids[1] }) });