Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.
TypeScript cloud-alternativecollaborationcollaborative-editingdropboxfile-serverfile-sharingfile-storagefile-syncfull-text-searchhomelabnextcloudon-premiseopen-sourceprivacy-firstprivate-cloudrealtimesecureself-hostedstorage-serverwebdav
import { HttpException, HttpStatus, Injectable, Logger, StreamableFile } from '@nestjs/common'
import { FastifyReply } from 'fastify'
import fs from 'fs/promises'
import { Dirent, Stats } from 'node:fs'
import path from 'node:path'
import { toErrorMessage } from '../../../common/errors.js'
import { regExpPathPattern } from '../../../common/functions.js'
import { FILE_ERROR } from '../../files/constants/errors.js'
import { FILE_OPERATION } from '../../files/constants/operations.js'
import { FileError } from '../../files/models/file-error.js'
import type { SaveStreamTmpFileValidationContext } from '../../files/interfaces/save-stream.interface.js'
import { LockConflict } from '../../files/models/file-lock-error.js'
import { FilesManager } from '../../files/services/files-manager.service.js'
import { checksumFile, isInternalTemporaryEntry, isPathExists, isPathIsDir, touchFile } from '../../files/utils/files.js'
import { SendFile } from '../../files/utils/send-file.js'
import { ParseDiffContext } from '../../spaces/interfaces/space-diff.interface.js'
import { FastifySpaceRequest } from '../../spaces/interfaces/space-request.interface.js'
import { SpaceEnv } from '../../spaces/models/space-env.model.js'
import { SpacesManager } from '../../spaces/services/spaces-manager.service.js'
import { UserModel } from '../../users/models/user.model.js'
import { F_SPECIAL_STAT, F_STAT, SYNC_CHECKSUM_ALG, SYNC_DIFF_DONE } from '../constants/sync.js'
import type { SyncCopyMoveDto, SyncDiffDto, SyncMakeDto, SyncPropsDto } from '../dtos/sync-operations.dto.js'
import type { SyncUploadDto } from '../dtos/sync-upload.dto.js'
import { SyncFileSpecialStats, SyncFileStats } from '../interfaces/sync-diff.interface.js'
import { getSyncTmpFilePath } from '../utils/functions.js'
import { SYNC_PATH_TO_SPACE_SEGMENTS } from '../utils/routes.js'
import { SyncQueries } from './sync-queries.service.js'
@Injectable()
export class SyncManager {
private readonly logger = new Logger(SyncManager.name)
constructor(
private readonly spacesManager: SpacesManager,
private readonly filesManager: FilesManager,
private readonly syncQueries: SyncQueries
) {}
async download(req: FastifySpaceRequest, res: FastifyReply): Promise<StreamableFile> {
const sendFile: SendFile = this.filesManager.sendFileFromSpace(req.space)
try {
await sendFile.checks()
return await sendFile.stream(req, res)
} catch (e) {
this.handleError(req.space, req.method, e)
}
}
async upload(req: FastifySpaceRequest, syncUploadDto: SyncUploadDto): Promise<{ ino: number }> {
const tmpPath = getSyncTmpFilePath(req.space.realPath)
try {
await this.filesManager.saveStream(req.user, req.space, req, {
tmpPath: tmpPath,
expectedUploadSize: syncUploadDto.size,
...(syncUploadDto.checksum && { checksumAlg: SYNC_CHECKSUM_ALG }),
validateTmpFile: async ({ tmpPath, checksum }: SaveStreamTmpFileValidationContext) => {
const tmpStats = await fs.stat(tmpPath)
if (tmpStats.size !== syncUploadDto.size) {
throw new FileError(HttpStatus.BAD_REQUEST, `sizes are not identical : ${tmpStats.size} != ${syncUploadDto.size}`)
}
if (syncUploadDto.checksum && checksum !== syncUploadDto.checksum) {
throw new FileError(HttpStatus.BAD_REQUEST, 'checksums are not identical')
}
// Apply mtime before moving the file and publishing its event.
await touchFile(tmpPath, syncUploadDto.mtime)
}
})
// return inode number
const fileStats = await fs.stat(req.space.realPath)
return { ino: fileStats.ino }
} catch (e) {
this.handleError(req.space, req.method, e)
}
}
async delete(req: FastifySpaceRequest): Promise<void> {
try {
return await this.filesManager.delete(req.user, req.space)
} catch (e) {
this.handleError(req.space, FILE_OPERATION.DELETE, e)
}
}
async props(req: FastifySpaceRequest, syncPropsDto: SyncPropsDto): Promise<void> {
try {
await this.filesManager.touch(req.user, req.space, syncPropsDto.mtime, false)
} catch (e) {
this.handleError(req.space, FILE_OPERATION.TOUCH, e)
}
}
async make(req: FastifySpaceRequest, syncMakeDto: SyncMakeDto): Promise<{ ino: number }> {
try {
if (syncMakeDto.type === 'directory') {
await this.filesManager.mkDir(req.user, req.space, true)
} else {
await this.filesManager.mkFile(req.user, req.space, true)
}
await touchFile(req.space.realPath, syncMakeDto.mtime)
return { ino: (await fs.stat(req.space.realPath)).ino }
} catch (e) {
this.handleError(req.space, `${FILE_OPERATION.MAKE} ${syncMakeDto.type}`, e)
}
}
async copyMove(req: FastifySpaceRequest, syncCopyMoveDto: SyncCopyMoveDto, isMove: true): Promise<void>
async copyMove(req: FastifySpaceRequest, syncCopyMoveDto: SyncCopyMoveDto, isMove: false): Promise<{ ino: number; mtime: number }>
async copyMove(req: FastifySpaceRequest, syncCopyMoveDto: SyncCopyMoveDto, isMove: boolean): Promise<void | { ino: number; mtime: number }> {
const dstSpace = await this.spacesManager.spaceEnv(req.user, SYNC_PATH_TO_SPACE_SEGMENTS(syncCopyMoveDto.destination))
if (!dstSpace) {
throw new HttpException('Destination space not found', HttpStatus.NOT_FOUND)
}
try {
await this.filesManager.copyMove(req.user, req.space, dstSpace, isMove, true, true)
if (!isMove) {
if (syncCopyMoveDto.mtime) {
// update mtime
await touchFile(dstSpace.realPath, syncCopyMoveDto.mtime)
}
// return inode & mtime
const stats = await fs.stat(dstSpace.realPath)
return { ino: stats.ino, mtime: Math.floor(stats.mtime.getTime() / 1000) }
}
} catch (e) {
this.handleError(req.space, isMove ? FILE_OPERATION.MOVE : FILE_OPERATION.COPY, e, dstSpace)
}
}
async diff(user: UserModel, pathId: number, syncDiff: SyncDiffDto, res: FastifyReply): Promise<void> {
if (!user.clientId) {
throw new HttpException('Client id is missing', HttpStatus.BAD_REQUEST)
}
const syncPathSettings = await this.syncQueries.getPathSettings(user.clientId, pathId)
if (!syncPathSettings) {
throw new HttpException('Path not found', HttpStatus.NOT_FOUND)
}
let space: SpaceEnv | null
try {
space = await this.spacesManager.spaceEnv(user, SYNC_PATH_TO_SPACE_SEGMENTS(syncPathSettings.remotePath))
} catch (e) {
throw new HttpException(toErrorMessage(e), HttpStatus.BAD_REQUEST)
}
if (!space) {
throw new HttpException('Space not found', HttpStatus.NOT_FOUND)
}
if (space.quotaIsExceeded) {
throw new HttpException(FILE_ERROR.STORAGE_QUOTA_EXCEEDED, HttpStatus.INSUFFICIENT_STORAGE)
}
if (!(await isPathExists(space.realPath))) {
throw new HttpException(`Remote path not found : ${syncPathSettings.remotePath}`, HttpStatus.NOT_FOUND)
}
if (!(await isPathIsDir(space.realPath))) {
throw new HttpException('Remote path must be a directory', HttpStatus.BAD_REQUEST)
}
// start
res.raw.writeHead(200, {
'Content-Type': 'text/plain; charset=utf-8',
'Transfer-Encoding': 'chunked'
})
try {
for await (const f of this.parseSyncPath(space, syncDiff)) {
res.raw.write(`${JSON.stringify(f)}\n`)
}
res.raw.write(SYNC_DIFF_DONE)
} catch (e) {
const errorMessage = toErrorMessage(e)
this.logger.error({ tag: this.diff.name, msg: errorMessage })
res.raw.write(`${errorMessage}\n`)
res.status(HttpStatus.INTERNAL_SERVER_ERROR)
}
res.raw.end()
}
private async *parseSyncPath(space: SpaceEnv, syncDiff: SyncDiffDto): AsyncGenerator<Record<string, SyncFileStats | SyncFileSpecialStats>> {
const context: ParseDiffContext = {
regexBasePath: regExpPathPattern(space.realPath),
syncDiff: syncDiff
}
yield* this.parseFiles(space.realPath, context)
}
private async *parseFiles(dir: string, ctx: ParseDiffContext): AsyncGenerator<Record<string, SyncFileStats | SyncFileSpecialStats>> {
try {
for (const entry of await fs.readdir(dir, { withFileTypes: true })) {
const realPath = path.join(entry.parentPath, entry.name)
if (isInternalTemporaryEntry(entry.name)) {
this.logger.verbose({ tag: this.parseFiles.name, msg: `ignore internal temporary entry: ${realPath}` })
continue
}
if (!entry.isDirectory() && !entry.isFile()) {
this.logger.log({ tag: this.parseFiles.name, msg: `ignore special file: ${realPath}` })
continue
}
if (entry.isDirectory()) {
const dirStats = await this.analyzeFile(entry, ctx)
if (dirStats !== null) {
yield dirStats
}
yield* this.parseFiles(realPath, ctx)
} else {
const fileStats = await this.analyzeFile(entry, ctx)
if (fileStats !== null) {
yield fileStats
}
}
}
} catch (e) {
this.logger.error({ tag: this.parseFiles.name, msg: `unable to parse directory : ${dir} : ${toErrorMessage(e)}` })
throw new Error('Unable to parse path')
}
}
private async analyzeFile(entry: Dirent, ctx: ParseDiffContext): Promise<Record<string, SyncFileStats | SyncFileSpecialStats> | null> {
if (ctx.syncDiff.defaultFilters.has(entry.name)) {
return null
}
const realPath = path.join(entry.parentPath, entry.name)
const filePath = realPath.replace(ctx.regexBasePath, '')
let stats: Stats
try {
stats = await fs.stat(realPath)
} catch (e) {
this.logger.warn({ tag: this.analyzeFile.name, msg: `unable to get file stats : ${realPath} : ${toErrorMessage(e)}` })
return { [filePath]: [F_SPECIAL_STAT.ERROR, toErrorMessage(e)] }
}
if (ctx.syncDiff.pathFilters && ctx.syncDiff.pathFilters.test(filePath)) {
this.logger.verbose({ tag: this.analyzeFile.name, msg: `ignore filtered file : ${realPath}` })
return { [filePath]: [F_SPECIAL_STAT.FILTERED, stats.isDirectory()] }
}
const fileStats: SyncFileStats = [
stats.isDirectory(),
stats.isDirectory() ? 0 : stats.size,
Math.floor(stats.mtime.getTime() / 1000),
stats.ino,
null
]
if (ctx.syncDiff.secureDiff && !fileStats[F_STAT.IS_DIR]) {
try {
await this.checkSumFile(ctx, filePath, realPath, fileStats)
} catch (e) {
this.logger.error({ tag: this.analyzeFile.name, msg: `file error : ${realPath} - ${toErrorMessage(e)}` })
return { [filePath]: [F_SPECIAL_STAT.ERROR, toErrorMessage(e)] }
}
}
return { [filePath]: fileStats }
}
private async checkSumFile(ctx: ParseDiffContext, filePath: string, realPath: string, fileStats: SyncFileStats): Promise<void> {
if (!ctx.syncDiff.firstSync && ctx.syncDiff.snapshot!.has(filePath)) {
const snapFileStats = ctx.syncDiff.snapshot!.get(filePath)!
if (
snapFileStats[F_STAT.CHECKSUM] &&
snapFileStats[F_STAT.MTIME] == fileStats[F_STAT.MTIME] &&
snapFileStats[F_STAT.SIZE] == fileStats[F_STAT.SIZE] &&
snapFileStats[F_STAT.INO] == fileStats[F_STAT.INO]
) {
fileStats[F_STAT.CHECKSUM] = snapFileStats[F_STAT.CHECKSUM]
return
}
}
fileStats[F_STAT.CHECKSUM] = await checksumFile(realPath, SYNC_CHECKSUM_ALG)
}
private handleError(space: SpaceEnv, action: string, e: unknown, dstSpace?: SpaceEnv): never {
this.logger.error({
tag: this.handleError.name,
msg: `unable to ${action} ${space.url}${dstSpace?.url ? ` -> ${dstSpace.url}` : ''} : ${toErrorMessage(e)}`
})
// Remove the last part to avoid exposing the path
const errorMsg = toErrorMessage(e).split(',')[0]
if (e instanceof LockConflict) {
throw new HttpException('The file is locked', HttpStatus.LOCKED)
} else if (e instanceof FileError) {
throw new HttpException(errorMsg, e.httpCode)
}
throw new HttpException(errorMsg, HttpStatus.INTERNAL_SERVER_ERROR)
}
}