Skip to content

Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.

19.7 KB
Raw Blame
import { HttpException, HttpStatus } from '@nestjs/common'
import { WriteStream } from 'fs'
import fse from 'fs-extra'
import mime from 'mime-types'
import crypto from 'node:crypto'
import { createReadStream, createWriteStream, Dirent, statSync } from 'node:fs'
import fs from 'node:fs/promises'
import path from 'node:path'
import { Readable, Transform } from 'node:stream'
import { pipeline } from 'node:stream/promises'
import { toErrorMessage } from '../../../common/errors.js'
import { formatDateISOString } from '../../../common/functions.js'
import { currentTimeStamp, isValidFileName, regExpPreventPathTraversal } from '../../../common/shared.js'
import { DEFAULT_CHECKSUM_ALGORITHM, DEFAULT_HIGH_WATER_MARK, EXTRA_MIMES_TYPE, TEMPORARY_FILE_PREFIX, TEMPORARY_PATH } from '../constants/files.js'
import { SYNC_TEMPORARY_FILE_PREFIX } from '../../sync/constants/sync.js'
import type { FileDBProps } from '../interfaces/file-db-props.interface.js'
import type { FileProps } from '../interfaces/file-props.interface.js'
import type { WriteFromStreamOptions, WriteUploadStreamOptions } from '../interfaces/write-stream.interface.js'
import { FileError } from '../models/file-error.js'
import { maxFileSizeExceededError } from './errors.js'

export function sanitizePath(fPath: string): string {
  return path.normalize(fPath).replace(regExpPreventPathTraversal, '')
}

export function isPathInside(basePath: string, candidatePath: string, allowBasePath = false): boolean {
  // Prevent lexical path traversal and prefix collisions by checking the resolved candidate against the base directory boundary.
  const resolvedBasePath = path.resolve(basePath)
  const resolvedCandidatePath = path.resolve(candidatePath)
  if (resolvedCandidatePath === resolvedBasePath) {
    return allowBasePath
  }
  const basePathPrefix = resolvedBasePath.endsWith(path.sep) ? resolvedBasePath : `${resolvedBasePath}${path.sep}`
  return resolvedCandidatePath.startsWith(basePathPrefix)
}

export function isInternalTemporaryEntry(name: string): boolean {
  return name === TEMPORARY_PATH.STORAGE || name.startsWith(SYNC_TEMPORARY_FILE_PREFIX)
}

export function isInternalTemporaryPath(basePath: string, candidatePath: string): boolean {
  const relativePath = path.relative(path.resolve(basePath), path.resolve(candidatePath))
  if (!relativePath || path.isAbsolute(relativePath) || relativePath.startsWith(`..${path.sep}`) || relativePath === '..') return false
  return relativePath.split(path.sep).some(isInternalTemporaryEntry)
}

export function sanitizeName(name: string): string {
  return name
    .replace(/[/\\]/g, '') // remove slashes
    .replace(/\.\./g, '') // remove '..'
    .replace(/^\s+|[. ]+$/g, '') // trimStart + trimEnd + strip trailing dots
}

export function isValidFileReferenceId(fileId: unknown): fileId is number {
  return typeof fileId === 'number' && Number.isInteger(fileId) && (fileId < 0 || (fileId > 0 && Number.isSafeInteger(fileId)))
}

export function assertValidFileReferenceId(fileId: number | undefined): asserts fileId is number {
  if (!isValidFileReferenceId(fileId)) {
    throw new HttpException('Invalid file id', HttpStatus.BAD_REQUEST)
  }
}

export function checkFileName(fPath: string): string {
  const fName = fileName(fPath)
  try {
    isValidFileName(fName)
    return fName
  } catch {
    throw new FileError(HttpStatus.BAD_REQUEST, 'Forbidden characters')
  }
}

export function isPathExists(rPath: string): Promise<boolean> {
  return fs.access(rPath).then(
    () => true,
    () => false
  )
}

async function existingParentPath(rPath: string): Promise<string> {
  let parentPath = path.dirname(rPath)
  while (!(await isPathExists(parentPath))) {
    const nextParentPath = path.dirname(parentPath)
    if (nextParentPath === parentPath) break
    parentPath = nextParentPath
  }
  return parentPath
}

export async function isCrossDevice(srcPath: string, dstPath: string): Promise<boolean> {
  const [srcStats, dstParentStats] = await Promise.all([fs.lstat(srcPath), existingParentPath(dstPath).then((parentPath) => fs.stat(parentPath))])
  return srcStats.dev !== dstParentStats.dev
}

export async function isPathIsReadable(rPath: string): Promise<boolean> {
  try {
    await fs.access(rPath, fs.constants.R_OK)
  } catch {
    return false
  }
  return true
}

export async function isPathIsWriteable(rPath: string): Promise<boolean> {
  try {
    await fs.access(rPath, fs.constants.W_OK)
  } catch {
    return false
  }
  return true
}

export async function isPathIsDir(rPath: string): Promise<boolean> {
  return (await fs.stat(rPath)).isDirectory()
}

export function fileName(fPath: string): string {
  return path.posix.basename(fPath)
}

export function dirName(fPath: string): string {
  return path.dirname(fPath)
}

export async function fileSize(rPath: string): Promise<number> {
  return (await fs.stat(rPath)).size
}

export function createEmptyFile(rPath: string): Promise<void> {
  return fs.writeFile(rPath, '')
}

export function makeDir(rPath: string, recursive?: boolean): Promise<string | undefined> {
  return fs.mkdir(rPath, { recursive: recursive })
}

const MAX_TEMPORARY_FILE_NAME_BYTES = 255

function temporaryNameSegment(value: string, label: string): string {
  const segment = value.replace(/[^a-zA-Z0-9_-]+/g, '-').replace(/^-+|-+$/g, '')
  if (!segment) throw new Error(`Invalid temporary-file ${label}`)
  return segment
}

function truncateUtf8(value: string, maxBytes: number): string {
  let result = ''
  let length = 0
  for (const character of value) {
    const characterLength = Buffer.byteLength(character)
    if (length + characterLength > maxBytes) break
    result += character
    length += characterLength
  }
  return result
}

function truncateTemporaryBasename(name: string, maxBytes: number): string {
  if (Buffer.byteLength(name) <= maxBytes) return name
  const extension = path.extname(name)
  const extensionLength = Buffer.byteLength(extension)
  if (extension && extensionLength < maxBytes) {
    const stem = path.basename(name, extension)
    return `${truncateUtf8(stem, maxBytes - extensionLength)}${extension}`
  }
  return truncateUtf8(name, maxBytes)
}

export function temporaryFilePrefix(operation: string, executionId: string): string {
  return `${TEMPORARY_FILE_PREFIX}${temporaryNameSegment(operation, 'operation')}-${temporaryNameSegment(executionId, 'execution id')}-`
}

export function temporaryFileName(targetPath: string, operation: string, executionId: string = crypto.randomUUID()): string {
  const prefix = temporaryFilePrefix(operation, executionId)
  const basename = sanitizeName(fileName(targetPath)) || 'file'
  const basenameBytes = MAX_TEMPORARY_FILE_NAME_BYTES - Buffer.byteLength(prefix)
  if (basenameBytes < 1) throw new Error('Temporary-file prefix exceeds the filesystem filename limit')
  return `${prefix}${truncateTemporaryBasename(basename, basenameBytes)}`
}

export function temporaryFilePath(parentPath: string, targetPath: string, operation: string, executionId?: string): string {
  return path.join(parentPath, temporaryFileName(targetPath, operation, executionId))
}

export async function makeTemporaryDirectory(parentPath: string, targetPath: string, operation: string, executionId?: string): Promise<string> {
  await makeDir(parentPath, true)
  const temporaryPath = temporaryFilePath(parentPath, targetPath, operation, executionId)
  await fs.mkdir(temporaryPath)
  return temporaryPath
}

export function getMimeType(fPath: string, isDir: boolean): string {
  if (isDir) {
    return 'directory'
  }
  const extName: string = path.extname(fPath)
  const extraMime = EXTRA_MIMES_TYPE.get(extName)
  if (extraMime !== undefined) {
    return extraMime
  }
  const m = mime.lookup(extName)
  if (m) {
    return m.replace('/', '-')
  }
  return 'file'
}

export function getExtensionWithoutDot(fPath: string): string {
  return path.extname(fPath).slice(1).toLowerCase()
}

export function genEtag(file?: Pick<FileProps, 'size' | 'mtime'> | null, rPath?: string, weakPrefix = true): string {
  if (!file) {
    if (!rPath) throw new Error('File or path are missing')
    const stats = statSync(rPath)
    file = { size: stats.size, mtime: stats.mtime.getTime() }
  }
  const etag = `${file.size.toString(16)}-${file.mtime.toString(16)}`
  return weakPrefix ? `W/"${etag}"` : etag
}

export function genHash(str: string, algo = 'md5', encoding: crypto.BinaryToTextEncoding = 'hex'): string {
  return crypto.createHash(algo).update(str).digest(encoding)
}

export function genUniqHashFromFileDBProps(dbFile: FileDBProps) {
  const dbFileString = `${(Object.keys(dbFile) as (keyof FileDBProps)[])
    .sort()
    .map((k) => `${k}=${String(dbFile[k])}`)
    .join('|')}`
  return genHash(dbFileString, DEFAULT_CHECKSUM_ALGORITHM)
}

export function removeFiles(rPath: string): Promise<void> {
  // if the file does not exist, no error is thrown
  return fs.rm(rPath, { force: true, recursive: true })
}

export async function getProps(rPath: string, fPath?: string, isDir?: boolean): Promise<FileProps> {
  const stats = await fs.stat(rPath)
  const isDirectory = isDir === undefined ? stats.isDirectory() : isDir
  return {
    id: -stats.ino, // use negative number to avoid conflicts with existing database ids
    path: dirName(fPath !== undefined ? fPath : rPath),
    name: fileName(fPath !== undefined ? fPath : rPath),
    isDir: isDirectory,
    size: isDirectory ? 0 : stats.size,
    ctime: stats.birthtime.getTime(),
    mtime: stats.mtime.getTime(),
    mime: getMimeType(rPath, isDirectory)
  }
}

export function touchFile(rPath: string, mtime?: number): Promise<void> {
  if (!mtime) mtime = currentTimeStamp()
  return fs.utimes(rPath, mtime, mtime)
}

const REFLINK_UNAVAILABLE_ERROR_CODES = new Set(['EAGAIN', 'EINVAL', 'ENOSYS', 'ENOTSUP', 'ENOTTY', 'EOPNOTSUPP', 'EXDEV'])

export async function tryReflink(srcPath: string, dstPath: string): Promise<boolean> {
  // A reflink is a copy-on-write clone that shares data blocks and allocates new ones only for modified regions.
  try {
    // FORCE lets the caller distinguish a real reflink from Node's transparent byte-copy fallback.
    await fs.copyFile(srcPath, dstPath, fs.constants.COPYFILE_FICLONE_FORCE)
    return true
  } catch (error) {
    const errorCode = (error as NodeJS.ErrnoException)?.code
    if (errorCode && REFLINK_UNAVAILABLE_ERROR_CODES.has(errorCode)) {
      return false
    }
    throw error
  }
}

async function checkCopyParentPaths(srcPath: string, dstPath: string): Promise<void> {
  const srcStats = await fs.lstat(srcPath, { bigint: true })
  if (!srcStats.isDirectory()) return

  const srcParentPath = path.resolve(path.dirname(srcPath))
  let dstParentPath = path.resolve(path.dirname(dstPath))

  while (dstParentPath !== srcParentPath && dstParentPath !== path.parse(dstParentPath).root) {
    try {
      const dstParentStats = await fs.stat(dstParentPath, { bigint: true })
      if (srcStats.ino === dstParentStats.ino && srcStats.dev === dstParentStats.dev) {
        throw new Error(`Cannot copy '${srcPath}' to a subdirectory of itself, '${dstPath}'.`)
      }
    } catch (error) {
      if ((error as NodeJS.ErrnoException)?.code !== 'ENOENT') throw error
    }
    dstParentPath = path.dirname(dstParentPath)
  }
}

export async function copyFiles(srcPath: string, dstPath: string, overwrite = false, recursive = true, preserveTimestamps = true): Promise<void> {
  /*
    If src is a directory it will copy everything inside of this directory, not the entire directory itself
    If src is a file, dest cannot be a directory
   */
  if (!recursive && (await isPathIsDir(srcPath))) {
    await fs.mkdir(dstPath)
    if (preserveTimestamps) {
      const stat = await fs.stat(srcPath)
      await fs.utimes(dstPath, stat.atime, stat.mtime)
    }
  } else {
    const resolvedSrcPath = path.resolve(srcPath)
    const filter = (entryPath: string): boolean => path.resolve(entryPath) === resolvedSrcPath || !isInternalTemporaryEntry(path.basename(entryPath))

    await checkCopyParentPaths(srcPath, dstPath)
    try {
      await fs.cp(srcPath, dstPath, {
        dereference: false,
        force: overwrite,
        mode: fs.constants.COPYFILE_FICLONE,
        preserveTimestamps,
        recursive,
        // Match fs-extra: keep relative symlink targets relative instead of resolving them against the source directory.
        verbatimSymlinks: true,
        filter
      })
    } catch (error) {
      // TODO: Remove this fallback once https://github.com/nodejs/node/issues/65097 is fixed in every supported Node release.
      // Reproduced with fs.promises.cp on Node 24.20.0; fs-extra remains necessary for this case and for moveFiles' EXDEV fallback.
      if ((error as NodeJS.ErrnoException)?.code !== 'ERR_FS_CP_EINVAL') throw error
      await fse.copy(srcPath, dstPath, { overwrite, preserveTimestamps, filter })
    }
  }
}

export function moveFiles(srcPath: string, dstPath: string, overwrite = false): Promise<void> {
  /*
    If src is a file, dest must be a file and when src is a directory, dest must be a directory
   */
  return fse.move(srcPath, dstPath, { overwrite })
}

export async function checksumFile(filePath: string, alg: string): Promise<string> {
  const hash = crypto.createHash(alg)
  const stream = createReadStream(filePath, { highWaterMark: DEFAULT_HIGH_WATER_MARK })
  await pipeline(stream, hash)
  return hash.digest('hex')
}

export function createSizeLimiter(maxSize: number, maxSizeError: () => Error = maxFileSizeExceededError): (bytes: number) => void {
  let transferred = 0
  return (bytes: number) => {
    transferred += bytes
    if (transferred > maxSize) throw maxSizeError()
  }
}

export function createProgressTransform(
  onProgress?: (bytes: number) => void,
  maxSize?: number,
  maxSizeError: () => Error = maxFileSizeExceededError
): Transform {
  const checkSize = maxSize === undefined ? undefined : createSizeLimiter(maxSize, maxSizeError)
  return new Transform({
    transform(chunk: Buffer, _encoding, callback) {
      try {
        checkSize?.(chunk.length)
        onProgress?.(chunk.length)
      } catch (error) {
        callback(error instanceof Error ? error : new Error(toErrorMessage(error)))
        return
      }
      callback(null, chunk)
    }
  })
}

export function writeFromStream(rPath: string, stream: Readable, options: WriteFromStreamOptions = {}): Promise<void> {
  const { start = 0, signal, accountBytes } = options
  const dst: WriteStream = createWriteStream(rPath, { flags: start ? 'a' : 'w', start: start, highWaterMark: DEFAULT_HIGH_WATER_MARK })
  if (!accountBytes) {
    return pipeline(stream, dst, { signal })
  }
  const accounting = new Transform({
    transform(chunk, _encoding, callback) {
      try {
        accountBytes(chunk.length)
      } catch (error) {
        callback(error instanceof Error ? error : new Error(toErrorMessage(error)))
        return
      }
      callback(null, chunk)
    }
  })
  return pipeline(stream, accounting, dst, { signal })
}

export async function writeFromStreamAndChecksum(
  rPath: string,
  stream: Readable,
  alg: string,
  options: WriteFromStreamOptions = {}
): Promise<string> {
  const { start = 0, signal, accountBytes } = options
  const hash = crypto.createHash(alg)
  if (start) {
    // Seed the hash with the existing prefix so the result covers the complete resumed file.
    const src = createReadStream(rPath, { highWaterMark: DEFAULT_HIGH_WATER_MARK })
    await pipeline(src, hash, { end: false, signal })
  }
  const dst = createWriteStream(rPath, { flags: start ? 'a' : 'w', highWaterMark: DEFAULT_HIGH_WATER_MARK })
  await pipeline(
    stream,
    async function* (source) {
      for await (const chunk of source) {
        accountBytes?.(chunk.length)
        hash.update(chunk)
        yield chunk
      }
    },
    dst,
    { signal }
  )
  hash.end()
  return hash.digest('hex')
}

function uploadWriteOptions({ limiter, signal, onProgress }: WriteUploadStreamOptions): WriteFromStreamOptions {
  return {
    start: limiter.initialFileSize,
    signal,
    accountBytes: (bytes) => {
      // Progress must only include chunks accepted by both the file-size and quota boundaries.
      limiter.consume(bytes)
      onProgress?.(bytes)
    }
  }
}

export function writeUploadFromStream(rPath: string, stream: Readable, options: WriteUploadStreamOptions): Promise<void> {
  return writeFromStream(rPath, stream, uploadWriteOptions(options))
}

export function writeUploadFromStreamAndChecksum(rPath: string, stream: Readable, alg: string, options: WriteUploadStreamOptions): Promise<string> {
  return writeFromStreamAndChecksum(rPath, stream, alg, uploadWriteOptions(options))
}

export function copyFileContent(srcPath: string, dstPath: string): Promise<void> {
  const srcStream = createReadStream(srcPath, { highWaterMark: DEFAULT_HIGH_WATER_MARK })
  return writeFromStream(dstPath, srcStream)
}

export async function walkDir(
  rPath: string,
  onEntry: (entry: Dirent, entryPath: string) => Promise<void> | void,
  errors?: Record<string, string>,
  includeEntry?: (entry: Dirent, entryPath: string) => boolean
): Promise<void> {
  let entries: Dirent[]

  try {
    entries = await fs.readdir(rPath, { withFileTypes: true })
  } catch (e) {
    if (!errors) throw e
    errors[rPath] = toErrorMessage(e)
    return
  }

  for (const entry of entries) {
    const entryPath = path.join(rPath, entry.name)
    if (includeEntry && !includeEntry(entry, entryPath)) continue
    await onEntry(entry, entryPath)
    if (entry.isDirectory()) {
      await walkDir(entryPath, onEntry, errors, includeEntry)
    }
  }
}

export async function dirSize(rPath: string): Promise<[number, any]> {
  let size = 0
  const errors: Record<string, string> = {}

  await walkDir(
    rPath,
    async (entry, entryPath) => {
      if (!entry.isFile()) return
      try {
        size += (await fs.stat(entryPath)).size
      } catch (e) {
        errors[entryPath] = toErrorMessage(e)
      }
    },
    errors
  )
  return [size, errors]
}

export async function dirListFileNames(rPath: string): Promise<string[]> {
  return (await fs.readdir(rPath)).map((path: string) => fileName(path))
}

export async function dirHasChildren(rPath: string, mustContainsDirs = true): Promise<boolean> {
  for await (const file of await fs.opendir(rPath)) {
    if (mustContainsDirs) {
      if (file.isDirectory()) return true
    } else {
      return true
    }
  }
  return false
}

export async function uniqueFilePathFromDir(rPath: string): Promise<string> {
  if (await isPathExists(rPath)) {
    const parentDir = path.dirname(rPath)
    const extension = path.extname(rPath)
    const nameWithoutExtension = path.basename(rPath, extension)
    let count = 1
    while (await isPathExists(path.join(parentDir, `${nameWithoutExtension} (${count})${extension}`))) {
      count++
    }
    return path.join(parentDir, `${nameWithoutExtension} (${count})${extension}`)
  }
  return rPath
}

export async function uniqueDatedFilePath(rPath: string, knownIsDir?: boolean): Promise<{ isDir: boolean; path: string }> {
  const date = formatDateISOString(new Date())
  const isDir = knownIsDir ?? (await isPathIsDir(rPath))
  const extension = isDir ? '' : path.extname(rPath)
  const nameWithoutExtension = path.basename(rPath, extension)
  const datedName = `${nameWithoutExtension}-${date}`
  const parentDir = path.dirname(rPath)
  let candidate = path.join(parentDir, `${datedName}${extension}`)
  let count = 1
  while (await isPathExists(candidate)) {
    candidate = path.join(parentDir, `${datedName} (${count})${extension}`)
    count++
  }
  return { isDir, path: candidate }
}

export async function checkExternalPath(rPath: string) {
  if (!(await isPathExists(rPath))) {
    throw new FileError(HttpStatus.NOT_FOUND, 'The location does not exist')
  }
  if (!(await isPathIsReadable(rPath))) {
    throw new FileError(HttpStatus.NOT_ACCEPTABLE, 'The location is not readable')
  }
  if (!(await isPathIsWriteable(rPath))) {
    throw new FileError(HttpStatus.NOT_ACCEPTABLE, 'The location is not writeable')
  }
}