Skip to content

Self-hosted personal gallery application with online photo management and albums, supporting Live/Motion Photos, EXIF parsing, geolocation recognition, and an explore map.

13.6 KB
Raw Blame
import type { S3StorageConfig } from '@chronoframe/contracts/storage'
import type { StorageLogger } from '../logger.ts'
import type { _Object, S3ClientConfig } from '@aws-sdk/client-s3'
import {
  DeleteObjectCommand,
  GetObjectCommand,
  HeadObjectCommand,
  ListObjectsCommand,
  PutObjectCommand,
  S3Client,
  CreateMultipartUploadCommand,
  UploadPartCommand,
  CompleteMultipartUploadCommand,
  AbortMultipartUploadCommand,
  ListPartsCommand,
} from '@aws-sdk/client-s3'
import { getSignedUrl } from '@aws-sdk/s3-request-presigner'
import { createReadStream } from 'node:fs'
import { stat } from 'node:fs/promises'
import type {
  StorageObject,
  StorageProvider,
  UploadOptions,
  DirectUploadSetup,
} from '../interfaces.ts'
import { UPLOAD_CHUNK_SIZE } from '@chronoframe/contracts/upload'
import { logUpload } from '../upload-log.ts'
import { canFallbackToSingleUpload } from '../multipart-fallback.ts'

const createClient = (config: S3StorageConfig): S3Client => {
  if (config.provider !== 's3') {
    throw new Error('Invalid provider for S3 client creation')
  }

  const { accessKeyId, secretAccessKey, region, endpoint } = config
  if (!accessKeyId || !secretAccessKey) {
    throw new Error('Missing required accessKeyId or secretAccessKey')
  }

  const clientConfig: S3ClientConfig = {
    endpoint,
    region,
    forcePathStyle: config.forcePathStyle,
    responseChecksumValidation: 'WHEN_REQUIRED',
    requestChecksumCalculation: 'WHEN_REQUIRED',
    credentials: {
      accessKeyId,
      secretAccessKey,
    },
  }

  return new S3Client(clientConfig)
}

const convertToStorageObject = (s3object: _Object): StorageObject => {
  return {
    key: s3object.Key || '',
    size: s3object.Size,
    lastModified: s3object.LastModified,
    etag: s3object.ETag,
  }
}

export class S3StorageProvider implements StorageProvider {
  config: S3StorageConfig
  private logger?: StorageLogger
  private client: S3Client

  constructor(config: S3StorageConfig, logger?: StorageLogger) {
    this.config = config
    this.logger = logger
    this.client = createClient(config)
  }

  async create(
    key: string,
    data: Buffer,
    contentType?: string,
  ): Promise<StorageObject> {
    try {
      const configuredPrefix = this.config.prefix ?? ''
      let start = 0,
        end = configuredPrefix.length,
        keyStart = 0
      while (start < end && configuredPrefix[start] === '/') start++
      while (end > start && configuredPrefix[end - 1] === '/') end--
      while (keyStart < key.length && key[keyStart] === '/') keyStart++
      const prefix = configuredPrefix.slice(start, end),
        cleanKey = key.slice(keyStart)
      const absoluteKey =
        prefix && !cleanKey.startsWith(prefix + '/')
          ? prefix + '/' + cleanKey
          : cleanKey
      const cmd = new PutObjectCommand({
        Bucket: this.config.bucket,
        Key: absoluteKey,
        Body: data,
        ContentType: contentType || 'application/octet-stream',
      })

      const resp = await this.client.send(cmd)

      this.logger?.success(`Created object with key: ${absoluteKey}`)

      return {
        key: absoluteKey,
        size: data.length,
        lastModified: new Date(),
        etag: resp.ETag,
      }
    } catch (error) {
      this.logger?.error(`Failed to create object with key: ${key}`, error)
      throw error
    }
  }

  async delete(key: string): Promise<void> {
    try {
      const cmd = new DeleteObjectCommand({
        Bucket: this.config.bucket,
        Key: key,
      })

      await this.client.send(cmd)
      this.logger?.success(`Deleted object with key: ${key}`)
    } catch (error) {
      this.logger?.error(`Failed to delete object with key: ${key}`, error)
      throw error
    }
  }

  async createFromFile(
    key: string,
    filePath: string,
    contentType: string,
  ): Promise<StorageObject> {
    const { size } = await stat(filePath)
    const body = createReadStream(filePath)
    try {
      const result = await this.client.send(
        new PutObjectCommand({
          Bucket: this.config.bucket,
          Key: key,
          Body: body,
          ContentLength: size,
          ContentType: contentType,
        }),
      )
      return { key, size, etag: result.ETag }
    } finally {
      body.destroy()
    }
  }

  async get(key: string): Promise<Buffer | null> {
    try {
      const cmd = new GetObjectCommand({
        Bucket: this.config.bucket,
        Key: key,
      })

      const resp = await this.client.send(cmd)

      if (!resp.Body) {
        return null
      }

      if (resp.Body instanceof Buffer) {
        return resp.Body
      }

      const chunks: Uint8Array[] = []
      const stream = resp.Body as NodeJS.ReadableStream

      return new Promise<Buffer>((resolve, reject) => {
        stream.on('data', (chunk: Uint8Array) => {
          chunks.push(chunk)
        })

        stream.on('end', () => {
          resolve(Buffer.concat(chunks))
        })

        stream.on('error', (err) => {
          reject(err)
        })
      })
    } catch {
      return null
    }
  }

  getPublicUrl(key: string): string {
    const { cdnUrl, bucket, region, endpoint } = this.config

    // CDN URL
    if (cdnUrl) {
      return `${cdnUrl.replace(/\/$/, '')}/${key}`
    }

    // Default AWS S3 endpoint
    if (!endpoint) {
      return `https://${bucket}.s3.${region}.amazonaws.com/${key}`
    }
    const hostname = new URL(endpoint).hostname.toLowerCase()
    if (hostname === 'amazonaws.com' || hostname.endsWith('.amazonaws.com')) {
      return `https://${bucket}.s3.${region}.amazonaws.com/${key}`
    }

    // Alibaba Cloud OSS
    if (hostname === 'aliyuncs.com' || hostname.endsWith('.aliyuncs.com')) {
      const baseUrl = endpoint.replace(/\/$/, '')
      if (baseUrl.indexOf('//') === -1) {
        throw new Error('Invalid endpoint URL')
      }
      const protocol = baseUrl.split('//')[0]
      const remainder = baseUrl.split('//')[1]
      return `${protocol}//${bucket}.${remainder}/${key}`
    }

    // Custom endpoint
    return `${endpoint.replace(/\/$/, '')}/${bucket}/${key}`
  }

  async getSignedUrl(
    key: string,
    expiresIn: number = 3600,
    options?: UploadOptions,
  ): Promise<string> {
    const cmd = new PutObjectCommand({
      Bucket: this.config.bucket,
      Key: key,
      ContentType: options?.contentType || 'application/octet-stream',
    })

    const url = await getSignedUrl(this.client, cmd, {
      expiresIn,
      // 为了更好的 CORS 支持,添加一些额外参数
      unhoistableHeaders: new Set(['Content-Type']),
    })
    return url
  }

  async prepareDirectUpload(
    key: string,
    size: number,
    contentType: string,
  ): Promise<DirectUploadSetup> {
    const single = async (): Promise<DirectUploadSetup> => ({
      mode: 'single',
      url: await this.getSignedUrl(key, 3600, { contentType }),
    })
    if (size <= UPLOAD_CHUNK_SIZE) return single()
    const base = { Bucket: this.config.bucket, Key: key }
    const startedAt = Date.now()
    logUpload(this.logger, 's3.multipart.initialize.started', {
      key,
      size,
      bucket: this.config.bucket,
    })
    let uploadId: string | undefined
    try {
      uploadId = (
        await this.client.send(
          new CreateMultipartUploadCommand({
            ...base,
            ContentType: contentType,
          }),
        )
      ).UploadId
    } catch (error) {
      logUpload(
        this.logger,
        's3.multipart.initialize.failed',
        {
          key,
          size,
          bucket: this.config.bucket,
          durationMs: Date.now() - startedAt,
        },
        error,
      )
      const unsupported = error as {
        name?: string
        Code?: string
        $metadata?: { httpStatusCode?: number }
      }
      if (canFallbackToSingleUpload(error)) {
        // A provider can allow ordinary PUT uploads while denying multipart
        // initiation. Keep the file on the direct path in that case.
        logUpload(this.logger, 's3.multipart.fallback.single', {
          key,
          size,
          reason:
            unsupported.name ||
            unsupported.Code ||
            String(unsupported.$metadata?.httpStatusCode),
        })
        return single()
      }
      throw error
    }
    if (!uploadId) throw new Error('S3 did not return a multipart upload ID')
    logUpload(this.logger, 's3.multipart.initialize.succeeded', {
      key,
      uploadId,
      durationMs: Date.now() - startedAt,
    })
    const params = { ...base, UploadId: uploadId }
    const abort = async () => {
      logUpload(this.logger, 's3.multipart.abort.started', { key, uploadId })
      try {
        await this.client.send(new AbortMultipartUploadCommand(params))
        logUpload(this.logger, 's3.multipart.abort.succeeded', {
          key,
          uploadId,
        })
      } catch (error) {
        logUpload(
          this.logger,
          's3.multipart.abort.failed',
          { key, uploadId },
          error,
        )
        throw error
      }
    }
    const partSize = Math.max(
      UPLOAD_CHUNK_SIZE,
      Math.ceil(size / 10_000 / 1024 / 1024) * 1024 * 1024,
    )
    const count = Math.ceil(size / partSize)
    try {
      const partUrls = await Promise.all(
        Array.from({ length: count }, (_, index) =>
          getSignedUrl(
            this.client,
            new UploadPartCommand({ ...params, PartNumber: index + 1 }),
            { expiresIn: 3600 },
          ),
        ),
      )
      logUpload(this.logger, 's3.multipart.parts.signed', {
        key,
        uploadId,
        partSize,
        partCount: count,
      })
      return {
        mode: 'multipart',
        partSize,
        partUrls,
        abort,
        complete: async (parts) => {
          logUpload(this.logger, 's3.multipart.verify.started', {
            key,
            uploadId,
            partCount: count,
            size,
          })
          // Verify remote part sizes before publishing the object. Presigned
          // upload URLs alone do not enforce the declared total file size.
          const remote = new Map<number, { size?: number; etag?: string }>()
          let marker: string | undefined
          do {
            const result = await this.client.send(
              new ListPartsCommand({ ...params, PartNumberMarker: marker }),
            )
            for (const part of result.Parts || []) {
              if (part.PartNumber !== undefined)
                remote.set(part.PartNumber, {
                  size: part.Size,
                  etag: part.ETag,
                })
            }
            marker = result.IsTruncated
              ? result.NextPartNumberMarker
              : undefined
          } while (marker)
          if (
            remote.size !== count ||
            parts.length !== count ||
            new Set(parts.map((part) => part.partNumber)).size !== count ||
            parts.some((part) => {
              const uploaded = remote.get(part.partNumber)
              return (
                !Number.isInteger(part.partNumber) ||
                part.partNumber < 1 ||
                part.partNumber > count ||
                uploaded?.etag !== part.etag ||
                uploaded?.size !==
                  Math.min(partSize, size - (part.partNumber - 1) * partSize)
              )
            })
          )
            throw new Error('S3 multipart upload size or ETag mismatch')
          for (const [partNumber, part] of remote) {
            logUpload(this.logger, 's3.multipart.part.verified', {
              key,
              uploadId,
              partNumber,
              bytes: part.size,
            })
          }
          logUpload(this.logger, 's3.multipart.complete.started', {
            key,
            uploadId,
            partCount: count,
          })
          await this.client.send(
            new CompleteMultipartUploadCommand({
              ...params,
              MultipartUpload: {
                Parts: parts.map((part) => ({
                  PartNumber: part.partNumber,
                  ETag: part.etag,
                })),
              },
            }),
          )
          logUpload(this.logger, 's3.multipart.complete.succeeded', {
            key,
            uploadId,
            size,
            durationMs: Date.now() - startedAt,
          })
        },
      }
    } catch (error) {
      logUpload(
        this.logger,
        's3.multipart.signing.failed',
        { key, uploadId },
        error,
      )
      await abort().catch(() => {})
      throw error
    }
  }

  async getFileMeta(key: string): Promise<StorageObject | null> {
    try {
      const cmd = new HeadObjectCommand({
        Bucket: this.config.bucket,
        Key: key,
      })

      const resp = await this.client.send(cmd)

      return {
        key,
        size: resp.ContentLength,
        lastModified: resp.LastModified,
        etag: resp.ETag,
      }
    } catch (error) {
      if ((error as any).$metadata?.httpStatusCode === 404) {
        return null
      }
      this.logger?.error(`Failed to get metadata for key: ${key}`, error)
      throw error
    }
  }

  async listAll(): Promise<StorageObject[]> {
    const cmd = new ListObjectsCommand({
      Bucket: this.config.bucket,
      Prefix: this.config.prefix,
      MaxKeys: this.config.maxKeys,
    })

    const resp = await this.client.send(cmd)
    this.logger?.log(resp.Contents?.map(convertToStorageObject))
    return resp.Contents?.map(convertToStorageObject) || []
  }

  async listImages(): Promise<StorageObject[]> {
    const cmd = new ListObjectsCommand({
      Bucket: this.config.bucket,
      Prefix: this.config.prefix,
      MaxKeys: this.config.maxKeys,
    })

    const resp = await this.client.send(cmd)
    // TODO: filter supported image format
    return resp.Contents?.map(convertToStorageObject) || []
  }
}