Self-hosted personal gallery application with online photo management and albums, supporting Live/Motion Photos, EXIF parsing, geolocation recognition, and an explore map.
TypeScript albumsexif-extractiongeocodinglive-photomotion-photophoto-galleryphoto-sharingwebgl-viewer
chore(upload): add lifecycle logging for chunk and multipart uploads
Timothy Yinf280ef760ef5
diff --git a/.tmp/openlist-upload-reference b/.tmp/openlist-upload-referencenew file mode 160000index 00000000..ceb64124--- /dev/null+++ b/.tmp/openlist-upload-reference@@ -0,0 +1 @@+Subproject commit ceb64124a3e266445fe41b0e3bb9476c757d11f8diff --git a/server/api/photos/chunks/[id].ts b/server/api/photos/chunks/[id].tsindex 60a2dbf8..9ba70da0 100644--- a/server/api/photos/chunks/[id].ts+++ b/server/api/photos/chunks/[id].ts@@ -1,5 +1,6 @@import { open, stat, rm } from 'node:fs/promises'import { UPLOAD_CHUNK_SIZE } from '../../../../shared/utils/upload'+import { logUpload } from '../../../utils/upload-log'import {chunkUploads,chunkHash,@@ -17,6 +18,12 @@ export default eventHandler(async (event) => {})}if (event.method === 'DELETE') {+ logUpload('local.cancel.requested', {+ sessionId: id,+ key: upload.key,+ activeParts: upload.activeParts.size,+ busy: upload.busy,+ })await removeChunkUpload(id)return { ok: true }}@@ -27,9 +34,19 @@ export default eventHandler(async (event) => {upload.busy = event.method === 'POST'let activeIndex: number | undefinedupload.touched = Date.now()+ const startedAt = Date.now()try {if (event.method === 'POST') {- if (upload.completed) return { ok: true, key: upload.key }+ if (upload.completed) {+ logUpload('local.complete.repeated', { sessionId: id, key: upload.key })+ return { ok: true, key: upload.key }+ }+ logUpload('local.complete.started', {+ sessionId: id,+ key: upload.key,+ receivedParts: upload.hashes.size,+ size: upload.size,+ })if (upload.hashes.size !== Math.ceil(upload.size / UPLOAD_CHUNK_SIZE) ||(await stat(upload.filePath)).size !== upload.size@@ -45,6 +62,12 @@ export default eventHandler(async (event) => {upload.contentType,)upload.completed = true+ logUpload('local.complete.succeeded', {+ sessionId: id,+ key: upload.key,+ size: upload.size,+ durationMs: Date.now() - startedAt,+ })await rm(upload.filePath, { force: true }).catch(() => {})return { ok: true, key: upload.key }}@@ -70,6 +93,14 @@ export default eventHandler(async (event) => {UPLOAD_CHUNK_SIZE,upload.size - index * UPLOAD_CHUNK_SIZE,)+ logUpload('local.part.started', {+ sessionId: id,+ key: upload.key,+ partNumber: index + 1,+ partCount: count,+ expectedBytes: expected,+ activeParts: upload.activeParts.size,+ })const chunks: Buffer[] = []let size = 0// Nitro's dev proxy can provide a cached/web body rather than a live Node@@ -113,6 +144,13 @@ export default eventHandler(async (event) => {statusCode: 409,statusMessage: 'Chunk content mismatch',})+ logUpload('local.part.duplicate', {+ sessionId: id,+ key: upload.key,+ partNumber: index + 1,+ bytes: size,+ durationMs: Date.now() - startedAt,+ })return { ok: true }}const file = await open(upload.filePath, 'r+')@@ -131,7 +169,29 @@ export default eventHandler(async (event) => {await file.close()}upload.hashes.set(index, hash)+ logUpload('local.part.received', {+ sessionId: id,+ key: upload.key,+ partNumber: index + 1,+ bytes: size,+ receivedParts: upload.hashes.size,+ partCount: count,+ durationMs: Date.now() - startedAt,+ })return { ok: true }+ } catch (error) {+ logUpload(+ 'local.request.failed',+ {+ sessionId: id,+ key: upload.key,+ method: event.method,+ partNumber: activeIndex === undefined ? undefined : activeIndex + 1,+ durationMs: Date.now() - startedAt,+ },+ error,+ )+ throw error} finally {if (event.method === 'POST') upload.busy = falseif (activeIndex !== undefined) upload.activeParts.delete(activeIndex)diff --git a/server/api/photos/multipart/[id].ts b/server/api/photos/multipart/[id].tsindex e898711e..aee16759 100644--- a/server/api/photos/multipart/[id].ts+++ b/server/api/photos/multipart/[id].ts@@ -1,5 +1,6 @@import { directUploads, abortDirectUpload } from '~~/server/utils/direct-upload'import type { UploadPart } from '~~/shared/types/upload'+import { logUpload } from '../../../utils/upload-log'export default eventHandler(async (event) => {const user = await requireUserSession(event)@@ -11,6 +12,11 @@ export default eventHandler(async (event) => {statusMessage: 'Upload session not found',})if (event.method === 'DELETE') {+ logUpload('multipart.cancel.requested', {+ sessionId: id,+ key: session.key,+ busy: session.busy,+ })await abortDirectUpload(id)return { ok: true }}@@ -19,9 +25,21 @@ export default eventHandler(async (event) => {throw createError({ statusCode: 410, statusMessage: 'Upload cancelled' })if (session.busy)throw createError({ statusCode: 409, statusMessage: 'Upload is busy' })- if (session.completed) return { ok: true, key: session.key }+ if (session.completed) {+ logUpload('multipart.complete.repeated', {+ sessionId: id,+ key: session.key,+ })+ return { ok: true, key: session.key }+ }session.busy = truesession.touched = Date.now()+ const startedAt = Date.now()+ logUpload('multipart.complete.started', {+ sessionId: id,+ key: session.key,+ partCount: session.setup.partUrls.length,+ })try {const body = (await readBody(event)) as { parts?: UploadPart[] }const parts = body?.parts@@ -45,7 +63,20 @@ export default eventHandler(async (event) => {throw createError({ statusCode: 410, statusMessage: 'Upload cancelled' })await session.setup.complete(parts)session.completed = true+ logUpload('multipart.complete.succeeded', {+ sessionId: id,+ key: session.key,+ partCount: parts.length,+ durationMs: Date.now() - startedAt,+ })return { ok: true, key: session.key }+ } catch (error) {+ logUpload(+ 'multipart.complete.failed',+ { sessionId: id, key: session.key, durationMs: Date.now() - startedAt },+ error,+ )+ throw error} finally {session.busy = falsesession.touched = Date.now()diff --git a/server/services/storage/providers/s3.ts b/server/services/storage/providers/s3.tsindex 7a373d82..1e73d335 100644--- a/server/services/storage/providers/s3.ts+++ b/server/services/storage/providers/s3.ts@@ -22,6 +22,7 @@ import type {DirectUploadSetup,} from '../interfaces'import { UPLOAD_CHUNK_SIZE } from '../../../../shared/utils/upload'+import { logUpload } from '../../../utils/upload-log'const createClient = (config: S3StorageConfig): S3Client => {if (config.provider !== 's3') {@@ -238,6 +239,12 @@ export class S3StorageProvider implements StorageProvider {})if (size <= UPLOAD_CHUNK_SIZE) return single()const base = { Bucket: this.config.bucket, Key: key }+ const startedAt = Date.now()+ logUpload('s3.multipart.initialize.started', {+ key,+ size,+ bucket: this.config.bucket,+ })let uploadId: string | undefinedtry {uploadId = (@@ -249,6 +256,16 @@ export class S3StorageProvider implements StorageProvider {)).UploadId} catch (error) {+ logUpload(+ 's3.multipart.initialize.failed',+ {+ key,+ size,+ bucket: this.config.bucket,+ durationMs: Date.now() - startedAt,+ },+ error,+ )const unsupported = error as {name?: stringCode?: string@@ -265,17 +282,34 @@ export class S3StorageProvider implements StorageProvider {) {// A provider can allow ordinary PUT uploads while denying multipart// initiation. Keep the file on the direct path in that case.- this.logger?.warn(- `Multipart initialization rejected (${unsupported.name || unsupported.Code || unsupported.$metadata?.httpStatusCode}); falling back to direct PUT`,- )+ logUpload('s3.multipart.fallback.single', {+ key,+ size,+ reason:+ unsupported.name ||+ unsupported.Code ||+ String(unsupported.$metadata?.httpStatusCode),+ })return single()}throw error}if (!uploadId) throw new Error('S3 did not return a multipart upload ID')+ logUpload('s3.multipart.initialize.succeeded', {+ key,+ uploadId,+ durationMs: Date.now() - startedAt,+ })const params = { ...base, UploadId: uploadId }const abort = async () => {- await this.client.send(new AbortMultipartUploadCommand(params))+ logUpload('s3.multipart.abort.started', { key, uploadId })+ try {+ await this.client.send(new AbortMultipartUploadCommand(params))+ logUpload('s3.multipart.abort.succeeded', { key, uploadId })+ } catch (error) {+ logUpload('s3.multipart.abort.failed', { key, uploadId }, error)+ throw error+ }}const partSize = Math.max(UPLOAD_CHUNK_SIZE,@@ -292,12 +326,24 @@ export class S3StorageProvider implements StorageProvider {),),)+ logUpload('s3.multipart.parts.signed', {+ key,+ uploadId,+ partSize,+ partCount: count,+ })return {mode: 'multipart',partSize,partUrls,abort,complete: async (parts) => {+ logUpload('s3.multipart.verify.started', {+ key,+ uploadId,+ partCount: count,+ size,+ })// Verify remote part sizes before publishing the object. Presigned// upload URLs alone do not enforce the declared total file size.const remote = new Map<number, { size?: number; etag?: string }>()@@ -329,6 +375,19 @@ export class S3StorageProvider implements StorageProvider {}))throw new Error('S3 multipart upload size or ETag mismatch')+ for (const [partNumber, part] of remote) {+ logUpload('s3.multipart.part.verified', {+ key,+ uploadId,+ partNumber,+ bytes: part.size,+ })+ }+ logUpload('s3.multipart.complete.started', {+ key,+ uploadId,+ partCount: count,+ })await this.client.send(new CompleteMultipartUploadCommand({...params,@@ -340,9 +399,16 @@ export class S3StorageProvider implements StorageProvider {},}),)+ logUpload('s3.multipart.complete.succeeded', {+ key,+ uploadId,+ size,+ durationMs: Date.now() - startedAt,+ })},}} catch (error) {+ logUpload('s3.multipart.signing.failed', { key, uploadId }, error)await abort().catch(() => {})throw error}diff --git a/server/utils/chunk-upload.ts b/server/utils/chunk-upload.tsindex 76659263..53c4260a 100644--- a/server/utils/chunk-upload.ts+++ b/server/utils/chunk-upload.ts@@ -3,6 +3,8 @@ import { mkdtemp, rm, writeFile } from 'node:fs/promises'import { tmpdir } from 'node:os'import path from 'node:path'import type { StorageProvider } from '../services/storage/interfaces'+import { logUpload } from './upload-log'+import { UPLOAD_CHUNK_SIZE } from '../../shared/utils/upload'const TTL = 60 * 60 * 1000export const chunkUploads = new Map<string, ChunkUpload>()@@ -33,6 +35,12 @@ export async function removeChunkUpload(id: string) {if (upload.busy || upload.activeParts.size) returnawait rm(upload.directory, { recursive: true, force: true })chunkUploads.delete(id)+ logUpload('local.session.removed', {+ sessionId: id,+ key: upload.key,+ completed: upload.completed,+ receivedParts: upload.hashes.size,+ })}export async function createChunkUpload(@@ -60,6 +68,14 @@ export async function createChunkUpload(completed: false,touched: Date.now(),})+ logUpload('local.session.created', {+ sessionId: id,+ owner,+ key,+ size,+ partSize: UPLOAD_CHUNK_SIZE,+ partCount: Math.ceil(size / UPLOAD_CHUNK_SIZE),+ })return `/api/photos/chunks/${id}`}@@ -70,8 +86,13 @@ const cleanup = setInterval(() => {!upload.activeParts.size &&Date.now() - upload.touched > TTL) {+ logUpload('local.session.expired', { sessionId: id, key: upload.key })void removeChunkUpload(id).catch((error) =>- console.error('Upload cleanup failed', error),+ logUpload(+ 'local.cleanup.failed',+ { sessionId: id, key: upload.key },+ error,+ ),)}}diff --git a/server/utils/direct-upload.ts b/server/utils/direct-upload.tsindex 600730b7..5e74ea7d 100644--- a/server/utils/direct-upload.ts+++ b/server/utils/direct-upload.ts@@ -5,6 +5,7 @@ import type {} from '../services/storage/interfaces'import type { UploadPlan } from '../../shared/types/upload'import { createChunkUpload } from './chunk-upload'+import { logUpload } from './upload-log'export const directUploads = new Map<string,@@ -26,6 +27,11 @@ export async function abortDirectUpload(id: string) {if (session.busy) returnif (!session.completed) await session.setup.abort()directUploads.delete(id)+ logUpload('multipart.session.removed', {+ sessionId: id,+ key: session.key,+ completed: session.completed,+ })}export async function prepareUpload(@@ -36,9 +42,25 @@ export async function prepareUpload(provider: StorageProvider,chunked: boolean,): Promise<UploadPlan> {+ logUpload('prepare.started', {+ provider: provider.config?.provider,+ owner,+ key,+ size,+ chunked,+ })if (provider.prepareDirectUpload && Number.isSafeInteger(size) && size > 0) {const setup = await provider.prepareDirectUpload(key, size, contentType)- if (setup?.mode === 'single' || setup?.mode === 'range') return setup+ if (setup?.mode === 'single' || setup?.mode === 'range') {+ logUpload('prepare.direct', {+ provider: provider.config?.provider,+ key,+ size,+ mode: setup.mode,+ partSize: setup.mode === 'range' ? setup.partSize : undefined,+ })+ return setup+ }if (setup?.mode === 'multipart') {const id = randomUUID()directUploads.set(id, {@@ -50,6 +72,15 @@ export async function prepareUpload(completed: false,touched: Date.now(),})+ logUpload('multipart.session.created', {+ sessionId: id,+ provider: provider.config?.provider,+ owner,+ key,+ size,+ partSize: setup.partSize,+ partCount: setup.partUrls.length,+ })return {mode: 'multipart',url: `/api/photos/multipart/${id}`,@@ -59,6 +90,11 @@ export async function prepareUpload(}}if (provider.getSignedUrl) {+ logUpload('prepare.single', {+ provider: provider.config?.provider,+ key,+ size,+ })return {mode: 'single',url: await provider.getSignedUrl(key, 3600, { contentType }),@@ -79,8 +115,17 @@ export async function prepareUpload(const cleanup = setInterval(() => {for (const [id, session] of directUploads) {if (!session.busy && Date.now() - session.touched > 60 * 60_000) {+ logUpload('multipart.session.expired', {+ sessionId: id,+ key: session.key,+ completed: session.completed,+ })void abortDirectUpload(id).catch((error) =>- console.error('Multipart cleanup failed', error),+ logUpload(+ 'multipart.cleanup.failed',+ { sessionId: id, key: session.key },+ error,+ ),)}}diff --git a/server/utils/upload-log.ts b/server/utils/upload-log.tsnew file mode 100644index 00000000..b3f3979d--- /dev/null+++ b/server/utils/upload-log.ts@@ -0,0 +1,27 @@+import { logger } from './logger'++// Log lifecycle metadata only: never URLs, headers, credentials or file bodies.+export function logUpload(+ event: string,+ details: Record<string, string | number | boolean | undefined>,+ error?: unknown,+) {+ if (error !== undefined) {+ const failure = error as {+ name?: string+ Code?: string+ statusCode?: number+ statusMessage?: string+ $metadata?: { httpStatusCode?: number }+ } | null+ logger.storage.warn(`[upload] ${event}`, {+ ...details,+ errorName: failure?.name,+ errorCode: failure?.Code,+ statusCode: failure?.statusCode ?? failure?.$metadata?.httpStatusCode,+ reason: failure?.statusMessage,+ })+ } else {+ logger.storage.info(`[upload] ${event}`, details)+ }+}