Skip to content

Self-hosted personal gallery application with online photo management and albums, supporting Live/Motion Photos, EXIF parsing, geolocation recognition, and an explore map.

chore(upload): add lifecycle logging for chunk and multipart uploads

Timothy Yinf280ef760ef5
diff --git a/.tmp/openlist-upload-reference b/.tmp/openlist-upload-reference
new file mode 160000
index 00000000..ceb64124
--- /dev/null
+++ b/.tmp/openlist-upload-reference
@@ -0,0 +1 @@
+Subproject commit ceb64124a3e266445fe41b0e3bb9476c757d11f8
diff --git a/server/api/photos/chunks/[id].ts b/server/api/photos/chunks/[id].ts
index 60a2dbf8..9ba70da0 100644
--- a/server/api/photos/chunks/[id].ts
+++ b/server/api/photos/chunks/[id].ts
@@ -1,5 +1,6 @@
import { open, stat, rm } from 'node:fs/promises'
import { UPLOAD_CHUNK_SIZE } from '../../../../shared/utils/upload'
+import { logUpload } from '../../../utils/upload-log'
import {
chunkUploads,
chunkHash,
@@ -17,6 +18,12 @@ export default eventHandler(async (event) => {
})
}
if (event.method === 'DELETE') {
+ logUpload('local.cancel.requested', {
+ sessionId: id,
+ key: upload.key,
+ activeParts: upload.activeParts.size,
+ busy: upload.busy,
+ })
await removeChunkUpload(id)
return { ok: true }
}
@@ -27,9 +34,19 @@ export default eventHandler(async (event) => {
upload.busy = event.method === 'POST'
let activeIndex: number | undefined
upload.touched = Date.now()
+ const startedAt = Date.now()
try {
if (event.method === 'POST') {
- if (upload.completed) return { ok: true, key: upload.key }
+ if (upload.completed) {
+ logUpload('local.complete.repeated', { sessionId: id, key: upload.key })
+ return { ok: true, key: upload.key }
+ }
+ logUpload('local.complete.started', {
+ sessionId: id,
+ key: upload.key,
+ receivedParts: upload.hashes.size,
+ size: upload.size,
+ })
if (
upload.hashes.size !== Math.ceil(upload.size / UPLOAD_CHUNK_SIZE) ||
(await stat(upload.filePath)).size !== upload.size
@@ -45,6 +62,12 @@ export default eventHandler(async (event) => {
upload.contentType,
)
upload.completed = true
+ logUpload('local.complete.succeeded', {
+ sessionId: id,
+ key: upload.key,
+ size: upload.size,
+ durationMs: Date.now() - startedAt,
+ })
await rm(upload.filePath, { force: true }).catch(() => {})
return { ok: true, key: upload.key }
}
@@ -70,6 +93,14 @@ export default eventHandler(async (event) => {
UPLOAD_CHUNK_SIZE,
upload.size - index * UPLOAD_CHUNK_SIZE,
)
+ logUpload('local.part.started', {
+ sessionId: id,
+ key: upload.key,
+ partNumber: index + 1,
+ partCount: count,
+ expectedBytes: expected,
+ activeParts: upload.activeParts.size,
+ })
const chunks: Buffer[] = []
let size = 0
// Nitro's dev proxy can provide a cached/web body rather than a live Node
@@ -113,6 +144,13 @@ export default eventHandler(async (event) => {
statusCode: 409,
statusMessage: 'Chunk content mismatch',
})
+ logUpload('local.part.duplicate', {
+ sessionId: id,
+ key: upload.key,
+ partNumber: index + 1,
+ bytes: size,
+ durationMs: Date.now() - startedAt,
+ })
return { ok: true }
}
const file = await open(upload.filePath, 'r+')
@@ -131,7 +169,29 @@ export default eventHandler(async (event) => {
await file.close()
}
upload.hashes.set(index, hash)
+ logUpload('local.part.received', {
+ sessionId: id,
+ key: upload.key,
+ partNumber: index + 1,
+ bytes: size,
+ receivedParts: upload.hashes.size,
+ partCount: count,
+ durationMs: Date.now() - startedAt,
+ })
return { ok: true }
+ } catch (error) {
+ logUpload(
+ 'local.request.failed',
+ {
+ sessionId: id,
+ key: upload.key,
+ method: event.method,
+ partNumber: activeIndex === undefined ? undefined : activeIndex + 1,
+ durationMs: Date.now() - startedAt,
+ },
+ error,
+ )
+ throw error
} finally {
if (event.method === 'POST') upload.busy = false
if (activeIndex !== undefined) upload.activeParts.delete(activeIndex)
diff --git a/server/api/photos/multipart/[id].ts b/server/api/photos/multipart/[id].ts
index e898711e..aee16759 100644
--- a/server/api/photos/multipart/[id].ts
+++ b/server/api/photos/multipart/[id].ts
@@ -1,5 +1,6 @@
import { directUploads, abortDirectUpload } from '~~/server/utils/direct-upload'
import type { UploadPart } from '~~/shared/types/upload'
+import { logUpload } from '../../../utils/upload-log'
export default eventHandler(async (event) => {
const user = await requireUserSession(event)
@@ -11,6 +12,11 @@ export default eventHandler(async (event) => {
statusMessage: 'Upload session not found',
})
if (event.method === 'DELETE') {
+ logUpload('multipart.cancel.requested', {
+ sessionId: id,
+ key: session.key,
+ busy: session.busy,
+ })
await abortDirectUpload(id)
return { ok: true }
}
@@ -19,9 +25,21 @@ export default eventHandler(async (event) => {
throw createError({ statusCode: 410, statusMessage: 'Upload cancelled' })
if (session.busy)
throw createError({ statusCode: 409, statusMessage: 'Upload is busy' })
- if (session.completed) return { ok: true, key: session.key }
+ if (session.completed) {
+ logUpload('multipart.complete.repeated', {
+ sessionId: id,
+ key: session.key,
+ })
+ return { ok: true, key: session.key }
+ }
session.busy = true
session.touched = Date.now()
+ const startedAt = Date.now()
+ logUpload('multipart.complete.started', {
+ sessionId: id,
+ key: session.key,
+ partCount: session.setup.partUrls.length,
+ })
try {
const body = (await readBody(event)) as { parts?: UploadPart[] }
const parts = body?.parts
@@ -45,7 +63,20 @@ export default eventHandler(async (event) => {
throw createError({ statusCode: 410, statusMessage: 'Upload cancelled' })
await session.setup.complete(parts)
session.completed = true
+ logUpload('multipart.complete.succeeded', {
+ sessionId: id,
+ key: session.key,
+ partCount: parts.length,
+ durationMs: Date.now() - startedAt,
+ })
return { ok: true, key: session.key }
+ } catch (error) {
+ logUpload(
+ 'multipart.complete.failed',
+ { sessionId: id, key: session.key, durationMs: Date.now() - startedAt },
+ error,
+ )
+ throw error
} finally {
session.busy = false
session.touched = Date.now()
diff --git a/server/services/storage/providers/s3.ts b/server/services/storage/providers/s3.ts
index 7a373d82..1e73d335 100644
--- a/server/services/storage/providers/s3.ts
+++ b/server/services/storage/providers/s3.ts
@@ -22,6 +22,7 @@ import type {
DirectUploadSetup,
} from '../interfaces'
import { UPLOAD_CHUNK_SIZE } from '../../../../shared/utils/upload'
+import { logUpload } from '../../../utils/upload-log'
const createClient = (config: S3StorageConfig): S3Client => {
if (config.provider !== 's3') {
@@ -238,6 +239,12 @@ export class S3StorageProvider implements StorageProvider {
})
if (size <= UPLOAD_CHUNK_SIZE) return single()
const base = { Bucket: this.config.bucket, Key: key }
+ const startedAt = Date.now()
+ logUpload('s3.multipart.initialize.started', {
+ key,
+ size,
+ bucket: this.config.bucket,
+ })
let uploadId: string | undefined
try {
uploadId = (
@@ -249,6 +256,16 @@ export class S3StorageProvider implements StorageProvider {
)
).UploadId
} catch (error) {
+ logUpload(
+ 's3.multipart.initialize.failed',
+ {
+ key,
+ size,
+ bucket: this.config.bucket,
+ durationMs: Date.now() - startedAt,
+ },
+ error,
+ )
const unsupported = error as {
name?: string
Code?: string
@@ -265,17 +282,34 @@ export class S3StorageProvider implements StorageProvider {
) {
// A provider can allow ordinary PUT uploads while denying multipart
// initiation. Keep the file on the direct path in that case.
- this.logger?.warn(
- `Multipart initialization rejected (${unsupported.name || unsupported.Code || unsupported.$metadata?.httpStatusCode}); falling back to direct PUT`,
- )
+ logUpload('s3.multipart.fallback.single', {
+ key,
+ size,
+ reason:
+ unsupported.name ||
+ unsupported.Code ||
+ String(unsupported.$metadata?.httpStatusCode),
+ })
return single()
}
throw error
}
if (!uploadId) throw new Error('S3 did not return a multipart upload ID')
+ logUpload('s3.multipart.initialize.succeeded', {
+ key,
+ uploadId,
+ durationMs: Date.now() - startedAt,
+ })
const params = { ...base, UploadId: uploadId }
const abort = async () => {
- await this.client.send(new AbortMultipartUploadCommand(params))
+ logUpload('s3.multipart.abort.started', { key, uploadId })
+ try {
+ await this.client.send(new AbortMultipartUploadCommand(params))
+ logUpload('s3.multipart.abort.succeeded', { key, uploadId })
+ } catch (error) {
+ logUpload('s3.multipart.abort.failed', { key, uploadId }, error)
+ throw error
+ }
}
const partSize = Math.max(
UPLOAD_CHUNK_SIZE,
@@ -292,12 +326,24 @@ export class S3StorageProvider implements StorageProvider {
),
),
)
+ logUpload('s3.multipart.parts.signed', {
+ key,
+ uploadId,
+ partSize,
+ partCount: count,
+ })
return {
mode: 'multipart',
partSize,
partUrls,
abort,
complete: async (parts) => {
+ logUpload('s3.multipart.verify.started', {
+ key,
+ uploadId,
+ partCount: count,
+ size,
+ })
// Verify remote part sizes before publishing the object. Presigned
// upload URLs alone do not enforce the declared total file size.
const remote = new Map<number, { size?: number; etag?: string }>()
@@ -329,6 +375,19 @@ export class S3StorageProvider implements StorageProvider {
})
)
throw new Error('S3 multipart upload size or ETag mismatch')
+ for (const [partNumber, part] of remote) {
+ logUpload('s3.multipart.part.verified', {
+ key,
+ uploadId,
+ partNumber,
+ bytes: part.size,
+ })
+ }
+ logUpload('s3.multipart.complete.started', {
+ key,
+ uploadId,
+ partCount: count,
+ })
await this.client.send(
new CompleteMultipartUploadCommand({
...params,
@@ -340,9 +399,16 @@ export class S3StorageProvider implements StorageProvider {
},
}),
)
+ logUpload('s3.multipart.complete.succeeded', {
+ key,
+ uploadId,
+ size,
+ durationMs: Date.now() - startedAt,
+ })
},
}
} catch (error) {
+ logUpload('s3.multipart.signing.failed', { key, uploadId }, error)
await abort().catch(() => {})
throw error
}
diff --git a/server/utils/chunk-upload.ts b/server/utils/chunk-upload.ts
index 76659263..53c4260a 100644
--- a/server/utils/chunk-upload.ts
+++ b/server/utils/chunk-upload.ts
@@ -3,6 +3,8 @@ import { mkdtemp, rm, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import path from 'node:path'
import type { StorageProvider } from '../services/storage/interfaces'
+import { logUpload } from './upload-log'
+import { UPLOAD_CHUNK_SIZE } from '../../shared/utils/upload'
const TTL = 60 * 60 * 1000
export const chunkUploads = new Map<string, ChunkUpload>()
@@ -33,6 +35,12 @@ export async function removeChunkUpload(id: string) {
if (upload.busy || upload.activeParts.size) return
await rm(upload.directory, { recursive: true, force: true })
chunkUploads.delete(id)
+ logUpload('local.session.removed', {
+ sessionId: id,
+ key: upload.key,
+ completed: upload.completed,
+ receivedParts: upload.hashes.size,
+ })
}
export async function createChunkUpload(
@@ -60,6 +68,14 @@ export async function createChunkUpload(
completed: false,
touched: Date.now(),
})
+ logUpload('local.session.created', {
+ sessionId: id,
+ owner,
+ key,
+ size,
+ partSize: UPLOAD_CHUNK_SIZE,
+ partCount: Math.ceil(size / UPLOAD_CHUNK_SIZE),
+ })
return `/api/photos/chunks/${id}`
}
@@ -70,8 +86,13 @@ const cleanup = setInterval(() => {
!upload.activeParts.size &&
Date.now() - upload.touched > TTL
) {
+ logUpload('local.session.expired', { sessionId: id, key: upload.key })
void removeChunkUpload(id).catch((error) =>
- console.error('Upload cleanup failed', error),
+ logUpload(
+ 'local.cleanup.failed',
+ { sessionId: id, key: upload.key },
+ error,
+ ),
)
}
}
diff --git a/server/utils/direct-upload.ts b/server/utils/direct-upload.ts
index 600730b7..5e74ea7d 100644
--- a/server/utils/direct-upload.ts
+++ b/server/utils/direct-upload.ts
@@ -5,6 +5,7 @@ import type {
} from '../services/storage/interfaces'
import type { UploadPlan } from '../../shared/types/upload'
import { createChunkUpload } from './chunk-upload'
+import { logUpload } from './upload-log'
export const directUploads = new Map<
string,
@@ -26,6 +27,11 @@ export async function abortDirectUpload(id: string) {
if (session.busy) return
if (!session.completed) await session.setup.abort()
directUploads.delete(id)
+ logUpload('multipart.session.removed', {
+ sessionId: id,
+ key: session.key,
+ completed: session.completed,
+ })
}
export async function prepareUpload(
@@ -36,9 +42,25 @@ export async function prepareUpload(
provider: StorageProvider,
chunked: boolean,
): Promise<UploadPlan> {
+ logUpload('prepare.started', {
+ provider: provider.config?.provider,
+ owner,
+ key,
+ size,
+ chunked,
+ })
if (provider.prepareDirectUpload && Number.isSafeInteger(size) && size > 0) {
const setup = await provider.prepareDirectUpload(key, size, contentType)
- if (setup?.mode === 'single' || setup?.mode === 'range') return setup
+ if (setup?.mode === 'single' || setup?.mode === 'range') {
+ logUpload('prepare.direct', {
+ provider: provider.config?.provider,
+ key,
+ size,
+ mode: setup.mode,
+ partSize: setup.mode === 'range' ? setup.partSize : undefined,
+ })
+ return setup
+ }
if (setup?.mode === 'multipart') {
const id = randomUUID()
directUploads.set(id, {
@@ -50,6 +72,15 @@ export async function prepareUpload(
completed: false,
touched: Date.now(),
})
+ logUpload('multipart.session.created', {
+ sessionId: id,
+ provider: provider.config?.provider,
+ owner,
+ key,
+ size,
+ partSize: setup.partSize,
+ partCount: setup.partUrls.length,
+ })
return {
mode: 'multipart',
url: `/api/photos/multipart/${id}`,
@@ -59,6 +90,11 @@ export async function prepareUpload(
}
}
if (provider.getSignedUrl) {
+ logUpload('prepare.single', {
+ provider: provider.config?.provider,
+ key,
+ size,
+ })
return {
mode: 'single',
url: await provider.getSignedUrl(key, 3600, { contentType }),
@@ -79,8 +115,17 @@ export async function prepareUpload(
const cleanup = setInterval(() => {
for (const [id, session] of directUploads) {
if (!session.busy && Date.now() - session.touched > 60 * 60_000) {
+ logUpload('multipart.session.expired', {
+ sessionId: id,
+ key: session.key,
+ completed: session.completed,
+ })
void abortDirectUpload(id).catch((error) =>
- console.error('Multipart cleanup failed', error),
+ logUpload(
+ 'multipart.cleanup.failed',
+ { sessionId: id, key: session.key },
+ error,
+ ),
)
}
}
diff --git a/server/utils/upload-log.ts b/server/utils/upload-log.ts
new file mode 100644
index 00000000..b3f3979d
--- /dev/null
+++ b/server/utils/upload-log.ts
@@ -0,0 +1,27 @@
+import { logger } from './logger'
+
+// Log lifecycle metadata only: never URLs, headers, credentials or file bodies.
+export function logUpload(
+ event: string,
+ details: Record<string, string | number | boolean | undefined>,
+ error?: unknown,
+) {
+ if (error !== undefined) {
+ const failure = error as {
+ name?: string
+ Code?: string
+ statusCode?: number
+ statusMessage?: string
+ $metadata?: { httpStatusCode?: number }
+ } | null
+ logger.storage.warn(`[upload] ${event}`, {
+ ...details,
+ errorName: failure?.name,
+ errorCode: failure?.Code,
+ statusCode: failure?.statusCode ?? failure?.$metadata?.httpStatusCode,
+ reason: failure?.statusMessage,
+ })
+ } else {
+ logger.storage.info(`[upload] ${event}`, details)
+ }
+}