Skip to content

ActivityPub-federated video streaming platform using P2P directly in your web browser

Support short uuids when parsing logs

Chocobozzzbad768ddfa43
diff --git a/packages/tests/src/api/server/logs.ts b/packages/tests/src/api/server/logs.ts
index 10a2d23ff0..cf6fb8d7a9 100644
--- a/packages/tests/src/api/server/logs.ts
+++ b/packages/tests/src/api/server/logs.ts
@@ -94,7 +94,7 @@ describe('Test logs', function () {
it('Should filter by tag', async function () {
const now = new Date()
- const { uuid } = await server.videos.upload({ attributes: { name: 'video 6' } })
+ const { uuid, shortUUID } = await server.videos.upload({ attributes: { name: 'video 6' } })
await waitJobs([ server ])
{
@@ -102,8 +102,8 @@ describe('Test logs', function () {
expect(body).to.have.lengthOf(0)
}
- {
- const body = await logsCommand.getLogs({ startDate: now, level: 'debug', tagsOneOf: [ uuid ] })
+ for (const tag of [ uuid, shortUUID ]) {
+ const body = await logsCommand.getLogs({ startDate: now, level: 'debug', tagsOneOf: [ tag ] })
expect(body).to.not.have.lengthOf(0)
for (const line of body) {
diff --git a/server/core/controllers/api/server/logs.ts b/server/core/controllers/api/server/logs.ts
index 7d7888ea42..33da50575b 100644
--- a/server/core/controllers/api/server/logs.ts
+++ b/server/core/controllers/api/server/logs.ts
@@ -1,10 +1,10 @@
-import express from 'express'
-import { readdir, readFile } from 'fs/promises'
-import { join } from 'path'
import { pick } from '@peertube/peertube-core-utils'
import { ClientLogCreate, HttpStatusCode, ServerLogLevel, UserRight } from '@peertube/peertube-models'
import { isArray } from '@server/helpers/custom-validators/misc.js'
-import { createLogger, mtimeSortFilesDesc } from '@server/helpers/logger.js'
+import { buildSearchTags, createLogger, mtimeSortFilesDesc } from '@server/helpers/logger.js'
+import express from 'express'
+import { readdir, readFile } from 'fs/promises'
+import { join } from 'path'
import { CONFIG } from '../../../initializers/config.js'
import { AUDIT_LOG_FILENAME, LOG_FILENAME, MAX_LOGS_OUTPUT_CHARACTERS } from '../../../initializers/constants.js'
import { asyncMiddleware, authenticate, buildRateLimiter, ensureUserHasRight, optionalAuthenticate } from '../../../middlewares/index.js'
@@ -19,26 +19,11 @@ const createClientLogRateLimiter = buildRateLimiter({
const logsRouter = express.Router()
-logsRouter.post('/logs/client',
- createClientLogRateLimiter,
- optionalAuthenticate,
- createClientLogValidator,
- createClientLog
-)
-
-logsRouter.get('/logs',
- authenticate,
- ensureUserHasRight(UserRight.MANAGE_LOGS),
- getLogsValidator,
- asyncMiddleware(getLogs)
-)
-
-logsRouter.get('/audit-logs',
- authenticate,
- ensureUserHasRight(UserRight.MANAGE_LOGS),
- getAuditLogsValidator,
- asyncMiddleware(getAuditLogs)
-)
+logsRouter.post('/logs/client', createClientLogRateLimiter, optionalAuthenticate, createClientLogValidator, createClientLog)
+
+logsRouter.get('/logs', authenticate, ensureUserHasRight(UserRight.MANAGE_LOGS), getLogsValidator, asyncMiddleware(getLogs))
+
+logsRouter.get('/audit-logs', authenticate, ensureUserHasRight(UserRight.MANAGE_LOGS), getAuditLogsValidator, asyncMiddleware(getAuditLogs))
// ---------------------------------------------------------------------------
@@ -81,7 +66,7 @@ async function getLogs (req: express.Request, res: express.Response) {
startDateQuery: req.query.startDate,
endDateQuery: req.query.endDate,
level: req.query.level || 'info',
- tagsOneOf: req.query.tagsOneOf,
+ tagsOneOf: buildSearchTags(req.query.tagsOneOf),
nameFilter: logNameFilter
})
@@ -94,13 +79,9 @@ async function generateOutput (options: {
level: ServerLogLevel
nameFilter: RegExp
- tagsOneOf?: string[]
+ tagsOneOf?: Set<string>
}) {
- const { startDateQuery, level, nameFilter } = options
-
- const tagsOneOf = Array.isArray(options.tagsOneOf) && options.tagsOneOf.length !== 0
- ? new Set(options.tagsOneOf)
- : undefined
+ const { startDateQuery, level, nameFilter, tagsOneOf } = options
const logFiles = await readdir(CONFIG.STORAGE.LOG_DIR)
const sortedLogFiles = await mtimeSortFilesDesc(logFiles, CONFIG.STORAGE.LOG_DIR)
@@ -145,7 +126,7 @@ async function getOutputFromFile (options: {
let logTime: number
- const logsLevel: { [ id in ServerLogLevel ]: number } = {
+ const logsLevel: { [id in ServerLogLevel]: number } = {
audit: -1,
debug: 0,
info: 1,
diff --git a/server/core/helpers/logger.ts b/server/core/helpers/logger.ts
index 3dc1fe4873..344fb7d95e 100644
--- a/server/core/helpers/logger.ts
+++ b/server/core/helpers/logger.ts
@@ -1,5 +1,6 @@
import { context, trace } from '@opentelemetry/api'
-import { isTestOrDevInstance } from '@peertube/peertube-node-utils'
+import { arrayify } from '@peertube/peertube-core-utils'
+import { isShortUUID, isTestOrDevInstance } from '@peertube/peertube-node-utils'
import { AsyncLocalStorage } from 'async_hooks'
import { stat } from 'fs/promises'
import { join } from 'path'
@@ -10,6 +11,7 @@ import { FileTransportOptions } from 'winston/lib/winston/transports/index.js'
import { isMainThread } from 'worker_threads'
import { CONFIG } from '../initializers/config.js'
import { LOG_FILENAME } from '../initializers/constants.js'
+import { toCompleteUUID } from './custom-validators/misc.js'
const label = CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT
@@ -530,3 +532,22 @@ function exitOnCrash () {
rootWinstonLogger.end()
}
+
+// ---------------------------------------------------------------------------
+
+export function buildSearchTags (searchTagsArg: string[]): Set<string> | null {
+ const searchTags = arrayify(searchTagsArg)
+ if (!searchTags || searchTags.length === 0) return null
+
+ const tagsOneOf = new Set(searchTags)
+
+ for (const tag of searchTags) {
+ if (isShortUUID(tag)) { // So we can also search for short UUIDs
+ tagsOneOf.add(toCompleteUUID(tag))
+ }
+ }
+
+ if (tagsOneOf.size === 0) return null
+
+ return tagsOneOf
+}
diff --git a/server/scripts/create-import-video-file-job.ts b/server/scripts/create-import-video-file-job.ts
index 004c3f2e91..d4e95c12c7 100644
--- a/server/scripts/create-import-video-file-job.ts
+++ b/server/scripts/create-import-video-file-job.ts
@@ -1,6 +1,6 @@
+import { isUUIDValid, toCompleteUUID } from '@server/helpers/custom-validators/misc.js'
import { program } from 'commander'
import { resolve } from 'path'
-import { isUUIDValid, toCompleteUUID } from '@server/helpers/custom-validators/misc.js'
import { initDatabaseModels } from '../core/initializers/database.js'
import { JobQueue } from '../core/lib/job-queue/index.js'
import { VideoModel } from '../core/models/video/video.js'
diff --git a/server/scripts/parse-log.ts b/server/scripts/parse-log.ts
index a7485523f3..ac4d0ebeb2 100755
--- a/server/scripts/parse-log.ts
+++ b/server/scripts/parse-log.ts
@@ -1,4 +1,6 @@
-import { program } from 'commander'
+import { createCommand } from '@commander-js/extra-typings'
+import { buildSearchTags, labelFormatter, mtimeSortFilesDesc } from '@server/helpers/logger.js'
+import { CONFIG } from '@server/initializers/config.js'
import { createReadStream } from 'fs'
import { readdir } from 'fs/promises'
import { join } from 'path'
@@ -7,18 +9,20 @@ import { createInterface } from 'readline'
import { format as sqlFormat } from 'sql-formatter'
import { inspect } from 'util'
import * as winston from 'winston'
-import { labelFormatter, mtimeSortFilesDesc } from '@server/helpers/logger.js'
-import { CONFIG } from '@server/initializers/config.js'
-program
- .option('-l, --level [level]', 'Level log (debug/info/warn/error)')
- .option('-f, --files [file...]', 'Files to parse. If not provided, the script will parse the latest log file from config)')
- .option('-t, --tags [tags...]', 'Display only lines with these tags')
- .option('-n, --not-tags [tags...]', 'Do not display lines containing these tags')
+const program = createCommand()
+ .description('Parse PeerTube log files and display them in a human readable format')
+ .option('-l, --level <level>', 'Level log (debug/info/warn/error)')
+ .option('-f, --files <file...>', 'Files to parse. If not provided, the script will parse the latest log file from config)')
+ .option('-t, --tags <tags...>', 'Display only lines with these tags')
+ .option('-n, --not-tags <tags...>', 'Do not display lines containing these tags')
.parse(process.argv)
const options = program.opts()
+const tags = buildSearchTags(options.tags)
+const notTags = buildSearchTags(options.notTags)
+
const excludedKeys = {
level: true,
message: true,
@@ -32,7 +36,7 @@ function keysExcluder (key, value) {
return excludedKeys[key] === true ? undefined : value
}
-const loggerFormat = winston.format.printf((info) => {
+const loggerFormat = winston.format.printf(info => {
let additionalInfos = JSON.stringify(info, keysExcluder, 2)
if (additionalInfos === '{}') additionalInfos = ''
else additionalInfos = ' ' + additionalInfos
@@ -101,11 +105,11 @@ function readFile (file: string) {
rl.on('line', line => {
try {
const log = JSON.parse(line)
- if (options.tags && !containsTags(log.tags, options.tags)) {
+ if (tags && !containsTags(log.tags, tags)) {
return
}
- if (options.notTags && containsTags(log.tags, options.notTags)) {
+ if (notTags && containsTags(log.tags, notTags)) {
return
}
@@ -148,13 +152,11 @@ function toTimeFormat (time: string) {
return d.toLocaleString() + `.${d.getMilliseconds()}`
}
-function containsTags (loggerTags: string[], optionsTags: string[]) {
+function containsTags (loggerTags: string[], optionsTags: Set<string>) {
if (!loggerTags) return false
for (const lt of loggerTags) {
- for (const ot of optionsTags) {
- if (lt === ot) return true
- }
+ if (optionsTags.has(lt)) return true
}
return false