ActivityPub-federated video streaming platform using P2P directly in your web browser
Support short uuids when parsing logs
Chocobozzzbad768ddfa43
diff --git a/packages/tests/src/api/server/logs.ts b/packages/tests/src/api/server/logs.tsindex 10a2d23ff0..cf6fb8d7a9 100644--- a/packages/tests/src/api/server/logs.ts+++ b/packages/tests/src/api/server/logs.ts@@ -94,7 +94,7 @@ describe('Test logs', function () {it('Should filter by tag', async function () {const now = new Date()- const { uuid } = await server.videos.upload({ attributes: { name: 'video 6' } })+ const { uuid, shortUUID } = await server.videos.upload({ attributes: { name: 'video 6' } })await waitJobs([ server ]){@@ -102,8 +102,8 @@ describe('Test logs', function () {expect(body).to.have.lengthOf(0)}- {- const body = await logsCommand.getLogs({ startDate: now, level: 'debug', tagsOneOf: [ uuid ] })+ for (const tag of [ uuid, shortUUID ]) {+ const body = await logsCommand.getLogs({ startDate: now, level: 'debug', tagsOneOf: [ tag ] })expect(body).to.not.have.lengthOf(0)for (const line of body) {diff --git a/server/core/controllers/api/server/logs.ts b/server/core/controllers/api/server/logs.tsindex 7d7888ea42..33da50575b 100644--- a/server/core/controllers/api/server/logs.ts+++ b/server/core/controllers/api/server/logs.ts@@ -1,10 +1,10 @@-import express from 'express'-import { readdir, readFile } from 'fs/promises'-import { join } from 'path'import { pick } from '@peertube/peertube-core-utils'import { ClientLogCreate, HttpStatusCode, ServerLogLevel, UserRight } from '@peertube/peertube-models'import { isArray } from '@server/helpers/custom-validators/misc.js'-import { createLogger, mtimeSortFilesDesc } from '@server/helpers/logger.js'+import { buildSearchTags, createLogger, mtimeSortFilesDesc } from '@server/helpers/logger.js'+import express from 'express'+import { readdir, readFile } from 'fs/promises'+import { join } from 'path'import { CONFIG } from '../../../initializers/config.js'import { AUDIT_LOG_FILENAME, LOG_FILENAME, MAX_LOGS_OUTPUT_CHARACTERS } from '../../../initializers/constants.js'import { asyncMiddleware, authenticate, buildRateLimiter, ensureUserHasRight, optionalAuthenticate } from '../../../middlewares/index.js'@@ -19,26 +19,11 @@ const createClientLogRateLimiter = buildRateLimiter({const logsRouter = express.Router()-logsRouter.post('/logs/client',- createClientLogRateLimiter,- optionalAuthenticate,- createClientLogValidator,- createClientLog-)--logsRouter.get('/logs',- authenticate,- ensureUserHasRight(UserRight.MANAGE_LOGS),- getLogsValidator,- asyncMiddleware(getLogs)-)--logsRouter.get('/audit-logs',- authenticate,- ensureUserHasRight(UserRight.MANAGE_LOGS),- getAuditLogsValidator,- asyncMiddleware(getAuditLogs)-)+logsRouter.post('/logs/client', createClientLogRateLimiter, optionalAuthenticate, createClientLogValidator, createClientLog)++logsRouter.get('/logs', authenticate, ensureUserHasRight(UserRight.MANAGE_LOGS), getLogsValidator, asyncMiddleware(getLogs))++logsRouter.get('/audit-logs', authenticate, ensureUserHasRight(UserRight.MANAGE_LOGS), getAuditLogsValidator, asyncMiddleware(getAuditLogs))// ---------------------------------------------------------------------------@@ -81,7 +66,7 @@ async function getLogs (req: express.Request, res: express.Response) {startDateQuery: req.query.startDate,endDateQuery: req.query.endDate,level: req.query.level || 'info',- tagsOneOf: req.query.tagsOneOf,+ tagsOneOf: buildSearchTags(req.query.tagsOneOf),nameFilter: logNameFilter})@@ -94,13 +79,9 @@ async function generateOutput (options: {level: ServerLogLevelnameFilter: RegExp- tagsOneOf?: string[]+ tagsOneOf?: Set<string>}) {- const { startDateQuery, level, nameFilter } = options-- const tagsOneOf = Array.isArray(options.tagsOneOf) && options.tagsOneOf.length !== 0- ? new Set(options.tagsOneOf)- : undefined+ const { startDateQuery, level, nameFilter, tagsOneOf } = optionsconst logFiles = await readdir(CONFIG.STORAGE.LOG_DIR)const sortedLogFiles = await mtimeSortFilesDesc(logFiles, CONFIG.STORAGE.LOG_DIR)@@ -145,7 +126,7 @@ async function getOutputFromFile (options: {let logTime: number- const logsLevel: { [ id in ServerLogLevel ]: number } = {+ const logsLevel: { [id in ServerLogLevel]: number } = {audit: -1,debug: 0,info: 1,diff --git a/server/core/helpers/logger.ts b/server/core/helpers/logger.tsindex 3dc1fe4873..344fb7d95e 100644--- a/server/core/helpers/logger.ts+++ b/server/core/helpers/logger.ts@@ -1,5 +1,6 @@import { context, trace } from '@opentelemetry/api'-import { isTestOrDevInstance } from '@peertube/peertube-node-utils'+import { arrayify } from '@peertube/peertube-core-utils'+import { isShortUUID, isTestOrDevInstance } from '@peertube/peertube-node-utils'import { AsyncLocalStorage } from 'async_hooks'import { stat } from 'fs/promises'import { join } from 'path'@@ -10,6 +11,7 @@ import { FileTransportOptions } from 'winston/lib/winston/transports/index.js'import { isMainThread } from 'worker_threads'import { CONFIG } from '../initializers/config.js'import { LOG_FILENAME } from '../initializers/constants.js'+import { toCompleteUUID } from './custom-validators/misc.js'const label = CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT@@ -530,3 +532,22 @@ function exitOnCrash () {rootWinstonLogger.end()}++// ---------------------------------------------------------------------------++export function buildSearchTags (searchTagsArg: string[]): Set<string> | null {+ const searchTags = arrayify(searchTagsArg)+ if (!searchTags || searchTags.length === 0) return null++ const tagsOneOf = new Set(searchTags)++ for (const tag of searchTags) {+ if (isShortUUID(tag)) { // So we can also search for short UUIDs+ tagsOneOf.add(toCompleteUUID(tag))+ }+ }++ if (tagsOneOf.size === 0) return null++ return tagsOneOf+}diff --git a/server/scripts/create-import-video-file-job.ts b/server/scripts/create-import-video-file-job.tsindex 004c3f2e91..d4e95c12c7 100644--- a/server/scripts/create-import-video-file-job.ts+++ b/server/scripts/create-import-video-file-job.ts@@ -1,6 +1,6 @@+import { isUUIDValid, toCompleteUUID } from '@server/helpers/custom-validators/misc.js'import { program } from 'commander'import { resolve } from 'path'-import { isUUIDValid, toCompleteUUID } from '@server/helpers/custom-validators/misc.js'import { initDatabaseModels } from '../core/initializers/database.js'import { JobQueue } from '../core/lib/job-queue/index.js'import { VideoModel } from '../core/models/video/video.js'diff --git a/server/scripts/parse-log.ts b/server/scripts/parse-log.tsindex a7485523f3..ac4d0ebeb2 100755--- a/server/scripts/parse-log.ts+++ b/server/scripts/parse-log.ts@@ -1,4 +1,6 @@-import { program } from 'commander'+import { createCommand } from '@commander-js/extra-typings'+import { buildSearchTags, labelFormatter, mtimeSortFilesDesc } from '@server/helpers/logger.js'+import { CONFIG } from '@server/initializers/config.js'import { createReadStream } from 'fs'import { readdir } from 'fs/promises'import { join } from 'path'@@ -7,18 +9,20 @@ import { createInterface } from 'readline'import { format as sqlFormat } from 'sql-formatter'import { inspect } from 'util'import * as winston from 'winston'-import { labelFormatter, mtimeSortFilesDesc } from '@server/helpers/logger.js'-import { CONFIG } from '@server/initializers/config.js'-program- .option('-l, --level [level]', 'Level log (debug/info/warn/error)')- .option('-f, --files [file...]', 'Files to parse. If not provided, the script will parse the latest log file from config)')- .option('-t, --tags [tags...]', 'Display only lines with these tags')- .option('-n, --not-tags [tags...]', 'Do not display lines containing these tags')+const program = createCommand()+ .description('Parse PeerTube log files and display them in a human readable format')+ .option('-l, --level <level>', 'Level log (debug/info/warn/error)')+ .option('-f, --files <file...>', 'Files to parse. If not provided, the script will parse the latest log file from config)')+ .option('-t, --tags <tags...>', 'Display only lines with these tags')+ .option('-n, --not-tags <tags...>', 'Do not display lines containing these tags').parse(process.argv)const options = program.opts()+const tags = buildSearchTags(options.tags)+const notTags = buildSearchTags(options.notTags)+const excludedKeys = {level: true,message: true,@@ -32,7 +36,7 @@ function keysExcluder (key, value) {return excludedKeys[key] === true ? undefined : value}-const loggerFormat = winston.format.printf((info) => {+const loggerFormat = winston.format.printf(info => {let additionalInfos = JSON.stringify(info, keysExcluder, 2)if (additionalInfos === '{}') additionalInfos = ''else additionalInfos = ' ' + additionalInfos@@ -101,11 +105,11 @@ function readFile (file: string) {rl.on('line', line => {try {const log = JSON.parse(line)- if (options.tags && !containsTags(log.tags, options.tags)) {+ if (tags && !containsTags(log.tags, tags)) {return}- if (options.notTags && containsTags(log.tags, options.notTags)) {+ if (notTags && containsTags(log.tags, notTags)) {return}@@ -148,13 +152,11 @@ function toTimeFormat (time: string) {return d.toLocaleString() + `.${d.getMilliseconds()}`}-function containsTags (loggerTags: string[], optionsTags: string[]) {+function containsTags (loggerTags: string[], optionsTags: Set<string>) {if (!loggerTags) return falsefor (const lt of loggerTags) {- for (const ot of optionsTags) {- if (lt === ot) return true- }+ if (optionsTags.has(lt)) return true}return false