ActivityPub-federated video streaming platform using P2P directly in your web browser
Fix missing timeout on geoip/youtube-dl requests
Chocobozzzec61d8247a4e
diff --git a/server/core/helpers/geo-ip.ts b/server/core/helpers/geo-ip.tsindex 5061d74ea5..4c7e08c6d2 100644--- a/server/core/helpers/geo-ip.ts+++ b/server/core/helpers/geo-ip.ts@@ -3,9 +3,10 @@ import { pathExists } from 'fs-extra/esm'import { writeFile } from 'fs/promises'import maxmind, { CityResponse, CountryResponse, Reader } from 'maxmind'import { join } from 'path'+import { REQUEST_TIMEOUTS } from '../initializers/constants.js'import { isArray } from './custom-validators/misc.js'import { logger, loggerTagsFactory } from './logger.js'-import { isBinaryResponse, unsafeSSRFGot } from './requests.js'+import { doBufferRequest, isBinaryResponse } from './requests.js'const lTags = loggerTagsFactory('geo-ip')@@ -101,10 +102,12 @@ export class GeoIP {private async updateDatabaseFile (url: string, destination: string) {logger.info('Updating GeoIP databases from %s.', url, lTags())- const gotOptions = { context: { bodyKBLimit: 800_000 }, responseType: 'buffer' as 'buffer' }-try {- const gotResult = await unsafeSSRFGot(url, gotOptions)+ const gotResult = await doBufferRequest(url, {+ bodyKBLimit: 800_000,+ timeout: REQUEST_TIMEOUTS.FILE,+ preventSSRF: false+ })if (!isBinaryResponse(gotResult)) {throw new Error('Not a binary response')diff --git a/server/core/helpers/requests.ts b/server/core/helpers/requests.tsindex 44da15b582..af6c22d4b0 100644--- a/server/core/helpers/requests.ts+++ b/server/core/helpers/requests.ts@@ -2,7 +2,15 @@ import { signAsDraftToRequest } from '@misskey-dev/node-http-message-signatures'import { CONFIG } from '@server/initializers/config.js'import { createWriteStream } from 'fs'import { remove } from 'fs-extra/esm'-import got, { OptionsInit, OptionsOfTextResponseBody, OptionsOfUnknownResponseBodyWrapped, Request, RequestError, Response } from 'got'+import got, {+ OptionsInit,+ OptionsOfBufferResponseBody,+ OptionsOfTextResponseBody,+ OptionsOfUnknownResponseBodyWrapped,+ Request,+ RequestError,+ Response+} from 'got'import { gotSsrf } from 'got-ssrf'import http from 'http'import https from 'https'@@ -44,7 +52,7 @@ export type PeerTubeRequestOptions = {signal?: AbortSignal} & Pick<OptionsInit, 'headers' | 'json' | 'method' | 'searchParams'>-export const unsafeSSRFGot = got.extend({+const unsafeSSRFGot = got.extend({...getProxyAgent(),headers: {@@ -162,6 +170,19 @@ export function doJSONRequest<T> (url: string, options: PeerTubeRequestOptions &})}+export function doBufferRequest (url: string, options: PeerTubeRequestOptions & { preventSSRF?: false } = {}) {+ const gotOptions = buildGotOptions(options) as OptionsOfBufferResponseBody++ const gotInstance = options.preventSSRF === false+ ? unsafeSSRFGot+ : peertubeGot++ return gotInstance(url, { ...gotOptions, responseType: 'buffer' })+ .catch(err => {+ throw buildRequestError(err)+ })+}+export async function doRequestAndSaveToFile (url: string, destPath: string, options: PeerTubeRequestOptions = {}) {const gotOptions = buildGotOptions({ ...options, timeout: options.timeout ?? REQUEST_TIMEOUTS.FILE })diff --git a/server/core/helpers/youtube-dl/youtube-dl-cli.ts b/server/core/helpers/youtube-dl/youtube-dl-cli.tsindex 2e7aec2dc1..1b0a54dffa 100644--- a/server/core/helpers/youtube-dl/youtube-dl-cli.ts+++ b/server/core/helpers/youtube-dl/youtube-dl-cli.ts@@ -4,11 +4,11 @@ import { CONFIG } from '@server/initializers/config.js'import { execa, Options as ExecaNodeOptions } from 'execa'import { ensureDir, pathExists } from 'fs-extra/esm'import { chmod, writeFile } from 'fs/promises'-import { OptionsOfBufferResponseBody } from 'got'import { dirname, join } from 'path'+import { REQUEST_TIMEOUTS } from '../../initializers/constants.js'import { logger, loggerTagsFactory } from '../logger.js'import { getProxy, isProxyEnabled } from '../proxy.js'-import { isBinaryResponse, unsafeSSRFGot } from '../requests.js'+import { doBufferRequest, isBinaryResponse, PeerTubeRequestOptions } from '../requests.js'type ProcessOptions = Pick<ExecaNodeOptions, 'cwd' | 'maxBuffer'>@@ -32,19 +32,20 @@ export class YoutubeDLCLI {logger.info('Updating youtubeDL binary from %s.', url, lTags())- const gotOptions: OptionsOfBufferResponseBody = {- context: { bodyKBLimit: 100_000 },- responseType: 'buffer'+ const requestOptions: PeerTubeRequestOptions & { preventSSRF?: false } = {+ bodyKBLimit: 100_000,+ timeout: REQUEST_TIMEOUTS.FILE,+ preventSSRF: false}if (process.env.YOUTUBE_DL_DOWNLOAD_BEARER_TOKEN) {- gotOptions.headers = {+ requestOptions.headers = {authorization: 'Bearer ' + process.env.YOUTUBE_DL_DOWNLOAD_BEARER_TOKEN}}try {- let gotResult = await unsafeSSRFGot(url, gotOptions)+ let gotResult = await doBufferRequest(url, requestOptions)if (!isBinaryResponse(gotResult)) {const json = JSON.parse(Buffer.from(gotResult.rawBody).toString())@@ -55,7 +56,7 @@ export class YoutubeDLCLI {const releaseAsset = latest.assets.find(a => a.name === releaseName)if (!releaseAsset) throw new Error(`Cannot find appropriate release with name ${releaseName} in release assets`)- gotResult = await unsafeSSRFGot(releaseAsset.browser_download_url, gotOptions)+ gotResult = await doBufferRequest(releaseAsset.browser_download_url, requestOptions)}if (!isBinaryResponse(gotResult)) {