Skip to content

ActivityPub-federated video streaming platform using P2P directly in your web browser

Fix missing timeout on geoip/youtube-dl requests

Chocobozzzec61d8247a4e
diff --git a/server/core/helpers/geo-ip.ts b/server/core/helpers/geo-ip.ts
index 5061d74ea5..4c7e08c6d2 100644
--- a/server/core/helpers/geo-ip.ts
+++ b/server/core/helpers/geo-ip.ts
@@ -3,9 +3,10 @@ import { pathExists } from 'fs-extra/esm'
import { writeFile } from 'fs/promises'
import maxmind, { CityResponse, CountryResponse, Reader } from 'maxmind'
import { join } from 'path'
+import { REQUEST_TIMEOUTS } from '../initializers/constants.js'
import { isArray } from './custom-validators/misc.js'
import { logger, loggerTagsFactory } from './logger.js'
-import { isBinaryResponse, unsafeSSRFGot } from './requests.js'
+import { doBufferRequest, isBinaryResponse } from './requests.js'
const lTags = loggerTagsFactory('geo-ip')
@@ -101,10 +102,12 @@ export class GeoIP {
private async updateDatabaseFile (url: string, destination: string) {
logger.info('Updating GeoIP databases from %s.', url, lTags())
- const gotOptions = { context: { bodyKBLimit: 800_000 }, responseType: 'buffer' as 'buffer' }
-
try {
- const gotResult = await unsafeSSRFGot(url, gotOptions)
+ const gotResult = await doBufferRequest(url, {
+ bodyKBLimit: 800_000,
+ timeout: REQUEST_TIMEOUTS.FILE,
+ preventSSRF: false
+ })
if (!isBinaryResponse(gotResult)) {
throw new Error('Not a binary response')
diff --git a/server/core/helpers/requests.ts b/server/core/helpers/requests.ts
index 44da15b582..af6c22d4b0 100644
--- a/server/core/helpers/requests.ts
+++ b/server/core/helpers/requests.ts
@@ -2,7 +2,15 @@ import { signAsDraftToRequest } from '@misskey-dev/node-http-message-signatures'
import { CONFIG } from '@server/initializers/config.js'
import { createWriteStream } from 'fs'
import { remove } from 'fs-extra/esm'
-import got, { OptionsInit, OptionsOfTextResponseBody, OptionsOfUnknownResponseBodyWrapped, Request, RequestError, Response } from 'got'
+import got, {
+ OptionsInit,
+ OptionsOfBufferResponseBody,
+ OptionsOfTextResponseBody,
+ OptionsOfUnknownResponseBodyWrapped,
+ Request,
+ RequestError,
+ Response
+} from 'got'
import { gotSsrf } from 'got-ssrf'
import http from 'http'
import https from 'https'
@@ -44,7 +52,7 @@ export type PeerTubeRequestOptions = {
signal?: AbortSignal
} & Pick<OptionsInit, 'headers' | 'json' | 'method' | 'searchParams'>
-export const unsafeSSRFGot = got.extend({
+const unsafeSSRFGot = got.extend({
...getProxyAgent(),
headers: {
@@ -162,6 +170,19 @@ export function doJSONRequest<T> (url: string, options: PeerTubeRequestOptions &
})
}
+export function doBufferRequest (url: string, options: PeerTubeRequestOptions & { preventSSRF?: false } = {}) {
+ const gotOptions = buildGotOptions(options) as OptionsOfBufferResponseBody
+
+ const gotInstance = options.preventSSRF === false
+ ? unsafeSSRFGot
+ : peertubeGot
+
+ return gotInstance(url, { ...gotOptions, responseType: 'buffer' })
+ .catch(err => {
+ throw buildRequestError(err)
+ })
+}
+
export async function doRequestAndSaveToFile (url: string, destPath: string, options: PeerTubeRequestOptions = {}) {
const gotOptions = buildGotOptions({ ...options, timeout: options.timeout ?? REQUEST_TIMEOUTS.FILE })
diff --git a/server/core/helpers/youtube-dl/youtube-dl-cli.ts b/server/core/helpers/youtube-dl/youtube-dl-cli.ts
index 2e7aec2dc1..1b0a54dffa 100644
--- a/server/core/helpers/youtube-dl/youtube-dl-cli.ts
+++ b/server/core/helpers/youtube-dl/youtube-dl-cli.ts
@@ -4,11 +4,11 @@ import { CONFIG } from '@server/initializers/config.js'
import { execa, Options as ExecaNodeOptions } from 'execa'
import { ensureDir, pathExists } from 'fs-extra/esm'
import { chmod, writeFile } from 'fs/promises'
-import { OptionsOfBufferResponseBody } from 'got'
import { dirname, join } from 'path'
+import { REQUEST_TIMEOUTS } from '../../initializers/constants.js'
import { logger, loggerTagsFactory } from '../logger.js'
import { getProxy, isProxyEnabled } from '../proxy.js'
-import { isBinaryResponse, unsafeSSRFGot } from '../requests.js'
+import { doBufferRequest, isBinaryResponse, PeerTubeRequestOptions } from '../requests.js'
type ProcessOptions = Pick<ExecaNodeOptions, 'cwd' | 'maxBuffer'>
@@ -32,19 +32,20 @@ export class YoutubeDLCLI {
logger.info('Updating youtubeDL binary from %s.', url, lTags())
- const gotOptions: OptionsOfBufferResponseBody = {
- context: { bodyKBLimit: 100_000 },
- responseType: 'buffer'
+ const requestOptions: PeerTubeRequestOptions & { preventSSRF?: false } = {
+ bodyKBLimit: 100_000,
+ timeout: REQUEST_TIMEOUTS.FILE,
+ preventSSRF: false
}
if (process.env.YOUTUBE_DL_DOWNLOAD_BEARER_TOKEN) {
- gotOptions.headers = {
+ requestOptions.headers = {
authorization: 'Bearer ' + process.env.YOUTUBE_DL_DOWNLOAD_BEARER_TOKEN
}
}
try {
- let gotResult = await unsafeSSRFGot(url, gotOptions)
+ let gotResult = await doBufferRequest(url, requestOptions)
if (!isBinaryResponse(gotResult)) {
const json = JSON.parse(Buffer.from(gotResult.rawBody).toString())
@@ -55,7 +56,7 @@ export class YoutubeDLCLI {
const releaseAsset = latest.assets.find(a => a.name === releaseName)
if (!releaseAsset) throw new Error(`Cannot find appropriate release with name ${releaseName} in release assets`)
- gotResult = await unsafeSSRFGot(releaseAsset.browser_download_url, gotOptions)
+ gotResult = await doBufferRequest(releaseAsset.browser_download_url, requestOptions)
}
if (!isBinaryResponse(gotResult)) {