Skip to content

armgit / plexo

Public Imported from GitHub
0

Speed up downloads by combining multiple network connections in parallel

12.2 KB
Raw Blame
import { request as httpRequest } from 'node:http'
import { request as httpsRequest } from 'node:https'
import { basename, isAbsolute } from 'node:path'
import { URL } from 'node:url'
import type { BrowserContext, ProbeResult } from '../../shared/types'
import { testKnobs } from '../testKnobs'
import { requestHeaders } from './browserContext'
import { parseContentDispositionFilename } from './contentDisposition'
import { describeNzb, downloadNzbFile, indexerError, readNzbFile, sniffXml } from './nzb/nzbFile'
import { USENET_OFF, usenetEnabled } from './nzb/usenetServer'
import { normalizeEtag } from './fileVersion'
import {
  describeTorrent,
  downloadTorrentFile,
  fetchMagnetMetadata,
  readTorrentFile
} from './torrent/metadata'

const MAX_REDIRECTS = 5
// A server that accepts the connection and never answers would otherwise hang the probe — and
// the link field's "Checking…" — forever. Same budget as a stalled chunk, for the whole probe:
// redirects included, so a chain of slow hops can't stretch it.
const PROBE_TIMEOUT_MS = testKnobs.stallTimeoutMs
const NO_RESPONSE = 'The server did not respond — check the link and try again'
/** The last byte the probe asks for. Not 0 or 1: some file hosts answer a range that ends there
 * with the whole file while serving every real chunk as asked (#101). */
const PROBE_RANGE_END = 1023

type Headers = Record<string, string | string[] | undefined>

function headerValue(headers: Headers, name: string): string | undefined {
  const value = headers[name]
  return Array.isArray(value) ? value[0] : value
}

interface ProbeResponse {
  statusCode: number
  headers: Headers
}

/** GET with a small range: cheaper than fetching the body, and unlike HEAD it also tells us (via
 * the 206 status) whether range requests actually work. Not 1 byte: some file hosts answer a range
 * that ends at byte 0 or 1 with the whole file, while serving every real chunk as asked (#101).
 * `ranged: false` sends a plain GET instead; the body is still dropped once the headers arrive. */
function requestFirstBytes(
  url: URL,
  deadline: number,
  browser: BrowserContext | undefined,
  ranged: boolean,
  range = `bytes=0-${PROBE_RANGE_END}`
): Promise<ProbeResponse> {
  return new Promise((resolve, reject) => {
    const requester = url.protocol === 'https:' ? httpsRequest : httpRequest
    const req = requester(
      {
        method: 'GET',
        hostname: url.hostname.replace(/^\[|\]$/g, ''),
        port: url.port || undefined,
        path: `${url.pathname}${url.search}`,
        headers: ranged
          ? { ...requestHeaders(url, browser), Range: range }
          : requestHeaders(url, browser)
      },
      (res) => {
        clearTimeout(timer)
        res.destroy()
        resolve({ statusCode: res.statusCode ?? 0, headers: res.headers as Headers })
      }
    )
    const timer = setTimeout(
      () => req.destroy(new Error(NO_RESPONSE)),
      Math.max(0, deadline - Date.now())
    )
    req.on('error', (error) => {
      clearTimeout(timer)
      reject(error)
    })
    req.end()
  })
}

function fileNameFromHeaders(headers: Headers, url: URL): string {
  const disposition = headerValue(headers, 'content-disposition')
  if (disposition) {
    const parsed = parseContentDispositionFilename(disposition)
    if (parsed) return parsed
  }
  // Only the last segment is decoded: a malformed %-escape in a folder doesn't keep the file's
  // name escaped, and an escaped slash stays part of the name.
  let base = url.pathname.split('/').filter(Boolean).pop()
  try {
    if (base) base = decodeURIComponent(base)
  } catch {
    // A malformed %-escape: the raw name still names the file well enough.
  }
  return base && base.length > 0 ? base : 'download'
}

async function requestFollowingRedirects(
  rawUrl: string,
  browser: BrowserContext | undefined
): Promise<{ current: URL; response: ProbeResponse | null }> {
  let current = new URL(rawUrl)
  let response: ProbeResponse | null = null
  const deadline = Date.now() + PROBE_TIMEOUT_MS

  for (let redirect = 0; redirect <= MAX_REDIRECTS; redirect += 1) {
    let ranged = true
    response = await requestFirstBytes(current, deadline, browser, ranged)
    // Some servers refuse any Range request with 403 while serving the same URL in full (#131).
    // Ask once more without one: if that works, the file downloads as one stream from byte zero,
    // which sends no Range either — the same as on a server without range support.
    if (response.statusCode === 403 && ranged) {
      ranged = false
      response = await requestFirstBytes(current, deadline, browser, ranged)
    }
    if (response.statusCode >= 300 && response.statusCode < 400) {
      const location = headerValue(response.headers, 'location')
      if (!location) break
      if (redirect === MAX_REDIRECTS) {
        throw new Error('Too many redirects while checking the link')
      }
      current = new URL(location, current)
      continue
    }
    break
  }

  return { current, response }
}

/** Whether the server labels each byte range with an ETag of its own (pCloud does, #156): then
 * no chunk's ETag ever matches the probe's, and every one would be taken for a changed file.
 * The first range asked again keeps its ETag while the next range gets another. A load balancer
 * alternating labels per request changes it on the repeat instead, and keeps its ETag checked. */
async function etagIsPerRange(
  url: URL,
  browser: BrowserContext | undefined,
  first: ProbeResponse
): Promise<boolean> {
  const etag = headerValue(first.headers, 'etag')
  const total = /\/(\d+)$/.exec(headerValue(first.headers, 'content-range') ?? '')?.[1]
  if (first.statusCode !== 206 || !etag || !total || Number(total) <= 2048) return false
  const deadline = Date.now() + PROBE_TIMEOUT_MS
  try {
    const again = await requestFirstBytes(url, deadline, browser, true)
    const next = await requestFirstBytes(url, deadline, browser, true, 'bytes=1024-2047')
    const etagOf = (r: ProbeResponse): string => normalizeEtag(headerValue(r.headers, 'etag') ?? '')
    return (
      again.statusCode === 206 &&
      next.statusCode === 206 &&
      etagOf(again) === normalizeEtag(etag) &&
      etagOf(next) !== '' &&
      etagOf(next) !== normalizeEtag(etag)
    )
  } catch {
    return false
  }
}

/** A link the server says is a torrent, or that is named like one. */
function isTorrentLink(headers: Headers, url: URL): boolean {
  const contentType = headerValue(headers, 'content-type') ?? ''
  return /^application\/x-bittorrent\b/i.test(contentType) || /\.torrent$/i.test(url.pathname)
}

/** A link the server says is an NZB, or that is named like one. Indexers serve them from API
 * links that say nothing in their path, so the type is what usually tells. */
function isNzbLink(headers: Headers, url: URL, fileName: string): boolean {
  const contentType = headerValue(headers, 'content-type') ?? ''
  return (
    /^application\/x-nzb\b/i.test(contentType) ||
    /\.nzb$/i.test(url.pathname) ||
    /\.nzb$/i.test(fileName)
  )
}

/** What a link would download: a file over HTTP(S), a torrent — a magnet link, a link to a
 * .torrent, or the path of a .torrent on this computer (opened or dropped on the window) — or an
 * NZB, by link or path, the same way. `browser`: what the browser it came from sends with it (see
 * BrowserContext). */
export async function probeUrl(rawUrl: string, browser?: BrowserContext): Promise<ProbeResult> {
  if (/^magnet:/i.test(rawUrl)) return describeTorrent(await fetchMagnetMetadata(rawUrl), rawUrl)
  if (isAbsolute(rawUrl) && /\.torrent$/i.test(rawUrl)) {
    return describeTorrent(await readTorrentFile(rawUrl), rawUrl)
  }
  if (isAbsolute(rawUrl) && /\.nzb$/i.test(rawUrl)) {
    if (!(await usenetEnabled())) throw new Error(USENET_OFF)
    return describeNzb(await readNzbFile(rawUrl), rawUrl, basename(rawUrl))
  }

  const { current, response } = await requestFollowingRedirects(rawUrl, browser)

  // An empty file can't satisfy a request for its first bytes: the server answers 416 and gives
  // the size as `bytes */0`. That's a valid, empty download, not an error.
  if (
    response?.statusCode === 416 &&
    /^\s*bytes\s+\*\/0\s*$/i.test(headerValue(response.headers, 'content-range') ?? '')
  ) {
    return {
      kind: 'http',
      requestedUrl: rawUrl,
      finalUrl: current.toString(),
      supportsRanges: false,
      totalBytes: 0,
      suggestedFileName: fileNameFromHeaders(response.headers, current),
      contentType: headerValue(response.headers, 'content-type') ?? null,
      etag: headerValue(response.headers, 'etag') ?? null,
      lastModified: headerValue(response.headers, 'last-modified') ?? null
    }
  }

  // A redirect still standing here had no Location to follow: it isn't the file.
  if (!response || response.statusCode === 0 || response.statusCode >= 300) {
    throw new Error(`Server responded with status ${response?.statusCode || 'unknown'}`)
  }

  if (isTorrentLink(response.headers, current)) {
    const finalUrl = current.toString()
    const torrentFile = await downloadTorrentFile(
      finalUrl,
      PROBE_TIMEOUT_MS,
      requestHeaders(current, browser)
    )
    return describeTorrent(torrentFile, finalUrl)
  }

  const suggestedFileName = fileNameFromHeaders(response.headers, current)
  // Off, an NZB is a file like any other: for another newsreader to take.
  if (isNzbLink(response.headers, current, suggestedFileName) && (await usenetEnabled())) {
    const finalUrl = current.toString()
    const nzbFile = await downloadNzbFile(
      finalUrl,
      PROBE_TIMEOUT_MS,
      requestHeaders(current, browser)
    )
    return describeNzb(nzbFile, finalUrl, suggestedFileName)
  }
  // An indexer's API answers in XML either way: an NZB it doesn't label as one, or an error (a
  // wrong API key, its daily limit reached) where the NZB would be. Its start tells which.
  if (/xml/i.test(headerValue(response.headers, 'content-type') ?? '')) {
    const finalUrl = current.toString()
    const headers = requestHeaders(current, browser)
    const start = await sniffXml(finalUrl, PROBE_TIMEOUT_MS, headers)
    const refusal = indexerError(start)
    if (refusal) throw new Error(refusal)
    if (/<(?:[\w-]+:)?nzb\b/i.test(start) && (await usenetEnabled())) {
      const nzbFile = await downloadNzbFile(finalUrl, PROBE_TIMEOUT_MS, headers)
      return describeNzb(nzbFile, finalUrl, suggestedFileName)
    }
  }

  const contentRange = headerValue(response.headers, 'content-range')
  // A server that supports range requests must answer our range GET with 206 Partial Content.
  // If it returned 200 OK, it ignored the Range header and sent the whole file — even if its headers
  // statically claim `Accept-Ranges: bytes`.
  //
  // A 206 is only a promise if it sends us the bytes we asked for. Most say so in Content-Range,
  // but some hosts (Cloudflare-fronted file hosts among them) put the end of the *file* there
  // while setting Content-Length to the range they actually send. Trust whichever says the body
  // is bounded, so those keep their parallel streams instead of falling back to one.
  const served = /^bytes\s+(\d+)-(\d+)\//i.exec(contentRange ?? '')
  const contentLength = Number(headerValue(response.headers, 'content-length') ?? NaN)
  const servesOnlyProbe =
    served !== null &&
    Number(served[1]) === 0 &&
    (Number(served[2]) <= PROBE_RANGE_END || contentLength === PROBE_RANGE_END + 1)
  const supportsRanges = response.statusCode === 206 && servesOnlyProbe

  let totalBytes: number | null = null
  if (contentRange) {
    const match = /\/(\d+)$/.exec(contentRange)
    if (match) totalBytes = Number(match[1])
  }
  if (totalBytes === null && response.statusCode === 200) {
    // Only trust Content-Length on a full (200) response — on a 206 it
    // describes the bytes we asked for, not the whole file.
    const contentLength = headerValue(response.headers, 'content-length')
    if (contentLength) totalBytes = Number(contentLength)
  }

  const perRange = supportsRanges ? await etagIsPerRange(current, browser, response) : false

  return {
    kind: 'http',
    requestedUrl: rawUrl,
    finalUrl: current.toString(),
    supportsRanges,
    totalBytes,
    suggestedFileName,
    contentType: headerValue(response.headers, 'content-type') ?? null,
    etag: perRange ? null : (headerValue(response.headers, 'etag') ?? null),
    lastModified: headerValue(response.headers, 'last-modified') ?? null
  }
}