Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
10.5 KB
Raw Blame
import http, { type IncomingMessage } from 'node:http'
import https from 'node:https'
import { isIP } from 'node:net'
import { Transform, Writable } from 'node:stream'
import { pipeline } from 'node:stream/promises'
import { checkServerIdentity as tlsCheck } from 'node:tls'
import { createBrotliDecompress, createGunzip, createInflate } from 'node:zlib'
import { parsePageUrl, resolvePageAddress } from './fetchPageDestination.js'
import { FetchPageError } from './fetchPageError.js'

export const FETCH_PAGE_LIMITS = Object.freeze({
  timeoutMs: 15000,
  maxBytes: 1024 * 1024,
  maxRedirects: 5,
  maxConcurrent: 4,
})
let active = 0 // Process-wide, including distinct MCP sessions.
const redirects = new Set([301, 302, 303, 307, 308])

// HTML tag names are ASCII-case-insensitive. This copy preserves offsets into
// the original text, including when Unicode characters precede a tag.
function asciiLower(html: string): string {
  return html.replace(/[A-Z]/g, character => character.toLowerCase())
}

function tagEnd(html: string, start: number): number {
  let quote = ''
  for (let cursor = start + 1; cursor < html.length; cursor++) {
    const character = html[cursor]
    if (quote) {
      if (character === quote) quote = ''
    } else if (character === '"' || character === "'") {
      quote = character
    } else if (character === '<') {
      return -2
    } else if (character === '>') {
      return cursor
    }
  }
  return -1
}

function closingTag(html: string, lower: string, name: string, from: number) {
  const prefix = `</${name}`
  let start = lower.indexOf(prefix, from)
  while (start !== -1) {
    if (/[\s>]/.test(lower[start + prefix.length] ?? '')) {
      const end = tagEnd(html, start)
      // A malformed candidate is not a close; a later well-formed tag is.
      if (end >= 0) return { start, end }
    }
    start = lower.indexOf(prefix, start + prefix.length)
  }
  return undefined
}

const inertText = new Set([
  'script',
  'style',
  'textarea',
  'xmp',
  'iframe',
  'noembed',
  'noframes',
  'noscript',
  'template',
])

/** Extract visible page prose and the first real title in linear time. */
function scanPage(html: string, xmlSyntax = false): { title: string; content: string } {
  const lower = asciiLower(html)
  const pieces: string[] = []
  let title = ''
  let foundTitle = false
  let offset = 0
  while (offset < html.length) {
    const start = lower.indexOf('<', offset)
    if (start === -1) {
      pieces.push(html.slice(offset))
      break
    }
    pieces.push(html.slice(offset, start))
    if (lower.startsWith('<!--', start)) {
      // HTML recovers <!--> and <!---> as abrupt-closing empty comments.
      const abrupt = lower.startsWith('<!--->', start)
        ? 6
        : lower.startsWith('<!-->', start)
          ? 5
          : 0
      if (abrupt) {
        pieces.push(' ')
        offset = start + abrupt
        continue
      }
      const end = lower.indexOf('-->', start + 4)
      if (end === -1) break
      pieces.push(' ')
      offset = end + 3
      continue
    }
    if (lower[start + 1] === '!' || lower[start + 1] === '?') {
      const end = lower.indexOf('>', start + 2)
      if (end === -1) break
      pieces.push(' ')
      offset = end + 1
      continue
    }
    const end = tagEnd(html, start)
    if (end === -2) {
      pieces.push('<')
      offset = start + 1
      continue
    }
    if (end === -1) {
      if (!/^<\/?[a-z]/.test(lower.slice(start, start + 4))) pieces.push(html.slice(start))
      break
    }
    const match = /^<\/?([a-z][a-z0-9:-]*)(?=[\s/>])/.exec(lower.slice(start, end + 1))
    if (!match) {
      pieces.push('<')
      offset = start + 1
      continue
    }
    pieces.push(' ')
    offset = end + 1
    if (!match || lower[start + 1] === '/') continue
    const name = match[1]
    // Only XML media types honor '/>' for non-void elements. HTML does not.
    if (xmlSyntax && lower.slice(start, end).trimEnd().endsWith('/')) {
      if (name === 'title') foundTitle = true
      continue
    }
    if (name === 'plaintext') {
      pieces.push(html.slice(offset))
      break
    }
    if (name !== 'title' && !inertText.has(name)) continue
    const close = closingTag(html, lower, name, offset)
    if (!close) break
    if (name === 'title') {
      if (!foundTitle) {
        title = html.slice(offset, close.start).replace(/\s+/g, ' ').trim()
        foundTitle = true
      }
      pieces.push(html.slice(offset, close.start), ' ')
    }
    offset = close.end + 1
  }
  return { title, content: pieces.join('').replace(/\s+/g, ' ').trim() }
}

export function pageText(html: string): string {
  return scanPage(html).content
}

function connect(url: URL, address: string, signal: AbortSignal): Promise<IncomingMessage> {
  return new Promise((resolve, reject) => {
    const hostname = url.hostname.replace(/^\[|\]$/g, '')
    const client = url.protocol === 'https:' ? https : http
    const req = client.request(
      {
        protocol: url.protocol,
        hostname: address,
        port: url.port || (url.protocol === 'https:' ? 443 : 80),
        method: 'GET',
        path: url.pathname + url.search,
        agent: false,
        signal,
        maxHeaderSize: 16 * 1024,
        headers: {
          Host: url.host,
          Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
          'Accept-Encoding': 'identity',
          'User-Agent': 'EvenfireWebSearch/1.0',
        },
        ...(url.protocol === 'https:'
          ? {
              servername: isIP(hostname) ? undefined : hostname,
              checkServerIdentity: (_host: string, cert: import('node:tls').PeerCertificate) =>
                // HTTPS connects to the pinned IP, but authenticates the URL host.
                tlsCheck(hostname, cert),
            }
          : {}),
      },
      resolve
    )
    req.once('error', reject)
    // A protocol switch does not emit a normal response. Explicitly release
    // its socket and settle the call; otherwise the concurrency slot can leak.
    req.once('upgrade', (_response, socket) => {
      socket.destroy()
      reject(new FetchPageError('upstream_failure'))
    })
    req.once('close', () => reject(new FetchPageError('upstream_failure')))
    req.end()
  })
}
async function readBody(res: IncomingMessage, signal: AbortSignal): Promise<string> {
  const length = res.headers['content-length']
  if (
    length !== undefined &&
    (!/^\d+$/.test(length) || Number(length) > FETCH_PAGE_LIMITS.maxBytes)
  ) {
    throw new FetchPageError('response_too_large')
  }
  const encoding = (res.headers['content-encoding'] ?? 'identity').trim().toLowerCase()
  const decoder =
    encoding === 'identity'
      ? undefined
      : encoding === 'gzip'
        ? createGunzip()
        : encoding === 'deflate'
          ? createInflate()
          : encoding === 'br'
            ? createBrotliDecompress()
            : null
  if (decoder === null) throw new FetchPageError('unsupported_encoding')
  let wireBytes = 0
  let decodedBytes = 0
  const chunks: Buffer[] = []
  const wire = new Transform({
    transform(chunk: Buffer, _encoding, callback) {
      wireBytes += chunk.length
      callback(
        wireBytes > FETCH_PAGE_LIMITS.maxBytes ? new FetchPageError('response_too_large') : null,
        chunk
      )
    },
  })
  const sink = new Writable({
    write(chunk: Buffer, _encoding, callback) {
      decodedBytes += chunk.length
      if (decodedBytes > FETCH_PAGE_LIMITS.maxBytes) {
        callback(new FetchPageError('response_too_large'))
        return
      }
      chunks.push(chunk)
      callback()
    },
  })
  if (decoder) await pipeline(res, wire, decoder, sink, { signal })
  else await pipeline(res, wire, sink, { signal })
  return Buffer.concat(chunks).toString('utf8')
}

export async function fetchPage(
  input: string,
  maxChars: number,
  callerSignal?: AbortSignal
): Promise<{ title: string; content: string }> {
  if (!Number.isInteger(maxChars) || maxChars < 100 || maxChars > 100000)
    throw new FetchPageError('invalid_max_chars')
  if (active >= FETCH_PAGE_LIMITS.maxConcurrent) throw new FetchPageError('busy')
  active++
  const controller = new AbortController()
  const timeout = setTimeout(
    () => controller.abort(new FetchPageError('deadline_exceeded')),
    FETCH_PAGE_LIMITS.timeoutMs
  )
  const cancel = () => controller.abort(new FetchPageError('cancelled'))
  if (callerSignal?.aborted) cancel()
  else callerSignal?.addEventListener('abort', cancel, { once: true })
  const seen = new Set<string>()
  try {
    let url = parsePageUrl(input)
    for (let hop = 0; ; hop++) {
      controller.signal.throwIfAborted()
      if (seen.has(url.href)) throw new FetchPageError('redirect_limit')
      seen.add(url.href)
      const address = await resolvePageAddress(url, controller.signal)
      const res = await connect(url, address, controller.signal)
      try {
        if (redirects.has(res.statusCode ?? 0)) {
          if (hop >= FETCH_PAGE_LIMITS.maxRedirects) throw new FetchPageError('redirect_limit')
          const location = res.headers.location
          if (!location) throw new FetchPageError('upstream_failure')
          let next: URL
          try {
            next = parsePageUrl(new URL(location, url).href)
          } catch {
            throw new FetchPageError('invalid_url')
          }
          if (url.protocol === 'https:' && next.protocol !== 'https:')
            throw new FetchPageError('destination_blocked')
          url = next
          // Do not consume untrusted redirect bodies at all. No bytes from
          // intermediate responses enter our accumulated body buffers.
          continue
        }
        if (!res.statusCode || res.statusCode < 200 || res.statusCode >= 300)
          throw new FetchPageError('upstream_failure')
        const html = await readBody(res, controller.signal)
        controller.signal.throwIfAborted()
        const contentType = res.headers['content-type']
        const mediaType =
          typeof contentType === 'string' ? contentType.split(';', 1)[0].trim().toLowerCase() : ''
        const xmlSyntax =
          mediaType === 'application/xml' || mediaType === 'text/xml' || mediaType.endsWith('+xml')
        const page = scanPage(html, xmlSyntax)
        return {
          title: page.title.slice(0, maxChars),
          content: page.content.slice(0, maxChars),
        }
      } finally {
        res.destroy()
      }
    }
  } catch (error) {
    if (controller.signal.aborted) throw controller.signal.reason
    throw error instanceof FetchPageError ? error : new FetchPageError('upstream_failure')
  } finally {
    clearTimeout(timeout)
    callerSignal?.removeEventListener('abort', cancel)
    active--
  }
}