Skip to content

Interactive interface for browsing global, full-resolution satellite imagery

1.9 KB
Raw Blame
const cheerio = require('cheerio')
const fs = require('fs')
// walk through directory to collect html file paths, parse for links, and return array of link objects

// create collection of HTML file addresses from directory
const walk = (dir) => {
  let results = []
  const list = fs.readdirSync(dir)
  list.forEach((file) => {
    file = `${dir}/${file}`
    const stat = fs.statSync(file)
    if (stat && stat.isDirectory()) {
      /* Recurse into a subdirectory */
      results = results.concat(walk(file))
    } else {
      const fileType = file.split('.').pop()
      if (fileType === 'html') {
        results.push(file)
      }
    }
  })
  return results
}

// get URLS from HTML in ./build/options using cheerio
const getUrls = (htmlArray) => {
  // skip exact (rel and href) doubles of links
  const trackDoubles = {}
  const scrapedHTML = []

  for (let i = 0; i < htmlArray.length; i += 1) {
    const $ = cheerio.load(fs.readFileSync(htmlArray[i]))
    const links = $('a')

    // create url object:
    // { 'AIRS - The AIRS instrument suite physical retrievals': 'http://airs.jpl.nasa.gov/data/physical_retrievals' }
    $(links).each((i, link) => {
      const linkRel = $(link).text() || 'EMPTY'
      const linkHref = $(link).attr('href')

      if (trackDoubles[linkRel] !== linkHref) {
        scrapedHTML.push({ [linkRel]: linkHref })
        trackDoubles[linkRel] = linkHref
      }
    })
  }
  return scrapedHTML
}

// get URLS from HTML files
// scraped URLs are saved in an array of objects with a key value pair of link text and href:
// 'Wildfire MB-CE042, Manitoba, Canada': 'https://eonet.gsfc.nasa.gov/api/v3/events/EONET_3518'
const initExtractor = async () => {
  // any html file with a url will be scraped and added
  const htmlFiles = await walk('./build/options')
  const htmlUrls = await getUrls(htmlFiles)
  return htmlUrls
}

module.exports = initExtractor