Skip to content

Interactive interface for browsing global, full-resolution satellite imagery

9.3 KB
Raw Blame
const moment = require('moment')
const axios = require('axios')
const fs = require('fs')
const path = require('path')
const yargs = require('yargs')
const { hideBin } = require('yargs/helpers')

const prog = path.basename(__filename)

const options = yargs(hideBin(process.argv))
  .usage('Usage: $0  <wv.json> <overrides_file>')
  .option('wvJsonFile', {
    demandOption: true,
    alias: 'w',
    type: 'string',
    description: 'wv.json file'
  })
  .option('overridesFile', {
    demandOption: true,
    alias: 'o',
    type: 'string',
    description: 'overrides file'
  })
  .option('featuresFile', {
    demandOption: true,
    alias: 'f',
    type: 'string',
    description: 'features file'
  })
  .option('cacheMode', {
    demandOption: false,
    alias: 'c',
    type: 'string',
    description: 'Cache mode for fetching data'
  })
  .epilog('Fetch preview images from WV Snapshots for any layers which they are missing.')

const { argv } = options
if (!argv.wvJsonFile && !argv.overridesFile && !argv.featuresFile) {
  throw new Error('Invalid number of arguments')
}

const wvJsonFile = argv.wvJsonFile
const overridesFile = argv.overridesFile
const featuresFile = argv.featuresFile
const cacheMode = argv.cacheMode

let overrideDatesDict = {}
const badSnapshots = []
let totalSuccessCount = 0
let totalFailureCount = 0
let snapshotsUrl = ''
const paramDict = {
  base: {
    REQUEST: 'GetSnapshot',
    FORMAT: 'image/jpeg'
  },
  geographic: {
    BBOX: '-90,-180,90,180',
    CRS: 'EPSG:4326',
    WIDTH: '768',
    HEIGHT: '384'
  },
  arctic: {
    BBOX: '-4195000,-4195000,4195000,4195000',
    CRS: 'EPSG:3413',
    WIDTH: '512',
    HEIGHT: '512'
  },
  antarctic: {
    BBOX: '-4195000,-4195000,4195000,4195000',
    CRS: 'EPSG:3031',
    WIDTH: '512',
    HEIGHT: '512'
  }
}
const standaloneLayers = [
  'Graticule_15m',
  'Coastlines_15m',
  'DoS_International_Boundaries',
  'Reference_Labels_15m'
]
const destImgDir = './web/images/layers/previews/'
const referenceLayers = {
  geographic: 'OSM_Land_Water_Map',
  arctic: 'OSM_Land_Water_Map',
  antarctic: 'SCAR_Land_Water_Map'
}

async function main () {
  // Check to see if this feature is enabled in features.json before continuing
  const featuresFileContent = fs.readFileSync(featuresFile, { encoding: 'utf-8' })
  const featuresDict = JSON.parse(featuresFileContent)
  if (featuresDict.features.previewSnapshots === false) {
    throw new Error('previewSnapshots note enabled in features.json')
  }

  // Allow manual configuration of layer ID to specific date to generate desired preview
  const overridesFileContent = fs.readFileSync(overridesFile, { encoding: 'utf-8' })
  overrideDatesDict = JSON.parse(overridesFileContent)

  const wvJsonFileContent = fs.readFileSync(wvJsonFile, { encoding: 'utf-8' })
  const wvJsonDict = JSON.parse(wvJsonFileContent)
  const layers = wvJsonDict.layers
  snapshotsUrl = wvJsonDict.features.imageDownload.url
  const fetchSnapshots = featuresDict.features.previewSnapshots
  if (!fetchSnapshots) {
    throw new Error(`${prog}: Layer preview fetching disabled. Exiting.`)
  }

  const layerValues = Object.values(layers)

  console.warn(`${prog}: Fetching snapshots for up to ${layerValues.length} layers...`)
  const promises = layerValues.map((layer) => getSnapshots(layer))

  await Promise.allSettled(promises)

  if (badSnapshots.length > 0) {
    console.warn(`\n${prog}: WARNING: ${badSnapshots.length} snapshots returned no content. See below for details: `)
    for (const badLayer of badSnapshots) {
      console.warn(`\n\t Layer: ${badLayer.id}`)
      console.warn(`\t URL: ${badLayer.url}`)
    }
  }

  if (totalSuccessCount > 0) {
    console.warn(`\n${prog}: Successfully retrieved ${totalSuccessCount} snapshots!`)
  }
  if (totalFailureCount > 0) {
    console.warn(`\n${prog}: WARNING: Failed to retrieve ${totalFailureCount} snapshots!`)
  }
  if (totalFailureCount === 0 && totalSuccessCount === 0) {
    console.warn(`\n${prog}: No snapshots were retrieved. All layers found in wv.json have existing preview images!`)
  }
}

async function trackBadSnapshots (layerId, projection, request, imgFile, badSnapshots) {
  const arcticBadSize = 9949
  const antarcticBadSize = 4060
  const geographicBadSize = 12088
  let size = 0

  fs.stat(imgFile, (err, stats) => {
    if (err) throw err
    size = stats.size
  })

  if (size === geographicBadSize || size === arcticBadSize || size === antarcticBadSize) {
    badSnapshots.push({
      id: layerId,
      projection,
      url: request.url
    })
  }
}

async function getBestDate (projection, period, dateRanges) {
  const lastRange = dateRanges[dateRanges.length - 1]
  const startDate = lastRange.startDate
  let endDate = lastRange.endDate
  const parsedStartDate = moment(startDate).utc()
  const parsedEndDate = moment(endDate).utc()
  const parsedEndYear = parsedEndDate.year()
  const parsedEndMonth = parsedEndDate.month() + 1
  let interval = parseInt(lastRange.dateInterval, 10)
  let alteredDate = null

  // Handle daily layers
  if (period === 'daily') {
    if (interval === 1) {
      // Go back a few more days for single day layers since something
      // too recent may not be processed yet
      interval = 3
    }
    alteredDate = moment(endDate).subtract(interval, 'days').utc().format()
  } else {
    alteredDate = moment(endDate).subtract(interval, 'months').utc().format()
  }

  // Choose a good daylight month for arctic
  if (projection === 'arctic' && ![4, 5, 6, 7, 8, 9].includes(parsedEndMonth)) {
    const currentYear = moment().year()
    const currentMonth = moment().month() + 1
    if (parsedEndYear === currentYear && currentMonth < 6) {
      alteredDate = moment({ year: parsedEndYear - 1, month: 4, day: 1 }).utc().format()
    } else {
      alteredDate = moment({ year: parsedEndYear, month: 4, day: 1 }).utc().format()
    }
  }

  // Choose a good daylight month for antarctic
  if (projection === 'antarctic' && ![10, 11, 12, 1, 2].includes(parsedEndMonth)) {
    alteredDate = moment({ year: parsedEndYear, month: 10, day: 1 }).utc().format()
  }

  // Make sure modified date isn't out of layer date range
  if (alteredDate && moment(alteredDate).isSameOrAfter(parsedStartDate)) {
    endDate = alteredDate
  }

  return endDate
}

async function getTimeParam (projection, layerId, layer) {
  // Only include TIME param for temporal layers
  const out = {}
  const dateRanges = layer.dateRanges
  const startDate = layer.startDate
  const period = layer.period

  if (dateRanges) {
    out.TIME = await getBestDate(projection, period, dateRanges)
  } else if (startDate) {
    out.TIME = startDate
  }

  // Use any configured override dates
  if (overrideDatesDict[layerId]) {
    out.TIME = overrideDatesDict[layerId]
  }

  return out
}

async function getSnapshots (layer) {
  for (const projection of Object.keys(layer.projections)) {
    const projDict = layer.projections[projection]
    const referenceLayer = referenceLayers[projection]
    let params = { ...paramDict.base, ...paramDict[projection] }

    // Sometimes a layer id is provided per projection (e.g. Land Mask layers)
    // We need to use this layer id to request the layer from WVS/GIBS
    // But, we need to use the WV id as the file name (since that's how we will look up the image in WV)
    let gibsLayerId, wvLayerId
    if (projDict.layer) {
      gibsLayerId = projDict.layer
      wvLayerId = layer.id
    } else {
      gibsLayerId = wvLayerId = layer.id
    }

    const destFileName = path.join(destImgDir, projection, `${wvLayerId}.jpg`)

    // Only get images that we don't have already
    const fileExists = await fs.promises.stat(destFileName)
      .then(() => true)
      .catch(() => false)
    if (fileExists) continue

    const timeParam = await getTimeParam(projection, wvLayerId, layer)
    params = Object.assign(params, timeParam)

    if (gibsLayerId !== referenceLayer && !standaloneLayers.includes(gibsLayerId)) {
      params.LAYERS = `${referenceLayer},${gibsLayerId}`
      params.OPACITIES = '0.50,1'
    } else {
      params.LAYERS = gibsLayerId
    }

    let headers = {}

    if (cacheMode === 'no-store') {
      const noCacheHeaders = {
        'Cache-Control': 'no-cache no-store',
        Pragma: 'no-cache',
        Expires: '0'
      }
      headers = Object.assign(headers, noCacheHeaders)
    }

    try {
      const imageReq = await axios({
        method: 'get',
        url: snapshotsUrl,
        params,
        responseType: 'stream',
        headers
      })
      let statusText
      if (imageReq.status === 200) {
        statusText = 'SUCCESS'
        totalSuccessCount += 1
        const dest = await fs.createWriteStream(destFileName, { flags: 'wx' })
        dest.on('finish', () => {
          console.warn(`File ${destFileName} has been written`)
          if (gibsLayerId === referenceLayers[projection]) {
            return
          }
          trackBadSnapshots(wvLayerId, projection, imageReq, destFileName)
        })
        imageReq.data.pipe(dest)
      } else {
        totalFailureCount += 1
        statusText = 'ERROR'
      }
      console.warn(`\n${prog}: Result: ${statusText} - ${imageReq.status}`)
      console.warn(`${prog}: Layer: ${wvLayerId}`)
      console.warn(`${prog}: URL: ${imageReq.config.url}`)
    } catch (e) {
      totalFailureCount += 1
      console.error(`${prog} ERROR: Unable to fetch layer: ${wvLayerId} proj:${projection}`)
      // console.error(`${prog} Error: ${e}`)
    }
  }
}

main().catch((err) => {
  console.error(err.stack)
})