Skip to content

Interactive interface for browsing global, full-resolution satellite imagery

7.2 KB
Raw Blame
const fs = require('fs')
const path = require('path')
const yargs = require('yargs')
const { hideBin } = require('yargs/helpers')
const console = require('console')
const axios = require('axios').default

const prog = path.basename(__filename)

const options = yargs(hideBin(process.argv))
  .usage('Usage: $0 [options]')
  .option('features', {
    demandOption: true,
    alias: 'f',
    type: 'string',
    description: 'features.json file'
  })
  .option('layerOrder', {
    demandOption: true,
    alias: 'o',
    type: 'string',
    description: 'layerOrder.json file'
  })
  .option('layerMetadata', {
    demandOption: true,
    alias: 'm',
    type: 'string',
    description: 'layer-metadata/all.json file'
  })
  .option('cacheMode', {
    demandOption: false,
    alias: 'c',
    type: 'string',
    description: 'Cache mode for fetching data'
  })
  .epilog('Creates a layer-metadata file containing all layers')

const { argv } = options
if (!argv.features && !argv.layerOrder && !argv.layerMetadata) {
  throw new Error('Invalid number of arguments')
}

const cacheMode = argv.cacheMode

const featuresFile = argv.features
let featuresData = fs.readFileSync(featuresFile)
let features = JSON.parse(featuresData)
if (fs.existsSync(featuresFile)) {
  featuresData = fs.readFileSync(featuresFile)
  features = JSON.parse(featuresData)
} else {
  throw new Error(`Error: ${featuresFile} feature file does not exist`)
}

const layerOrderFile = argv.layerOrder
let layerOrderData
let layerOrder
if (fs.existsSync(layerOrderFile)) {
  layerOrderData = fs.readFileSync(layerOrderFile)
  layerOrder = JSON.parse(layerOrderData)
} else {
  throw new Error(`Error: ${layerOrderFile} layer order file does not exist`)
}

const outputFile = argv.layerMetadata

const metadataConfig = features.features.vismetadata
const url = metadataConfig.url
const daacMap = metadataConfig.daacMap || {}
const configuredSkipLayers = metadataConfig.skipLayers || []

// These are aliases or otherwise layers that don't exist in GIBS
const defaultSkipLayers = [
  'Land_Water_Map',
  'Land_Mask',
  'World_Database_on_Protected_Areas',
  'HLS_Shortwave_Infrared_Sentinel',
  'HLS_Shortwave_Infrared_Landsat',
  'HLS_False_Color_Vegetation_Landsat',
  'HLS_False_Color_Vegetation_Sentinel',
  'HLS_False_Color_Urban_Sentinel',
  'HLS_False_Color_Urban_Landsat',
  'HLS_False_Color_Sentinel',
  'HLS_Customizable_Sentinel',
  'HLS_Customizable_Landsat',
  'HLS_NDVI_Landsat',
  'HLS_NDWI_Landsat',
  'HLS_NDSI_Landsat',
  'HLS_Moisture_Index_Landsat',
  'HLS_EVI_Landsat',
  'HLS_SAVI_Landsat',
  'HLS_MSAVI_Landsat',
  'HLS_NBR2_Landsat',
  'HLS_NBR_Landsat',
  'HLS_TVI_Landsat',
  'HLS_NDVI_Sentinel',
  'HLS_NDWI_Sentinel',
  'HLS_NDSI_Sentinel',
  'HLS_Moisture_Index_Sentinel',
  'HLS_EVI_Sentinel',
  'HLS_SAVI_Sentinel',
  'HLS_MSAVI_Sentinel',
  'HLS_NBR2_Sentinel',
  'HLS_NBR_Sentinel',
  'HLS_TVI_Sentinel',
  'HLS_False_Color_Landsat',
  'AERONET_AOD_500NM',
  'AERONET_ANGSTROM_440-870NM',
  'DAILY_AERONET_AOD_500NM',
  'DAILY_AERONET_ANGSTROM_440-870NM',
  'NOAA_2025_ERI_WMTS',
  'Reference_Labels_15m_arctic',
  'World_Imagery',
  'EUMETSAT_RING_Airmass_RGB',
  'EUMETSAT_RING_Ash_RGB',
  'EUMETSAT_RING_Dust_RGB',
  'EUMETSAT_RING_Natural_Color_RGB',
  'EUMETSAT_RING_IR108',
  'EUMETSAT_MSG_0Deg_Natural_Color_Enhanced_RGB_15min',
  'EUMETSAT_MSG_IDOC_Natural_Color_Enhanced_RGB_15min',
  'EUMETSAT_MTG_0Deg_GeoColor_RGB_10min',
  'EUMETSAT_MTG_0Deg_IR105_10min',
  'EUMETSAT_MSG_IDOC_IR108_10min',
  'OPERA_L2_Radiometric_Terrain_Corrected_SAR_Sentinel-1_12Day',
  'NISAR_L2_Geocoded_Polarimetric_Covariance_12Day',
  'Admin_Boundaries',
  'DoS_International_Boundaries'
]

const skipLayers = defaultSkipLayers.concat(configuredSkipLayers)

// NOTE: Only using these properties at this time
const useKeys = [
  'conceptIds',
  'dataCenter',
  'daynight',
  'orbitTracks',
  'orbitDirection',
  'ongoing',
  'layerPeriod',
  'title',
  'subtitle'
]

async function main (url) {
  layerOrder = layerOrder.layerOrder
  layerOrder = layerOrder.filter(x => !skipLayers.includes(x))

  console.warn(`${prog}: Fetching ${layerOrder.length} layer-metadata files`)

  const BATCH_SIZE = 15
  const results = []

  for (let i = 0; i < layerOrder.length; i += BATCH_SIZE) {
    const chunk = layerOrder.slice(i, i + BATCH_SIZE)
    const chunkPromises = chunk.map((layerId) => {
      if (!layerId.includes('_STD') && !layerId.includes('_NRT')) return getMetadata(layerId, url)
      return Promise.reject(new Error(`Skipped layer: ${layerId}`))
    })
    const chunkResults = await Promise.allSettled(chunkPromises)
    results.push(...chunkResults)
  }

  const { fulfilled = [], rejected = [] } = Object.groupBy(results, (item) => item.status)

  rejected.forEach(({ reason }) => {
    console.error(`${prog}: ERROR: ${reason}\n`)
  })
  const metadataMap = new Map(fulfilled.map(({ value }) => value))
  const layerMetadata = Object.fromEntries(metadataMap)

  const layers = Object.keys(layerMetadata).sort().reduce(
    (obj, key) => {
      obj[key] = layerMetadata[key]
      return obj
    },
    {}
  )

  fs.writeFileSync(outputFile, JSON.stringify({ layers }))
  console.warn(`${prog}: Combined all ${Object.keys(layerMetadata).length} layer-metadata files into ${path.parse(outputFile).base}`)
}

function getDAAC (metadata) {
  if (!Array.isArray(metadata.conceptIds) || !metadata.conceptIds.length) {
    return metadata
  }
  for (const collection of metadata.conceptIds) {
    const origDataCenter = collection.dataCenter
    const dataCenter = daacMap[origDataCenter]
    if (!dataCenter) {
      continue
    }
    delete collection.dataCenter
    if (!metadata.dataCenter) {
      metadata.dataCenter = [dataCenter]
    } else if (!metadata.dataCenter.includes(dataCenter)) {
      metadata.dataCenter.push(dataCenter)
    }
  }
  return metadata
}

let headers = {}

if (cacheMode === 'no-store') {
  const noCacheHeaders = {
    'Cache-Control': 'no-cache no-store',
    Pragma: 'no-cache',
    Expires: '0'
  }
  headers = Object.assign(headers, noCacheHeaders)
}

async function getMetadata (layerId, baseUrl, count) {
  if (count) console.warn(`retry #${count} for ${layerId}`)
  try {
    const response = await axios({
      method: 'get',
      url: `${baseUrl}${layerId}.json`,
      responseType: 'json',
      timeout: 10000,
      headers
    })
    const metadata = response.data
    const daac = getDAAC(metadata)
    let metadataKeys = Object.keys(daac)
    metadataKeys = metadataKeys.filter(x => !useKeys.includes(x))
    for (const key of metadataKeys) {
      delete daac[key]
    }
    return [layerId, daac]
  } catch (error) {
    return await handleException(error, layerId, baseUrl, count)
  }
}

async function handleException (error, layerId, url, count) {
  if (error.response && error.response.status === 404) {
    console.warn(`\n ${prog} WARN: Layer ${layerId} not found (404) at ${url}${layerId}.json`)
    return Promise.reject(new Error(`Layer ${layerId} returned 404`))
  }

  if (!count) count = 0
  count++
  if (count <= 5) {
    return getMetadata(layerId, url, count)
  } else {
    console.warn(`\n ${prog} WARN: Unable to fetch ${layerId} ${error}`)
    const msg = error.msg || error
    return Promise.reject(new Error(`Failed to fetch layer ${layerId}: ${msg}`))
  }
}

main(url).catch((err) => {
  console.error(err.stack)
})