Interactive interface for browsing global, full-resolution satellite imagery
const fs = require('fs')
const path = require('path')
const yargs = require('yargs')
const { hideBin } = require('yargs/helpers')
const console = require('console')
const axios = require('axios').default
const convert = require('xml-js')
const { promisify } = require('util')
const stream = require('stream')
const finished = promisify(stream.finished)
const prog = path.basename(__filename)
const options = yargs(hideBin(process.argv))
.usage('Usage: $0 [options]')
.option('config', {
demandOption: true,
alias: 'c',
type: 'string',
description: 'config file'
})
.option('getcapabilities', {
demandOption: true,
alias: 'g',
type: 'string',
description: 'getcapabilities file'
})
.option('mode', {
demandOption: true,
alias: 'm',
type: 'string',
description: 'mode'
})
.option('cacheMode', {
demandOption: false,
alias: 'f',
type: 'string',
description: 'Cache mode for fetching data'
})
.epilog('Pulls GetCapabilities XML and linked metadata from configured locations')
const { argv } = options
if (!argv.config && !argv.getcapabilities) {
throw new Error('Invalid number of arguments')
}
const configFile = argv.config
const outputDir = argv.getcapabilities
const cacheMode = argv.cacheMode
const colormaps = {}
const vectorstyles = {}
const vectordata = {}
const colormapsDir = path.join(outputDir, 'colormaps')
const vectorstylesDir = path.join(outputDir, 'vectorstyles')
const vectordataDir = path.join(outputDir, 'vectordata')
const configData = fs.readFileSync(configFile)
const config = JSON.parse(configData)
let headers = {}
if (cacheMode === 'no-store') {
const noCacheHeaders = {
'Cache-Control': 'no-cache no-store',
Pragma: 'no-cache',
Expires: '0'
}
headers = Object.assign(headers, noCacheHeaders)
}
async function main () {
if (!fs.existsSync(outputDir)) {
fs.mkdirSync(outputDir)
}
await getCapabilities()
if (colormaps) {
await gatherProcess(colormaps, 'colormap files', colormapsDir, '.xml')
}
if (vectorstyles) {
await gatherProcess(vectorstyles, 'vectorstyle files', vectorstylesDir, '.json')
}
if (vectordata) {
await gatherProcess(vectordata, 'vectordata files', vectordataDir, '.json')
}
}
async function getCapabilities () {
// Download each GC xml using the "from" attribute and put it in the "to" location
if (Object.prototype.hasOwnProperty.call(config, 'wv-options-fetch')) {
const fetchValues = config['wv-options-fetch']
const files = fetchValues.map(({ from, to }) => [from, `${outputDir}/${to}`])
await Promise.allSettled(files.map(([from, to]) => fetchConfigs(from, to)))
await Promise.allSettled(files.map(([, to]) => processGetCapabilities(to)))
}
}
// convert to superagent and use promises
async function fetchConfigs (inputFile, outputFile) {
const writer = await fs.createWriteStream(outputFile)
if (argv.mode === 'verbose') console.trace(`Fetching ${inputFile}...`)
const response = await axios({
method: 'get',
url: inputFile,
responseType: 'stream',
timeout: 100000,
headers
})
if (argv.mode === 'verbose') console.trace(`Writing ${outputFile}...`)
await response.data.pipe(writer)
return finished(writer)
}
async function handleException (error, link, dir, ext, count) {
if (!count) count = 0
count++
if (count <= 5) {
await processMetadata(link, dir, ext, count)
} else {
console.warn(`\n ${prog} WARN: Unable to fetch ${link} ${error}`)
}
}
async function processVectorData (layer) {
if (argv.mode === 'verbose') {
const ident = layer['ows:Identifier']._text
console.warn(`Processing vector data for ${ident}:`)
}
if (layer['ows:Metadata']) {
const metadataItems = Array.isArray(layer['ows:Metadata'])
? layer['ows:Metadata']
: [layer['ows:Metadata']];
Object.values(metadataItems).forEach((item) => {
const schemaVersion = item._attributes['xlink:role']
if (schemaVersion === 'http://earthdata.nasa.gov/gibs/metadata-type/layer/1.0') {
if (argv.mode === 'verbose') console.trace(` Processing Metadata: ${item._attributes['xlink:href']}`)
const vectorDataLink = item._attributes['xlink:href']
const vectorDataFile = path.basename(vectorDataLink)
const vectorDataId = path.parse(vectorDataFile).name
vectordata[vectorDataId] = vectorDataLink
}
})
}
}
async function processLayer (layer) {
const ident = layer['ows:Identifier']._text
if (argv.mode === 'verbose') console.trace(`Processing layer ${ident}...`)
if (layer['ows:Metadata']) {
const metadataItems = Array.isArray(layer['ows:Metadata'])
? layer['ows:Metadata']
: [layer['ows:Metadata']];
if (config.skipPalettes) {
console.warn(`${prog}: WARN: Skipping palette for ${ident} \n`)
} else {
Object.values(metadataItems).forEach((item) => {
if (argv.mode === 'verbose') console.trace(` Processing pallette: ${item._attributes['xlink:href']}`)
const schemaVersion = item._attributes['xlink:role']
if (schemaVersion === 'http://earthdata.nasa.gov/gibs/metadata-type/colormap/1.3') {
const colormapLink = item._attributes['xlink:href']
const colormapFile = path.basename(colormapLink)
const colormapId = path.parse(colormapFile).name
colormaps[colormapId] = colormapLink
} else if (schemaVersion === 'http://earthdata.nasa.gov/gibs/metadata-type/mapbox-gl-style/1.0') {
const vectorStyleLink = item._attributes['xlink:href']
const vectorStyleFile = path.basename(vectorStyleLink)
const vectorStyleId = path.parse(vectorStyleFile).name
vectorstyles[vectorStyleId] = vectorStyleLink
}
})
}
}
}
async function processGetCapabilities (outputFile) {
const xml = fs.readFileSync(outputFile, { encoding: 'utf-8' })
const gc = JSON.parse(convert.xml2json(xml, { compact: true, spaces: 2 }))
try {
if (!gc.Capabilities || !gc.Capabilities.Contents) {
throw new Error(`error: ${outputFile}: no layers`)
}
const layers = gc.Capabilities.Contents.Layer
Object.values(layers).forEach((layer) => {
processLayer(layer)
processVectorData(layer)
})
} catch (error) {
throw new Error(`ERROR: ${outputFile}: ${error}`)
}
}
async function processMetadata (link, dir, ext, count) {
// if (count) console.warn(`retry #${count} for ${link}`)
try {
const outputFile = path.join(dir, path.basename(link))
if (link.endsWith(ext)) {
await fetchConfigs(link, outputFile)
}
} catch (error) {
handleException(error, link, dir, ext, count)
}
}
async function gatherProcess (type, typeStr, dir, ext) {
console.warn(`${prog}: Fetching ${Object.keys(type).length} ${typeStr}`)
if (!fs.existsSync(dir)) {
fs.mkdirSync(dir)
}
const promises = Object.values(type).map((link) => processMetadata(link, dir, ext))
await Promise.allSettled(promises)
}
main().catch((err) => {
console.error(err.stack)
})