Skip to content

Interactive interface for browsing global, full-resolution satellite imagery

11.1 KB
Raw Blame
const fs = require('fs')
const path = require('path')
const convert = require('xml-js')
const { promisify } = require('util')
const yargs = require('yargs')
const { hideBin } = require('yargs/helpers')

const readFile = promisify(fs.readFile)
const writeFile = promisify(fs.writeFile)

const prog = path.basename(__filename)

const options = yargs(hideBin(process.argv))
  .usage('Usage: $0 [config] [inputDir] [outputDir] [layersDir]')
  .option('config', {
    demandOption: true,
    alias: 'c',
    type: 'string',
    description: 'config file'
  })
  .option('inputDir', {
    demandOption: true,
    alias: 'i',
    type: 'string',
    description: 'getcapabilities input directory'
  })
  .option('outputDir', {
    demandOption: true,
    alias: 'o',
    type: 'string',
    description: 'wmts output directory'
  })
  .option('layersDir', {
    demandOption: true,
    alias: 'l',
    type: 'string',
    description: 'wmts layers directory'
  })
  .epilog('Converts colormaps to JSON files')

const { argv } = options
if (!argv.config && !argv.inputDir && !argv.outputDir && !argv.layersDir) {
  throw new Error('Invalid number of arguments')
}

const config = argv.config
const inputDir = argv.inputDir
const outputDir = argv.outputDir
const layersDir = argv.layersDir

const skips = config.skipPalettes || []
let errorCount = 0
let fileCount = 0

const colormapTypes = {}

const walk = (dir) => {
  let results = []
  const list = fs.readdirSync(dir)
  list.forEach((file) => {
    file = `${dir}/${file}`
    const stat = fs.statSync(file)
    if (stat && stat.isDirectory()) {
      /* Recurse into a subdirectory */
      results = results.concat(walk(file))
    } else {
      const fileType = file.split('.').pop()
      if (fileType === 'xml') {
        results.push(file)
      }
    }
  })
  return results
}

async function main () {
  const files = await walk(inputDir)
  for (const file of files) {
    try {
      const { id, xml } = await readFileAsync(file)
      await processFile(id, xml)
      fileCount += 1
    } catch (error) {
      console.error(error)

      console.error(`${prog}: ERROR: ${error}`)
      errorCount += 1
    }
  }

  try {
    const wmtsFiles = fs.readdirSync(layersDir)
    for (const file of wmtsFiles) {
      if (!file.endsWith('.json')) continue
      const data = await fs.promises.readFile(path.join(layersDir, file), 'utf8')
      const jsonData = JSON.parse(data)
      Object.keys(jsonData.layers).forEach((id) => {
        const hasPaletteId = Object.prototype.hasOwnProperty.call(jsonData.layers[id], 'palette') &&
          Object.prototype.hasOwnProperty.call(jsonData.layers[id].palette, 'id')
        const colormapId = hasPaletteId ? jsonData.layers[id].palette.id : id
        if (Object.prototype.hasOwnProperty.call(colormapTypes, colormapId)) jsonData.layers[id].colormapType = colormapTypes[colormapId]
      })
      await writeFile(path.join(layersDir, file), JSON.stringify(jsonData, null, 2), { encoding: 'utf-8' })
    }
  } catch (error) {
    console.error(`${prog}: ERROR: ${error}`)
  }

  console.warn(`${prog}: ${errorCount} error(s), ${fileCount} file(s)`)

  if (errorCount > 0) {
    throw new Error(`${prog}: Error: ${errorCount} errors occured`)
  }
}

function toList (v) {
  return Array.isArray(v) ? v : [v]
}

/*
 * matchLegend: Determines if the given entry has a matching legend.id
 *
 * Parameters:
 * entry   [object] Single ColorMapEntry Object for this product
 * legends [object] collection of legends for this product
 *
 * Returns the matching legend if found
*/
async function matchLegend (entry, legends) {
  try {
    let matched = 'false'
    for (const legend of legends) {
      if (!legend._attributes.id) {
        throw new Error('No legend IDs')
      }
      if (legend._attributes.id === entry._attributes.ref) {
        matched = legend
      }
    }
    return matched
  } catch (e) {
    throw new Error(`Invalid reference: ${entry._attributes.ref}`)
  }
}

async function processEntries (colormap) {
  const entries = toList(colormap.Entries.ColorMapEntry)
  let transparentMap = 'true'

  // Check to see if there is at least one non-transparent entry.
  for (const entry of entries) {
    if (entry._attributes.transparent === 'false') {
      transparentMap = 'false'
      break
    }
  }

  if (transparentMap === 'true') {
    return 'transparent'
  }

  if (!colormap.Legend) {
    throw new Error('No Legend')
  }
  const mapType = colormap.Legend._attributes.type
  const legends = toList(colormap.Legend.LegendEntry)
  const colors = []
  const values = []
  const ticks = []
  const tooltips = []
  const legendColors = []
  const refsList = []
  const refSkipList = []
  let initializeDisabled = []

  // TODO: make this a separate function for entries?
  await Promise.all(
    entries.map(async (entry, index) => {
      const legend = await matchLegend(entry, legends)

      if (legend === 'false') {
        refSkipList.push(entry._attributes.ref)
        return
      }

      const [r, g, b] = entry._attributes.rgb.split(',')

      // Force alpha value to 255 always
      const a = 255

      // If entry is served transparent, add it to the disabled array
      if (entry._attributes.transparent !== 'false' && mapType === 'classification') {
        initializeDisabled.push(refsList.length)
      }

      if (!entry._attributes.ref) {
        throw new Error('No ref in legend')
      }

      if (mapType === 'classification') {
        refsList.push(index)
      } else {
        refsList.push(entry._attributes.ref)
      }
      const rHex = parseInt(r).toString(16).padStart(2, '0')
      const gHex = parseInt(g).toString(16).padStart(2, '0')
      const bHex = parseInt(b).toString(16).padStart(2, '0')
      const aHex = parseInt(a).toString(16).padStart(2, '0')
      const colorString = rHex + gHex + bHex + aHex

      colors.push(colorString)

      if (mapType === 'continuous' || mapType === 'discrete') {
        const items = entry._attributes.value.replace(/[()[\]]/g, '').split(',')
        try {
          const newItems = []
          for (const item of items) {
            let v = parseFloat(item)
            if (v === Number.POSITIVE_INFINITY) {
              v = Number.MAX_VALUE
            }
            if (v === Number.NEGATIVE_INFINITY) {
              v = Number.MIN_VALUE
            }
            newItems.push(v)
          }
          values.push(newItems)
        } catch (error) {
          throw new Error(`Invalid value: ${entry._attributes.value}`)
        }
      }
    })
  )

  // TODO: make this a separate function for legends?
  let skipIndex = 0
  const idList = []
  await Promise.all(
    legends.map(async (entry, index) => {
      if (refSkipList.includes(entry._attributes.id)) {
        skipIndex = skipIndex + 1
        return
      }
      const [r, g, b] = entry._attributes.rgb.split(',')

      // TODO: turn this into reusable function
      const rHex = parseInt(r).toString(16).padStart(2, '0')
      const gHex = parseInt(g).toString(16).padStart(2, '0')
      const bHex = parseInt(b).toString(16).padStart(2, '0')
      const aHex = parseInt(255).toString(16).padStart(2, '0')
      const colorString = rHex + gHex + bHex + aHex

      legendColors.push(colorString)

      if (!entry._attributes.tooltip) {
        throw new Error('No tooltips in legend')
      }

      tooltips.push(entry._attributes.tooltip)

      if (!entry._attributes.id) {
        throw new Error('No id in legend')
      }

      idList.push(entry._attributes.id)

      if (entry._attributes.showTick) {
        ticks.push(skipIndex - 1)
      }
    })
  )

  // Ensure all disabled colormap entries are included at the END
  // Update colors, legendColors & tooltips arrays accordingly
  if (mapType === 'classification') {
    const numDisabled = initializeDisabled.length
    if (numDisabled > 0) {
      const totalEntries = colors.length
      const disabledColorsArr = []
      const disabledLegendsArr = []
      const disabledTooltipsArr = []

      // Iterate backwards to maintain early indices
      for (let i = numDisabled - 1; i >= 0; i -= 1) {
        const thisIndex = initializeDisabled[i]

        disabledColorsArr.push(colors[thisIndex])
        disabledLegendsArr.push(legendColors[thisIndex])
        disabledTooltipsArr.push(tooltips[thisIndex])

        colors.splice(thisIndex, 1)
        legendColors.splice(thisIndex, 1)
        tooltips.splice(thisIndex, 1)
      }

      colors.push(...disabledColorsArr)
      legendColors.push(...disabledLegendsArr)
      tooltips.push(...disabledTooltipsArr)

      const firstDisabledIndex = totalEntries - numDisabled
      initializeDisabled = Array.from({
        length: numDisabled
      }, (item, index) => firstDisabledIndex + index)
    }
  }

  const result = {
    type: mapType,
    entries: {
      type: mapType,
      colors,
      refs: refsList
    },
    legend: {
      colors: legendColors,
      type: mapType,
      tooltips,
      ticks,
      refs: idList
    },
    disabled: initializeDisabled
  }
  if (mapType === 'continuous' || mapType === 'discrete') {
    result.entries.values = values
  }

  return result
}

async function readFileAsync (file) {
  const id = path.basename(file, path.extname(file))
  if (skips.includes(id)) {
    console.error(`${prog}:  WARN: [${file}] Skipping`)
    return
  }
  const xml = await readFile(file, { encoding: 'utf-8' })
  return { id, xml }
}

/*
 * processFile: Process provided xml file, determine relevant data & write data
 * This determines the colormap & Palette legend data used in WV Layer window
 *
 * Parameters:
 * id  [string] representing this product's layer name
 * xml [string] represention of this product's xml colormap file (served from GIBS)
*/
async function processFile (id, xml) {
  let document
  let colormaps = []
  try {
    document = JSON.parse(convert.xml2json(xml, { compact: true, spaces: 2 }))
    if (document && document.ColorMaps && document.ColorMaps.ColorMap) {
      colormaps = await toList(document.ColorMaps.ColorMap)
    }
    const maps = []
    for (const colormap of colormaps) {
      const result = await processEntries(colormap)
      if (result === 'transparent') {
        // There are no visible colors in the colormap so stop processing
        continue
      }
      result.title = colormap._attributes.title
      result.entries.title = colormap._attributes.title
      result.legend.title = colormap._attributes.title
      result.legend.id = `${id}_${maps.length}_legend`
      if ('minLabel' in colormap.Legend._attributes) {
        result.legend.minLabel = colormap.Legend._attributes.minLabel
      }
      if ('maxLabel' in colormap.Legend._attributes) {
        result.legend.maxLabel = colormap.Legend._attributes.maxLabel
      }
      if ('units' in colormap._attributes) {
        result.legend.units = colormap._attributes.units
      }
      maps.push(result)
    }

    const data = {
      id,
      maps
    }

    colormapTypes[id] = maps[0].legend.type

    const outputFile = path.join(outputDir, `${id}.json`)
    await writeFile(outputFile, JSON.stringify(data, null, 2), { encoding: 'utf-8' })
  } catch (error) {
    console.error(id)
    console.error(error)
  }
}

main().catch((err) => {
  console.error(err.stack)
})