Skip to content

Interactive interface for browsing global, full-resolution satellite imagery

Convert extractConfigFromWMTS, processVectorStyle/Data, wip processColomap

ZachTRiced069bbedd57f
diff --git a/package-lock.json b/package-lock.json
index c969603ec..678641023 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -146,7 +146,7 @@
"yargs": "^17.6.2"
},
"engines": {
- "node": ">= 16.18.0"
+ "node": ">= 16"
}
},
"node_modules/@ampproject/remapping": {
diff --git a/tasks/build/extractConfigFromWMTS.js b/tasks/build/extractConfigFromWMTS.js
index a00e4367b..4b4fed33a 100644
--- a/tasks/build/extractConfigFromWMTS.js
+++ b/tasks/build/extractConfigFromWMTS.js
@@ -50,10 +50,11 @@ if (!fs.existsSync(outputDir)) {
const tolerant = config.tolerant
const entries = config['wv-options-wmts']
const skip = config.skip || []
+const wvMatrixSets = {}
-const totalLayerCount = 0
+let totalLayerCount = 0
let totalWarningCount = 0
-const totalErrorCount = 0
+let totalErrorCount = 0
class SkipException extends Error {
constructor (message) {
@@ -64,16 +65,15 @@ class SkipException extends Error {
async function main () {
for (entry of entries) {
- // const { errorCount, warningCount, layerCount } = processEntry(entry)
- await processEntry(entry)
+ let { errorCount, warningCount, layerCount } = await processEntry(entry)
+ // await processEntry(entry)
+ console.warn(`${prog}: ${errorCount} errors, ${warningCount} warnings, ${layerCount} layers for ${entry.source}`)
+ totalErrorCount += errorCount
+ totalWarningCount += warningCount
+ totalLayerCount += layerCount
}
- console.warn(`
- ${prog}:
- ${totalErrorCount} errors,
- ${totalWarningCount} warnings,
- ${totalLayerCount} layers
- `)
+ console.warn(`${prog}:${totalErrorCount} errors, ${totalWarningCount} warnings, ${totalLayerCount} layers`)
if (totalErrorCount > 0) {
throw new Error(`${prog}: Error: ${totalErrorCount} errors occured`)
@@ -203,7 +203,7 @@ async function processEntry (entry) {
layers: {},
sources: {}
}
- const wvMatrixSets = {}
+
wv.sources[entry.source] = {
matrixSets: wvMatrixSets
}
@@ -222,7 +222,7 @@ async function processEntry (entry) {
process.stderr.write(`${prog}: ERROR: [${inputFile}] Unable to get GC: ${e}\n`)
errorCount += 1
}
- return [errorCount, warningCount, layerCount]
+ return { errorCount, warningCount, layerCount }
}
const gcContents = gc.Capabilities.Contents
@@ -231,7 +231,7 @@ async function processEntry (entry) {
if (!gcContents || !gcContents.Layer) {
errorCount += 1
process.stderr.write(`${prog}: ERROR: [${gcId}] No layers\n`)
- return [errorCount, warningCount, layerCount]
+ return { errorCount, warningCount, layerCount }
}
// if (typeof gc.Capabilities.Contents.Layer === 'object') {
@@ -268,13 +268,14 @@ async function processEntry (entry) {
}
// }
- if (typeof gcContents.TileMatrixSet === 'object') {
+ if (gcContents.TileMatrixSet === "Object") {
processMatrixSet(gcContents.TileMatrixSet)
} else {
gcContents.TileMatrixSet.forEach(gcMatrixSet => {
processMatrixSet(gcMatrixSet)
})
}
+ return { errorCount, warningCount, layerCount }
}
function processMatrixSet (gcMatrixSet) {
diff --git a/tasks/build/processColormap.js b/tasks/build/processColormap.js
old mode 100644
new mode 100755
index fef5d3bbd..48fe70671
--- a/tasks/build/processColormap.js
+++ b/tasks/build/processColormap.js
@@ -42,6 +42,47 @@ const inputDir = argv.inputDir
const outputDir = argv.outputDir
const skips = config.skipPalettes || []
+let errorCount = 0
+let fileCount = 0
+
+const walk = (dir) => {
+ let results = []
+ const list = fs.readdirSync(dir)
+ list.forEach((file) => {
+ file = `${dir}/${file}`
+ const stat = fs.statSync(file)
+ if (stat && stat.isDirectory()) {
+ /* Recurse into a subdirectory */
+ results = results.concat(walk(file))
+ } else {
+ const fileType = file.split('.').pop()
+ if (fileType === 'xml') {
+ results.push(file)
+ }
+ }
+ })
+ return results
+}
+
+async function main() {
+ const files = await walk(inputDir)
+ for (const file of files) {
+ try {
+ const { id, xml } = await readFileAsync(file)
+ await processFile(id, xml)
+ fileCount += 1
+ } catch (error) {
+ console.error(`${prog}: ERROR: ${error.message}`)
+ errorCount += 1
+ }
+ }
+
+ console.warn(`${prog}: ${errorCount} error(s), ${fileCount} file(s)`)
+
+ if (errorCount > 0) {
+ throw new Error(`${prog}: Error: ${errorCount} errors occured`)
+ }
+}
function toList (v) {
return Array.isArray(v) ? v : [v]
@@ -51,16 +92,16 @@ function matchLegend (entry, legends) {
try {
let matched = 'false'
for (const legend of legends) {
- if (!legend['@id']) {
+ if (!legend._attributes.id) {
throw new Error('No legend IDs')
}
- if (legend['@id'] === entry['@ref']) {
+ if (legend._attributes.id === entry._attributes.ref) {
matched = legend
}
}
return matched
} catch (e) {
- throw new Error(`Invalid reference: ${entry['@ref']}`)
+ throw new Error(`Invalid reference: ${entry._attributes.ref}`)
}
}
@@ -68,9 +109,8 @@ function processEntries (colormap) {
const entries = toList(colormap.Entries.ColorMapEntry)
let transparentMap = 'true'
-
for (const entry of entries) {
- if (entry['@transparent'] === 'false') {
+ if (entry._attributes.transparent === 'false') {
transparentMap = 'false'
}
}
@@ -81,7 +121,7 @@ function processEntries (colormap) {
if (!colormap.Legend) {
throw new Error('No Legend')
}
- const mapType = colormap.Legend['@type']
+ const mapType = colormap.Legend._attributes.type
const legends = toList(colormap.Legend.LegendEntry)
const colors = []
const values = []
@@ -95,25 +135,25 @@ function processEntries (colormap) {
const entry = entries[index]
const legend = matchLegend(entry, legends)
if (legend === 'false') {
- refSkipList.push(entry['@ref'])
+ refSkipList.push(entry._attributes.ref)
continue
}
- const [r, g, b] = entry['@rgb'].split(',')
+ const [r, g, b] = entry._attributes.rgb.split(',')
let a = 0
- if (entry.get('@transparent', 'false') === 'true') {
+ if (entry._attributes.transparent === 'true') {
a = 255
}
if (a === 0) {
- refSkipList.push(entry['@ref'])
+ refSkipList.push(entry._attributes.ref)
continue
}
- if (!entry['@ref']) {
+ if (!entry._attributes.ref) {
throw new Error('No ref in legend')
}
- refsList.push(entry['@ref'])
+ refsList.push(entry._attributes.ref)
colors.push(colorFormat.format(parseInt(r, 10), parseInt(g, 10), parseInt(b, 10), a))
if (mapType === 'continuous' || mapType === 'discrete') {
- const items = entry['@value'].replace(/[()[\]]/g, '').split(',')
+ const items = entry._attributes['value'].replace(/[()[\]]/g, '').split(',')
try {
const newItems = []
for (const item of items) {
@@ -127,27 +167,27 @@ function processEntries (colormap) {
newItems.push(v)
}
} catch (error) {
- throw new Error(`Invalid value: ${entry['@value']}`)
+ throw new Error(`Invalid value: ${entry._attributes.value}`)
}
let skipIndex = 0
const idList = []
for (const [index, entry] of legends.entries()) {
- if (refSkipList.includes(entry['@id'])) {
+ if (refSkipList.includes(entry._attributes.id)) {
skipIndex += 1
continue
}
- const [r, g, b] = entry['@rgb'].split(',')
+ const [r, g, b] = entry._attributes['rgb'].split(',')
legendColors.push(colorFormat.format(parseInt(r, 10), parseInt(g, 10), parseInt(b, 10), 255))
- if (!entry['@tooltip']) {
+ if (!entry._attributes.tooltip) {
throw new Error('No tooltips in legend')
}
- tooltips.push(entry['@tooltip'])
- if (!entry['@id']) {
+ tooltips.push(entry._attributes.tooltip)
+ if (!entry._attributes.id) {
throw new Error('No id in legend')
}
- idList.push(entry['@id'])
- if (entry['@showTick']) {
+ idList.push(entry._attributes.id)
+ if (entry._attributes.showTick) {
ticks.push(index - skipIndex)
}
}
@@ -188,26 +228,25 @@ async function readFileAsync (file) {
async function processFile (id, xml) {
const document = JSON.parse(convert.xml2json(xml, { compact: true, spaces: 2 }))
-
const colormaps = toList(document.ColorMaps.ColorMap)
const maps = []
for (const colormap of colormaps) {
- const result = processEntries(colormap)
+ const result = await processEntries(colormap)
if (result === 'transparent') {
continue
}
- result.title = colormap['@title']
- result.entries.title = colormap['@title']
- result.legend.title = colormap['@title']
+ result.title = colormap._attributes.title
+ result.entries.title = colormap._attributes.title
+ result.legend.title = colormap._attributes.title
result.legend.id = `${id}_${maps.length}_legend`
- if ('@minLabel' in colormap.Legend) {
- result.legend.minLabel = colormap.Legend['@minLabel']
+ if ('minLabel' in colormap.Legend._attributes) {
+ result.legend.minLabel = colormap.Legend._attributes.minLabel
}
- if ('@maxLabel' in colormap.Legend) {
- result.legend.maxLabel = colormap.Legend['@maxLabel']
+ if ('maxLabel' in colormap.Legend._attributes) {
+ result.legend.maxLabel = colormap.Legend._attributes.maxLabel
}
- if ('@units' in colormap) {
- result.legend.units = colormap['@units']
+ if ('units' in colormap._attributes) {
+ result.legend.units = colormap._attributes.units
}
maps.push(result)
}
@@ -225,27 +264,6 @@ async function processFile (id, xml) {
await writeFile(outputFile, JSON.stringify(data, null, jsonOptions.indent), { encoding: 'utf-8' })
}
-async function main () {
- const files = await readdir(inputDir)
- for (const file of files) {
- const inputFile = path.join(inputDir, file)
- try {
- const { id, xml } = await readFileAsync(inputFile)
- await processFile(id, xml)
- fileCount += 1
- } catch (e) {
- console.error(`${prog}: ERROR: [${id}] ${e.message}`)
- errorCount += 1
- }
- }
-
- console.log(`${prog}: ${errorCount} error(s), ${fileCount} file(s)`)
-
- if (errorCount > 0) {
- throw new Error(`${prog}: Error: ${errorCount} errors occured`)
- }
-}
-
main().catch((err) => {
console.error(err.stack)
})
diff --git a/tasks/build/processVectorData.js b/tasks/build/processVectorData.js
index e977cc46a..78c9f4fbc 100644
--- a/tasks/build/processVectorData.js
+++ b/tasks/build/processVectorData.js
@@ -37,15 +37,13 @@ async function main () {
let fileCount = 0
let errorCount = 0
- for (const files of inputDir) {
- for (const file of fs.readdirSync(files)) {
- try {
- fileCount += 1
- await copyFileAsync(file)
- } catch (error) {
- errorCount += 1
- console.error(`${prog}: ERROR: [${file}] ${e}\n.`)
- }
+ for (const file of fs.readdirSync(inputDir)) {
+ try {
+ fileCount += 1
+ await copyFileAsync(file)
+ } catch (error) {
+ errorCount += 1
+ console.error(`${prog}: ERROR: [${file}] ${error}\n.`)
}
}
@@ -57,18 +55,18 @@ async function main () {
}
async function copyFileAsync (file) {
- if (inputFile.endsWith('.json')) {
+ if (file.endsWith('.json')) {
const responseData = {}
const vectorLayerFilename = file
const vectorLayerId = vectorLayerFilename.split('.json', 1)[0]
responseData.vectorData = {}
responseData.vectorData[vectorLayerId] = {}
- const initialData = JSON.parse(await readFile(inputFile, 'utf-8'))
+ const initialData = JSON.parse(await readFile(`${inputDir}/${file}`, 'utf-8'))
for (const i in initialData) {
responseData.vectorData[vectorLayerId][i] = initialData[i]
}
- await writeFile(inputFile, JSON.stringify(responseData, null, 2), 'utf-8')
- await copyFile(inputFile, outputDir)
+ await writeFile(`${inputDir}/${file}`, JSON.stringify(responseData, null, 2), 'utf-8')
+ await copyFile(`${inputDir}/${file}`, `${outputDir}/${file}`)
}
}
diff --git a/tasks/build/processVectorStyles.js b/tasks/build/processVectorStyles.js
index 53e26895e..bfcff1e30 100644
--- a/tasks/build/processVectorStyles.js
+++ b/tasks/build/processVectorStyles.js
@@ -37,15 +37,13 @@ async function main () {
let fileCount = 0
let errorCount = 0
- for (const files of inputDir) {
- for (const file of fs.readdirSync(files)) {
- try {
- fileCount += 1
- await copyFileAsync(file)
- } catch (error) {
- errorCount += 1
- console.error(`${prog}: ERROR: [${file}] ${e}\n.`)
- }
+ for (const file of fs.readdirSync(inputDir)) {
+ try {
+ fileCount += 1
+ await copyFileAsync(file)
+ } catch (error) {
+ errorCount += 1
+ console.error(`${prog}: ERROR: [${file}] ${error}\n.`)
}
}
@@ -57,18 +55,18 @@ async function main () {
}
async function copyFileAsync (file) {
- if (inputFile.endsWith('.json')) {
+ if (file.endsWith('.json')) {
const responseData = {}
const vectorLayerFilename = file
const vectorLayerId = vectorLayerFilename.split('.json', 1)[0]
responseData.vectorStyles = {}
responseData.vectorStyles[vectorLayerId] = {}
- const initialData = JSON.parse(await readFile(inputFile, 'utf-8'))
+ const initialData = JSON.parse(await readFile(`${inputDir}/${file}`, 'utf-8'))
for (const i in initialData) {
responseData.vectorStyles[vectorLayerId][i] = initialData[i]
}
- await writeFile(inputFile, JSON.stringify(responseData, null, 2), 'utf-8')
- await copyFile(inputFile, outputDir)
+ await writeFile(`${inputDir}/${file}`, JSON.stringify(responseData, null, 2), 'utf-8')
+ await copyFile(`${inputDir}/${file}`, `${outputDir}/${file}`)
}
}
diff --git a/tasks/buildOptions.sh b/tasks/buildOptions.sh
index 7d2a67969..26f9d5634 100755
--- a/tasks/buildOptions.sh
+++ b/tasks/buildOptions.sh
@@ -68,23 +68,23 @@ mkdir -p "$BUILD_DIR/colormaps"
# fi
# Validate layers in wv.json with a JSON schema
-`node $SCRIPTS_DIR/validateConfigs.js \
- --inputDirectory "$SRC_DIR/common/config/wv.json/layers" \
- --schemaFile "$BASE/schemas/layer-config.json"`
+# `node $SCRIPTS_DIR/validateConfigs.js \
+# --inputDirectory "$SRC_DIR/common/config/wv.json/layers" \
+# --schemaFile "$BASE/schemas/layer-config.json"`
-if [ -e "$BUILD_DIR/features.json" ] ; then
- cp "$BUILD_DIR/features.json" "$BUILD_DIR/config/wv.json/_features.json"
-fi
+# if [ -e "$BUILD_DIR/features.json" ] ; then
+# cp "$BUILD_DIR/features.json" "$BUILD_DIR/config/wv.json/_features.json"
+# fi
-# # Run extractConfigFromWMTS.js script with config.json
-if [ -e "$BUILD_DIR/config.json" ] ; then
- `node $SCRIPTS_DIR/extractConfigFromWMTS.js \
- --config "$BUILD_DIR/config.json" \
- --inputDir "$BUILD_DIR/gc" \
- --outputDir "$BUILD_DIR/_wmts"`
-fi
+# # # Run extractConfigFromWMTS.js script with config.json
+# if [ -e "$BUILD_DIR/config.json" ] ; then
+# `node $SCRIPTS_DIR/extractConfigFromWMTS.js \
+# --config "$BUILD_DIR/config.json" \
+# --inputDir "$BUILD_DIR/gc" \
+# --outputDir "$BUILD_DIR/_wmts"`
+# fi
-# # Run processVectorStyles.js and move vectorstyles where we want them
+# Run processVectorStyles.js and move vectorstyles where we want them
if [ -e "$BUILD_DIR/gc/vectorstyles" ] ; then
mkdir -p "$BUILD_DIR/config/wv.json/vectorstyles"
`node $SCRIPTS_DIR/processVectorStyles.js \
@@ -112,51 +112,51 @@ if [ -e "$BUILD_DIR/colormaps" ] ; then
--outputDir "$BUILD_DIR/config/palettes"`
fi
-# Throw error if no categoryGroupOrder.json file present
-if [ ! -e "$BUILD_DIR/config/wv.json/categoryGroupOrder.json" ] ; then
- echo "categoryGroupOrder.json not found. Generating..."
- `node $SCRIPTS_DIR/generateCategoryGroupOrder.js \
- --inputDir "$SRC_DIR/common/config/wv.json/categories/" \
- --outputDir "$SRC_DIR/common/config/wv.json/"`
-fi
+# # Throw error if no categoryGroupOrder.json file present
+# if [ ! -e "$BUILD_DIR/config/wv.json/categoryGroupOrder.json" ] ; then
+# echo "categoryGroupOrder.json not found. Generating..."
+# `node $SCRIPTS_DIR/generateCategoryGroupOrder.js \
+# --inputDir "$SRC_DIR/common/config/wv.json/categories/" \
+# --outputDir "$SRC_DIR/common/config/wv.json/"`
+# fi
-if [ -e "$OPT_DIR/$OPT_SUBDIR/layer-metadata/all.json" ] ; then
- cp "$OPT_DIR/$OPT_SUBDIR/layer-metadata/all.json" "$BUILD_DIR/config/wv.json/layer-metadata.json"
-fi
+# if [ -e "$OPT_DIR/$OPT_SUBDIR/layer-metadata/all.json" ] ; then
+# cp "$OPT_DIR/$OPT_SUBDIR/layer-metadata/all.json" "$BUILD_DIR/config/wv.json/layer-metadata.json"
+# fi
-# Run mergeConfig.js on all directories in /config
-configs=$(ls "$BUILD_DIR/config")
-for config in $configs; do
- case $config in
- *.json)
- `node $SCRIPTS_DIR/mergeConfig.js \
- --inputFile "$BUILD_DIR/config/$config" \
- --outputFile "$DEST_DIR/config/$config"`
- ;;
- *)
- cp -r "$BUILD_DIR/config/$config" "$DEST_DIR/config/$config"
- ;;
- esac
-done
-
-# Run mergeConfigWithWMTS.js to merge layer metadata from WMTS GC with worldview layer configs into wv.json
-`node $SCRIPTS_DIR/mergeConfigWithWMTS.js \
- --inputDir "$BUILD_DIR/_wmts" \
- --outputDir "$DEST_DIR/config/wv.json"`
-
-# Copy brand files from build to dest
-cp -r "$BUILD_DIR/brand" "$DEST_DIR"
-cp "$BUILD_DIR/brand.json" "$DEST_DIR"
-
-# Validate the options build
-`node $SCRIPTS_DIR/validateOptions.js" \
- --inputFile "$BUILD_DIR/config.json" \
- --outputDir "$DEST_DIR/config`
-
-# Fetch preview images from WV Snapshots for any layers which they are missing
-`node $SCRIPTS_DIR/fetchPreviewSnapshots.js" \
- --inputFile "$DEST_DIR/config/wv.json" \
- --outputFile "$OPT_DIR/common/previewLayerOverrides.json" \
- --featuresFile "$BUILD_DIR/features.json"`
+# # Run mergeConfig.js on all directories in /config
+# configs=$(ls "$BUILD_DIR/config")
+# for config in $configs; do
+# case $config in
+# *.json)
+# `node $SCRIPTS_DIR/mergeConfig.js \
+# --inputFile "$BUILD_DIR/config/$config" \
+# --outputFile "$DEST_DIR/config/$config"`
+# ;;
+# *)
+# cp -r "$BUILD_DIR/config/$config" "$DEST_DIR/config/$config"
+# ;;
+# esac
+# done
+
+# # Run mergeConfigWithWMTS.js to merge layer metadata from WMTS GC with worldview layer configs into wv.json
+# `node $SCRIPTS_DIR/mergeConfigWithWMTS.js \
+# --inputDir "$BUILD_DIR/_wmts" \
+# --outputDir "$DEST_DIR/config/wv.json"`
+
+# # Copy brand files from build to dest
+# cp -r "$BUILD_DIR/brand" "$DEST_DIR"
+# cp "$BUILD_DIR/brand.json" "$DEST_DIR"
+
+# # Validate the options build
+# `node $SCRIPTS_DIR/validateOptions.js" \
+# --inputFile "$BUILD_DIR/config.json" \
+# --outputDir "$DEST_DIR/config`
+
+# # Fetch preview images from WV Snapshots for any layers which they are missing
+# `node $SCRIPTS_DIR/fetchPreviewSnapshots.js" \
+# --inputFile "$DEST_DIR/config/wv.json" \
+# --outputFile "$OPT_DIR/common/previewLayerOverrides.json" \
+# --featuresFile "$BUILD_DIR/features.json"`
exit 0