Interactive interface for browsing global, full-resolution satellite imagery
Convert extractConfigFromWMTS, processVectorStyle/Data, wip processColomap
ZachTRiced069bbedd57f
diff --git a/package-lock.json b/package-lock.jsonindex c969603ec..678641023 100644--- a/package-lock.json+++ b/package-lock.json@@ -146,7 +146,7 @@"yargs": "^17.6.2"},"engines": {- "node": ">= 16.18.0"+ "node": ">= 16"}},"node_modules/@ampproject/remapping": {diff --git a/tasks/build/extractConfigFromWMTS.js b/tasks/build/extractConfigFromWMTS.jsindex a00e4367b..4b4fed33a 100644--- a/tasks/build/extractConfigFromWMTS.js+++ b/tasks/build/extractConfigFromWMTS.js@@ -50,10 +50,11 @@ if (!fs.existsSync(outputDir)) {const tolerant = config.tolerantconst entries = config['wv-options-wmts']const skip = config.skip || []+const wvMatrixSets = {}-const totalLayerCount = 0+let totalLayerCount = 0let totalWarningCount = 0-const totalErrorCount = 0+let totalErrorCount = 0class SkipException extends Error {constructor (message) {@@ -64,16 +65,15 @@ class SkipException extends Error {async function main () {for (entry of entries) {- // const { errorCount, warningCount, layerCount } = processEntry(entry)- await processEntry(entry)+ let { errorCount, warningCount, layerCount } = await processEntry(entry)+ // await processEntry(entry)+ console.warn(`${prog}: ${errorCount} errors, ${warningCount} warnings, ${layerCount} layers for ${entry.source}`)+ totalErrorCount += errorCount+ totalWarningCount += warningCount+ totalLayerCount += layerCount}- console.warn(`- ${prog}:- ${totalErrorCount} errors,- ${totalWarningCount} warnings,- ${totalLayerCount} layers- `)+ console.warn(`${prog}:${totalErrorCount} errors, ${totalWarningCount} warnings, ${totalLayerCount} layers`)if (totalErrorCount > 0) {throw new Error(`${prog}: Error: ${totalErrorCount} errors occured`)@@ -203,7 +203,7 @@ async function processEntry (entry) {layers: {},sources: {}}- const wvMatrixSets = {}+wv.sources[entry.source] = {matrixSets: wvMatrixSets}@@ -222,7 +222,7 @@ async function processEntry (entry) {process.stderr.write(`${prog}: ERROR: [${inputFile}] Unable to get GC: ${e}\n`)errorCount += 1}- return [errorCount, warningCount, layerCount]+ return { errorCount, warningCount, layerCount }}const gcContents = gc.Capabilities.Contents@@ -231,7 +231,7 @@ async function processEntry (entry) {if (!gcContents || !gcContents.Layer) {errorCount += 1process.stderr.write(`${prog}: ERROR: [${gcId}] No layers\n`)- return [errorCount, warningCount, layerCount]+ return { errorCount, warningCount, layerCount }}// if (typeof gc.Capabilities.Contents.Layer === 'object') {@@ -268,13 +268,14 @@ async function processEntry (entry) {}// }- if (typeof gcContents.TileMatrixSet === 'object') {+ if (gcContents.TileMatrixSet === "Object") {processMatrixSet(gcContents.TileMatrixSet)} else {gcContents.TileMatrixSet.forEach(gcMatrixSet => {processMatrixSet(gcMatrixSet)})}+ return { errorCount, warningCount, layerCount }}function processMatrixSet (gcMatrixSet) {diff --git a/tasks/build/processColormap.js b/tasks/build/processColormap.jsold mode 100644new mode 100755index fef5d3bbd..48fe70671--- a/tasks/build/processColormap.js+++ b/tasks/build/processColormap.js@@ -42,6 +42,47 @@ const inputDir = argv.inputDirconst outputDir = argv.outputDirconst skips = config.skipPalettes || []+let errorCount = 0+let fileCount = 0++const walk = (dir) => {+ let results = []+ const list = fs.readdirSync(dir)+ list.forEach((file) => {+ file = `${dir}/${file}`+ const stat = fs.statSync(file)+ if (stat && stat.isDirectory()) {+ /* Recurse into a subdirectory */+ results = results.concat(walk(file))+ } else {+ const fileType = file.split('.').pop()+ if (fileType === 'xml') {+ results.push(file)+ }+ }+ })+ return results+}++async function main() {+ const files = await walk(inputDir)+ for (const file of files) {+ try {+ const { id, xml } = await readFileAsync(file)+ await processFile(id, xml)+ fileCount += 1+ } catch (error) {+ console.error(`${prog}: ERROR: ${error.message}`)+ errorCount += 1+ }+ }++ console.warn(`${prog}: ${errorCount} error(s), ${fileCount} file(s)`)++ if (errorCount > 0) {+ throw new Error(`${prog}: Error: ${errorCount} errors occured`)+ }+}function toList (v) {return Array.isArray(v) ? v : [v]@@ -51,16 +92,16 @@ function matchLegend (entry, legends) {try {let matched = 'false'for (const legend of legends) {- if (!legend['@id']) {+ if (!legend._attributes.id) {throw new Error('No legend IDs')}- if (legend['@id'] === entry['@ref']) {+ if (legend._attributes.id === entry._attributes.ref) {matched = legend}}return matched} catch (e) {- throw new Error(`Invalid reference: ${entry['@ref']}`)+ throw new Error(`Invalid reference: ${entry._attributes.ref}`)}}@@ -68,9 +109,8 @@ function processEntries (colormap) {const entries = toList(colormap.Entries.ColorMapEntry)let transparentMap = 'true'-for (const entry of entries) {- if (entry['@transparent'] === 'false') {+ if (entry._attributes.transparent === 'false') {transparentMap = 'false'}}@@ -81,7 +121,7 @@ function processEntries (colormap) {if (!colormap.Legend) {throw new Error('No Legend')}- const mapType = colormap.Legend['@type']+ const mapType = colormap.Legend._attributes.typeconst legends = toList(colormap.Legend.LegendEntry)const colors = []const values = []@@ -95,25 +135,25 @@ function processEntries (colormap) {const entry = entries[index]const legend = matchLegend(entry, legends)if (legend === 'false') {- refSkipList.push(entry['@ref'])+ refSkipList.push(entry._attributes.ref)continue}- const [r, g, b] = entry['@rgb'].split(',')+ const [r, g, b] = entry._attributes.rgb.split(',')let a = 0- if (entry.get('@transparent', 'false') === 'true') {+ if (entry._attributes.transparent === 'true') {a = 255}if (a === 0) {- refSkipList.push(entry['@ref'])+ refSkipList.push(entry._attributes.ref)continue}- if (!entry['@ref']) {+ if (!entry._attributes.ref) {throw new Error('No ref in legend')}- refsList.push(entry['@ref'])+ refsList.push(entry._attributes.ref)colors.push(colorFormat.format(parseInt(r, 10), parseInt(g, 10), parseInt(b, 10), a))if (mapType === 'continuous' || mapType === 'discrete') {- const items = entry['@value'].replace(/[()[\]]/g, '').split(',')+ const items = entry._attributes['value'].replace(/[()[\]]/g, '').split(',')try {const newItems = []for (const item of items) {@@ -127,27 +167,27 @@ function processEntries (colormap) {newItems.push(v)}} catch (error) {- throw new Error(`Invalid value: ${entry['@value']}`)+ throw new Error(`Invalid value: ${entry._attributes.value}`)}let skipIndex = 0const idList = []for (const [index, entry] of legends.entries()) {- if (refSkipList.includes(entry['@id'])) {+ if (refSkipList.includes(entry._attributes.id)) {skipIndex += 1continue}- const [r, g, b] = entry['@rgb'].split(',')+ const [r, g, b] = entry._attributes['rgb'].split(',')legendColors.push(colorFormat.format(parseInt(r, 10), parseInt(g, 10), parseInt(b, 10), 255))- if (!entry['@tooltip']) {+ if (!entry._attributes.tooltip) {throw new Error('No tooltips in legend')}- tooltips.push(entry['@tooltip'])- if (!entry['@id']) {+ tooltips.push(entry._attributes.tooltip)+ if (!entry._attributes.id) {throw new Error('No id in legend')}- idList.push(entry['@id'])- if (entry['@showTick']) {+ idList.push(entry._attributes.id)+ if (entry._attributes.showTick) {ticks.push(index - skipIndex)}}@@ -188,26 +228,25 @@ async function readFileAsync (file) {async function processFile (id, xml) {const document = JSON.parse(convert.xml2json(xml, { compact: true, spaces: 2 }))-const colormaps = toList(document.ColorMaps.ColorMap)const maps = []for (const colormap of colormaps) {- const result = processEntries(colormap)+ const result = await processEntries(colormap)if (result === 'transparent') {continue}- result.title = colormap['@title']- result.entries.title = colormap['@title']- result.legend.title = colormap['@title']+ result.title = colormap._attributes.title+ result.entries.title = colormap._attributes.title+ result.legend.title = colormap._attributes.titleresult.legend.id = `${id}_${maps.length}_legend`- if ('@minLabel' in colormap.Legend) {- result.legend.minLabel = colormap.Legend['@minLabel']+ if ('minLabel' in colormap.Legend._attributes) {+ result.legend.minLabel = colormap.Legend._attributes.minLabel}- if ('@maxLabel' in colormap.Legend) {- result.legend.maxLabel = colormap.Legend['@maxLabel']+ if ('maxLabel' in colormap.Legend._attributes) {+ result.legend.maxLabel = colormap.Legend._attributes.maxLabel}- if ('@units' in colormap) {- result.legend.units = colormap['@units']+ if ('units' in colormap._attributes) {+ result.legend.units = colormap._attributes.units}maps.push(result)}@@ -225,27 +264,6 @@ async function processFile (id, xml) {await writeFile(outputFile, JSON.stringify(data, null, jsonOptions.indent), { encoding: 'utf-8' })}-async function main () {- const files = await readdir(inputDir)- for (const file of files) {- const inputFile = path.join(inputDir, file)- try {- const { id, xml } = await readFileAsync(inputFile)- await processFile(id, xml)- fileCount += 1- } catch (e) {- console.error(`${prog}: ERROR: [${id}] ${e.message}`)- errorCount += 1- }- }-- console.log(`${prog}: ${errorCount} error(s), ${fileCount} file(s)`)-- if (errorCount > 0) {- throw new Error(`${prog}: Error: ${errorCount} errors occured`)- }-}-main().catch((err) => {console.error(err.stack)})diff --git a/tasks/build/processVectorData.js b/tasks/build/processVectorData.jsindex e977cc46a..78c9f4fbc 100644--- a/tasks/build/processVectorData.js+++ b/tasks/build/processVectorData.js@@ -37,15 +37,13 @@ async function main () {let fileCount = 0let errorCount = 0- for (const files of inputDir) {- for (const file of fs.readdirSync(files)) {- try {- fileCount += 1- await copyFileAsync(file)- } catch (error) {- errorCount += 1- console.error(`${prog}: ERROR: [${file}] ${e}\n.`)- }+ for (const file of fs.readdirSync(inputDir)) {+ try {+ fileCount += 1+ await copyFileAsync(file)+ } catch (error) {+ errorCount += 1+ console.error(`${prog}: ERROR: [${file}] ${error}\n.`)}}@@ -57,18 +55,18 @@ async function main () {}async function copyFileAsync (file) {- if (inputFile.endsWith('.json')) {+ if (file.endsWith('.json')) {const responseData = {}const vectorLayerFilename = fileconst vectorLayerId = vectorLayerFilename.split('.json', 1)[0]responseData.vectorData = {}responseData.vectorData[vectorLayerId] = {}- const initialData = JSON.parse(await readFile(inputFile, 'utf-8'))+ const initialData = JSON.parse(await readFile(`${inputDir}/${file}`, 'utf-8'))for (const i in initialData) {responseData.vectorData[vectorLayerId][i] = initialData[i]}- await writeFile(inputFile, JSON.stringify(responseData, null, 2), 'utf-8')- await copyFile(inputFile, outputDir)+ await writeFile(`${inputDir}/${file}`, JSON.stringify(responseData, null, 2), 'utf-8')+ await copyFile(`${inputDir}/${file}`, `${outputDir}/${file}`)}}diff --git a/tasks/build/processVectorStyles.js b/tasks/build/processVectorStyles.jsindex 53e26895e..bfcff1e30 100644--- a/tasks/build/processVectorStyles.js+++ b/tasks/build/processVectorStyles.js@@ -37,15 +37,13 @@ async function main () {let fileCount = 0let errorCount = 0- for (const files of inputDir) {- for (const file of fs.readdirSync(files)) {- try {- fileCount += 1- await copyFileAsync(file)- } catch (error) {- errorCount += 1- console.error(`${prog}: ERROR: [${file}] ${e}\n.`)- }+ for (const file of fs.readdirSync(inputDir)) {+ try {+ fileCount += 1+ await copyFileAsync(file)+ } catch (error) {+ errorCount += 1+ console.error(`${prog}: ERROR: [${file}] ${error}\n.`)}}@@ -57,18 +55,18 @@ async function main () {}async function copyFileAsync (file) {- if (inputFile.endsWith('.json')) {+ if (file.endsWith('.json')) {const responseData = {}const vectorLayerFilename = fileconst vectorLayerId = vectorLayerFilename.split('.json', 1)[0]responseData.vectorStyles = {}responseData.vectorStyles[vectorLayerId] = {}- const initialData = JSON.parse(await readFile(inputFile, 'utf-8'))+ const initialData = JSON.parse(await readFile(`${inputDir}/${file}`, 'utf-8'))for (const i in initialData) {responseData.vectorStyles[vectorLayerId][i] = initialData[i]}- await writeFile(inputFile, JSON.stringify(responseData, null, 2), 'utf-8')- await copyFile(inputFile, outputDir)+ await writeFile(`${inputDir}/${file}`, JSON.stringify(responseData, null, 2), 'utf-8')+ await copyFile(`${inputDir}/${file}`, `${outputDir}/${file}`)}}diff --git a/tasks/buildOptions.sh b/tasks/buildOptions.shindex 7d2a67969..26f9d5634 100755--- a/tasks/buildOptions.sh+++ b/tasks/buildOptions.sh@@ -68,23 +68,23 @@ mkdir -p "$BUILD_DIR/colormaps"# fi# Validate layers in wv.json with a JSON schema-`node $SCRIPTS_DIR/validateConfigs.js \- --inputDirectory "$SRC_DIR/common/config/wv.json/layers" \- --schemaFile "$BASE/schemas/layer-config.json"`+# `node $SCRIPTS_DIR/validateConfigs.js \+# --inputDirectory "$SRC_DIR/common/config/wv.json/layers" \+# --schemaFile "$BASE/schemas/layer-config.json"`-if [ -e "$BUILD_DIR/features.json" ] ; then- cp "$BUILD_DIR/features.json" "$BUILD_DIR/config/wv.json/_features.json"-fi+# if [ -e "$BUILD_DIR/features.json" ] ; then+# cp "$BUILD_DIR/features.json" "$BUILD_DIR/config/wv.json/_features.json"+# fi-# # Run extractConfigFromWMTS.js script with config.json-if [ -e "$BUILD_DIR/config.json" ] ; then- `node $SCRIPTS_DIR/extractConfigFromWMTS.js \- --config "$BUILD_DIR/config.json" \- --inputDir "$BUILD_DIR/gc" \- --outputDir "$BUILD_DIR/_wmts"`-fi+# # # Run extractConfigFromWMTS.js script with config.json+# if [ -e "$BUILD_DIR/config.json" ] ; then+# `node $SCRIPTS_DIR/extractConfigFromWMTS.js \+# --config "$BUILD_DIR/config.json" \+# --inputDir "$BUILD_DIR/gc" \+# --outputDir "$BUILD_DIR/_wmts"`+# fi-# # Run processVectorStyles.js and move vectorstyles where we want them+# Run processVectorStyles.js and move vectorstyles where we want themif [ -e "$BUILD_DIR/gc/vectorstyles" ] ; thenmkdir -p "$BUILD_DIR/config/wv.json/vectorstyles"`node $SCRIPTS_DIR/processVectorStyles.js \@@ -112,51 +112,51 @@ if [ -e "$BUILD_DIR/colormaps" ] ; then--outputDir "$BUILD_DIR/config/palettes"`fi-# Throw error if no categoryGroupOrder.json file present-if [ ! -e "$BUILD_DIR/config/wv.json/categoryGroupOrder.json" ] ; then- echo "categoryGroupOrder.json not found. Generating..."- `node $SCRIPTS_DIR/generateCategoryGroupOrder.js \- --inputDir "$SRC_DIR/common/config/wv.json/categories/" \- --outputDir "$SRC_DIR/common/config/wv.json/"`-fi+# # Throw error if no categoryGroupOrder.json file present+# if [ ! -e "$BUILD_DIR/config/wv.json/categoryGroupOrder.json" ] ; then+# echo "categoryGroupOrder.json not found. Generating..."+# `node $SCRIPTS_DIR/generateCategoryGroupOrder.js \+# --inputDir "$SRC_DIR/common/config/wv.json/categories/" \+# --outputDir "$SRC_DIR/common/config/wv.json/"`+# fi-if [ -e "$OPT_DIR/$OPT_SUBDIR/layer-metadata/all.json" ] ; then- cp "$OPT_DIR/$OPT_SUBDIR/layer-metadata/all.json" "$BUILD_DIR/config/wv.json/layer-metadata.json"-fi+# if [ -e "$OPT_DIR/$OPT_SUBDIR/layer-metadata/all.json" ] ; then+# cp "$OPT_DIR/$OPT_SUBDIR/layer-metadata/all.json" "$BUILD_DIR/config/wv.json/layer-metadata.json"+# fi-# Run mergeConfig.js on all directories in /config-configs=$(ls "$BUILD_DIR/config")-for config in $configs; do- case $config in- *.json)- `node $SCRIPTS_DIR/mergeConfig.js \- --inputFile "$BUILD_DIR/config/$config" \- --outputFile "$DEST_DIR/config/$config"`- ;;- *)- cp -r "$BUILD_DIR/config/$config" "$DEST_DIR/config/$config"- ;;- esac-done--# Run mergeConfigWithWMTS.js to merge layer metadata from WMTS GC with worldview layer configs into wv.json-`node $SCRIPTS_DIR/mergeConfigWithWMTS.js \- --inputDir "$BUILD_DIR/_wmts" \- --outputDir "$DEST_DIR/config/wv.json"`--# Copy brand files from build to dest-cp -r "$BUILD_DIR/brand" "$DEST_DIR"-cp "$BUILD_DIR/brand.json" "$DEST_DIR"--# Validate the options build-`node $SCRIPTS_DIR/validateOptions.js" \- --inputFile "$BUILD_DIR/config.json" \- --outputDir "$DEST_DIR/config`--# Fetch preview images from WV Snapshots for any layers which they are missing-`node $SCRIPTS_DIR/fetchPreviewSnapshots.js" \- --inputFile "$DEST_DIR/config/wv.json" \- --outputFile "$OPT_DIR/common/previewLayerOverrides.json" \- --featuresFile "$BUILD_DIR/features.json"`+# # Run mergeConfig.js on all directories in /config+# configs=$(ls "$BUILD_DIR/config")+# for config in $configs; do+# case $config in+# *.json)+# `node $SCRIPTS_DIR/mergeConfig.js \+# --inputFile "$BUILD_DIR/config/$config" \+# --outputFile "$DEST_DIR/config/$config"`+# ;;+# *)+# cp -r "$BUILD_DIR/config/$config" "$DEST_DIR/config/$config"+# ;;+# esac+# done++# # Run mergeConfigWithWMTS.js to merge layer metadata from WMTS GC with worldview layer configs into wv.json+# `node $SCRIPTS_DIR/mergeConfigWithWMTS.js \+# --inputDir "$BUILD_DIR/_wmts" \+# --outputDir "$DEST_DIR/config/wv.json"`++# # Copy brand files from build to dest+# cp -r "$BUILD_DIR/brand" "$DEST_DIR"+# cp "$BUILD_DIR/brand.json" "$DEST_DIR"++# # Validate the options build+# `node $SCRIPTS_DIR/validateOptions.js" \+# --inputFile "$BUILD_DIR/config.json" \+# --outputDir "$DEST_DIR/config`++# # Fetch preview images from WV Snapshots for any layers which they are missing+# `node $SCRIPTS_DIR/fetchPreviewSnapshots.js" \+# --inputFile "$DEST_DIR/config/wv.json" \+# --outputFile "$OPT_DIR/common/previewLayerOverrides.json" \+# --featuresFile "$BUILD_DIR/features.json"`exit 0