Interactive interface for browsing global, full-resolution satellite imagery
WV 2612 verbose mode 2 (#4521)
Patrick Mouldenfd7ed28655ff
diff --git a/package.json b/package.jsonindex ffd9640dc..cb44aa0f9 100644--- a/package.json+++ b/package.json@@ -28,8 +28,9 @@"scripts": {"analyze": "cross-env ANALYZE_MODE=true NODE_ENV=production webpack","build": "run-s clean getcapabilities build:config build:dev",+ "build:verbose": "run-s clean getcapabilities:verbose build:config build:dev","build:ci": "run-s getcapabilities build:config build:prod",- "build:options": "bash -c 'tasks/buildOptions.sh'",+ "build:options": "bash tasks/buildOptions.sh","build:config": "run-s build:options && node ./tasks/util/config.js","build:dev": "cross-env NODE_ENV=development webpack","build:prod": "cross-env NODE_ENV=production webpack",@@ -47,6 +48,7 @@"docker:stop": "docker stop -t 1 worldview ; true","e2e": "npm run docker:restart && docker exec -it worldview /bin/bash /build/e2e/docker-ci.sh ; npm run docker:stop","getcapabilities": "bash -c 'FETCH_GC=1 npm run build:options'",+ "getcapabilities:verbose": "bash -c 'FETCH_GC=1 npm run build:options -- -v'","generatePreviews": "node ./tasks/build-options/fetchPreviewSnapshots.py","linkcheck": "node ./tasks/link-check","lint": "run-s lint:js lint:scss",diff --git a/tasks/build-options/getCapabilities.js b/tasks/build-options/getCapabilities.jsindex 8ff825ab7..29f687a07 100644--- a/tasks/build-options/getCapabilities.js+++ b/tasks/build-options/getCapabilities.js@@ -24,6 +24,12 @@ const options = yargstype: 'string',description: 'getcapabilities file'})+ .option('mode', {+ demandOption: true,+ alias: 'm',+ type: 'string',+ description: 'mode'+ }).epilog('Pulls GetCapabilities XML and linked metadata from configured locations')const { argv } = options@@ -68,7 +74,9 @@ async function getCapabilities () {const inputFile = value.fromconst outputFile = `${outputDir}/${value.to}`+ if (argv.mode === 'verbose') console.warn(`Fetching config for ${inputFile} to ${outputFile}...`)await fetchConfigs(inputFile, outputFile)+ if (argv.mode === 'verbose') console.warn(`Processing capabilities for ${outputFile}...`)await processGetCapabilities(outputFile)}}@@ -83,6 +91,7 @@ async function fetchConfigs (inputFile, outputFile) {responseType: 'stream',timeout: 10000}).then(async (response) => {+ if (argv.mode === 'verbose') console.warn(`Writing ${outputFile}...`)await response.data.pipe(writer)return finished(writer)})@@ -99,10 +108,15 @@ async function handleException (error, link, dir, ext, count) {}async function processVectorData (layer) {+ if (argv.mode === 'verbose') {+ const ident = layer['ows:Identifier']._text+ console.warn(`Processing vector data for ${ident}:`)+ }if (layer['ows:Metadata']) {Object.values(layer['ows:Metadata']).forEach((item) => {const schemaVersion = item._attributes['xlink:role']if (schemaVersion === 'http://earthdata.nasa.gov/gibs/metadata-type/layer/1.0') {+ if (argv.mode === 'verbose') console.warn(` Processing Metadata: ${item._attributes['xlink:href']}`)const vectorDataLink = item._attributes['xlink:href']const vectorDataFile = path.basename(vectorDataLink)const vectorDataId = path.parse(vectorDataFile).name@@ -114,11 +128,13 @@ async function processVectorData (layer) {async function processLayer (layer) {const ident = layer['ows:Identifier']._text+ if (argv.mode === 'verbose') console.warn(`Processing layer ${ident}:`)if (layer['ows:Metadata']) {if (config.skipPalettes) {console.warn(`${prog}: WARN: Skipping palette for ${ident} \n`)} else {Object.values(layer['ows:Metadata']).forEach((item) => {+ if (argv.mode === 'verbose') console.warn(` Processing pallette: ${item._attributes['xlink:href']}`)const schemaVersion = item._attributes['xlink:role']if (schemaVersion === 'http://earthdata.nasa.gov/gibs/metadata-type/colormap/1.3') {const colormapLink = item._attributes['xlink:href']diff --git a/tasks/build-options/validateConfigs.js b/tasks/build-options/validateConfigs.jsindex cb089cc0d..3381117f2 100644--- a/tasks/build-options/validateConfigs.js+++ b/tasks/build-options/validateConfigs.js@@ -22,6 +22,12 @@ const options = yargstype: 'string',description: 'layer-config.json schema'})+ .option('mode', {+ demandOption: true,+ alias: 'm',+ type: 'string',+ description: 'mode'+ }).epilog('Validates layers using a JSON schema')const { argv } = options@@ -47,6 +53,7 @@ async function main () {validateFile(filePath)}if (invalidJsonFiles.length) {+ if (argv.mode === 'verbose') console.warn(`${prog}: Invalid JSON files: ${invalidJsonFiles}`)throw new Error(`${prog}: FAILED: ${invalidJsonFiles.length} layer configs failed validation.`)} else {console.warn(`${prog}: PASSED: All layer configs passed validation!`)@@ -54,6 +61,7 @@ async function main () {}async function validateFile (filePath) {+ if (argv.mode === 'verbose') console.warn(`${prog}: Validating ${filePath}`)const layerFile = fs.readFileSync(filePath)const layer = JSON.parse(layerFile)const valid = validate(layer)diff --git a/tasks/buildOptions.sh b/tasks/buildOptions.shindex f6d260b54..6f0124983 100755--- a/tasks/buildOptions.sh+++ b/tasks/buildOptions.sh@@ -1,14 +1,22 @@#!/bin/bash-PROG=$(basename "$0")BASE=$(dirname "$0")/..-SRC_DIR="$BASE/config/default"OPT_DIR="$BASE/config/default"BUILD_DIR="$BASE/build/options-build"DEST_DIR="$BASE/build/options"SCRIPTS_DIR="$BASE/tasks/build-options"+MODE="default"++while getopts ":v" option; do+ echo "Option -$option set"+ case $option in+ v)+ echo "Verbose Mode Activated"+ MODE="verbose";;+ esac+done# If there is an active directory, use instead of defaultsif [ -d "$BASE/config/active" ]; then@@ -50,24 +58,36 @@ mkdir -p "$BUILD_DIR/colormaps"# If $FETCH_GC is set, make various API requestsif [ "$FETCH_GC" ] ; then# Fetch GC files and create colormaps, vectordata and vectorstyle files+ if (( $MODE = "verbose" )) ; then+ echo "Fetch GC files and create colormaps, vectordata and vectorstyle files"+ firm -rf "$OPT_DIR/$OPT_SUBDIR/gc/*"rm -rf "$OPT_DIR/$OPT_SUBDIR/colormaps/gc/*"`node $SCRIPTS_DIR/getCapabilities.js \--config "$OPT_DIR/$OPT_SUBDIR/config.json" \- --getcapabilities "$OPT_DIR/$OPT_SUBDIR/gc"`+ --getcapabilities "$OPT_DIR/$OPT_SUBDIR/gc" \+ --mode "$MODE"`# Get metadata for files in layerOrder.json and combine this data into 1 file+ if (( $MODE = "verbose" )) ; then+ echo "Get metadata for files in layerOrder.json and combine this data into 1 file"+ firm -rf "$OPT_DIR/$OPT_SUBDIR/layer-metadata"mkdir -p "$OPT_DIR/$OPT_SUBDIR/layer-metadata"`node $SCRIPTS_DIR/getVisMetadata.js \--features "$BUILD_DIR/features.json" \--layerOrder "$BUILD_DIR/config/wv.json/layerOrder.json" \- --layerMetadata "$OPT_DIR/$OPT_SUBDIR/layer-metadata/all.json"`+ --layerMetadata "$OPT_DIR/$OPT_SUBDIR/layer-metadata/all.json" \+ --mode "$MODE"`else# Validate layers in wv.json with a JSON schema+ if (( $MODE = "verbose" )) ; then+ echo "Validate layers in wv.json with a JSON schema"+ fi`node $SCRIPTS_DIR/validateConfigs.js \--inputDirectory "$SRC_DIR/common/config/wv.json/layers" \- --schemaFile "$BASE/schemas/layer-config.json"`+ --schemaFile "$BASE/schemas/layer-config.json" \+ --mode "$MODE"`if [ -e "$BUILD_DIR/features.json" ] ; thencp "$BUILD_DIR/features.json" "$BUILD_DIR/config/wv.json/_features.json"@@ -75,30 +95,45 @@ else# Run extractConfigFromWMTS.js script with config.jsonif [ -e "$BUILD_DIR/config.json" ] ; then+ if (( $MODE = "verbose" )) ; then+ echo "Run extractConfigFromWMTS.js script with config.json"+ fi`node $SCRIPTS_DIR/extractConfigFromWMTS.js \--config "$BUILD_DIR/config.json" \--inputDir "$BUILD_DIR/gc" \- --outputDir "$BUILD_DIR/_wmts"`+ --outputDir "$BUILD_DIR/_wmts" \+ --mode "$MODE"`fi# Run processVectorStyles.js and move vectorstyles where we want themif [ -e "$BUILD_DIR/gc/vectorstyles" ] ; then+ if (( $MODE = "verbose" )) ; then+ echo "Run processVectorStyles.js and move vectorstyles where we want them"+ fimkdir -p "$BUILD_DIR/config/wv.json/vectorstyles"`node $SCRIPTS_DIR/processVectorStyles.js \--inputDir "$BUILD_DIR/gc/vectorstyles" \- --outputDir "$BUILD_DIR/config/wv.json/vectorstyles"`+ --outputDir "$BUILD_DIR/config/wv.json/vectorstyles" \+ --mode "$MODE"`fi# Run processVectorData.js and move vectordata where we want themif [ -e "$BUILD_DIR/gc/vectordata" ] ; then+ if (( $MODE = "verbose" )) ; then+ echo "Run processVectorData.js and move vectordata where we want them"+ fimkdir -p "$BUILD_DIR/config/wv.json/vectordata"`node $SCRIPTS_DIR/processVectorData.js \--inputDir "$BUILD_DIR/gc/vectordata" \- --outputDir "$BUILD_DIR/config/wv.json/vectordata"`+ --outputDir "$BUILD_DIR/config/wv.json/vectordata" \+ --mode "$MODE"`fi# Run processColormap.js and move colormaps where we want themif [ -e "$BUILD_DIR/colormaps" ] ; then+ if (( $MODE = "verbose" )) ; then+ echo "Run processColormap.js and move colormaps where we want them"+ fimkdir -p "$BUILD_DIR"/config/palettesif [ -d "$BUILD_DIR"/gc/colormaps ] ; thencp -r "$BUILD_DIR"/gc/colormaps "$BUILD_DIR"/colormaps/gc@@ -106,7 +141,8 @@ else`node $SCRIPTS_DIR/processColormap.js \--config "$OPT_DIR/$OPT_SUBDIR/config.json" \--inputDir "$BUILD_DIR/colormaps" \- --outputDir "$BUILD_DIR/config/palettes"`+ --outputDir "$BUILD_DIR/config/palettes" \+ --mode "$MODE"`fi# Throw error if no categoryGroupOrder.json file present@@ -114,7 +150,8 @@ elseecho "categoryGroupOrder.json not found. Generating..."`node $SCRIPTS_DIR/generateCategoryGroupOrder.js \--inputDir "$SRC_DIR/common/config/wv.json/categories/" \- --outputDir "$SRC_DIR/common/config/wv.json/"`+ --outputDir "$SRC_DIR/common/config/wv.json/" \+ --mode "$MODE"`fiif [ -e "$OPT_DIR/$OPT_SUBDIR/layer-metadata/all.json" ] ; then@@ -138,24 +175,39 @@ elsedone# Run mergeConfigWithWMTS.js to merge layer metadata from WMTS GC with worldview layer configs into wv.json+ if (( $MODE = "verbose" )) ; then+ echo "Run mergeConfigWithWMTS.js to merge layer metadata from WMTS GC with worldview layer configs into wv.json"+ fi`node $SCRIPTS_DIR/mergeConfigWithWMTS.js \--inputDir "$BUILD_DIR/_wmts" \- --outputFile "$DEST_DIR/config/wv.json"`+ --outputFile "$DEST_DIR/config/wv.json" \+ --mode "$MODE"`# Copy brand files from build to dest+ if (( $MODE = "verbose" )) ; then+ echo "Copy brand files from build to dest"+ ficp -r "$BUILD_DIR/brand" "$DEST_DIR"cp "$BUILD_DIR/brand.json" "$DEST_DIR"# Validate the options build+ if (( $MODE = "verbose" )) ; then+ echo "Validate the options build"+ fi`node $SCRIPTS_DIR/validateOptions.js \--optionsFile "$BUILD_DIR/config.json" \- --configDir "$DEST_DIR/config"`+ --configDir "$DEST_DIR/config" \+ --mode "$MODE"`# Fetch preview images from WV Snapshots for any layers which they are missing+ if (( $MODE = "verbose" )) ; then+ echo "Fetch preview images from WV Snapshots for any layers which they are missing"+ fi`node $SCRIPTS_DIR/fetchPreviewSnapshots.js \--wvJsonFile "$DEST_DIR/config/wv.json" \--overridesFile "$OPT_DIR/common/previewLayerOverrides.json" \- --featuresFile "$BUILD_DIR/features.json"`+ --featuresFile "$BUILD_DIR/features.json" \+ --mode "$MODE"`fiexit 0