Skip to content

Interactive interface for browsing global, full-resolution satellite imagery

WV 2612 verbose mode 2 (#4521)

Patrick Mouldenfd7ed28655ff
diff --git a/package.json b/package.json
index ffd9640dc..cb44aa0f9 100644
--- a/package.json
+++ b/package.json
@@ -28,8 +28,9 @@
"scripts": {
"analyze": "cross-env ANALYZE_MODE=true NODE_ENV=production webpack",
"build": "run-s clean getcapabilities build:config build:dev",
+ "build:verbose": "run-s clean getcapabilities:verbose build:config build:dev",
"build:ci": "run-s getcapabilities build:config build:prod",
- "build:options": "bash -c 'tasks/buildOptions.sh'",
+ "build:options": "bash tasks/buildOptions.sh",
"build:config": "run-s build:options && node ./tasks/util/config.js",
"build:dev": "cross-env NODE_ENV=development webpack",
"build:prod": "cross-env NODE_ENV=production webpack",
@@ -47,6 +48,7 @@
"docker:stop": "docker stop -t 1 worldview ; true",
"e2e": "npm run docker:restart && docker exec -it worldview /bin/bash /build/e2e/docker-ci.sh ; npm run docker:stop",
"getcapabilities": "bash -c 'FETCH_GC=1 npm run build:options'",
+ "getcapabilities:verbose": "bash -c 'FETCH_GC=1 npm run build:options -- -v'",
"generatePreviews": "node ./tasks/build-options/fetchPreviewSnapshots.py",
"linkcheck": "node ./tasks/link-check",
"lint": "run-s lint:js lint:scss",
diff --git a/tasks/build-options/getCapabilities.js b/tasks/build-options/getCapabilities.js
index 8ff825ab7..29f687a07 100644
--- a/tasks/build-options/getCapabilities.js
+++ b/tasks/build-options/getCapabilities.js
@@ -24,6 +24,12 @@ const options = yargs
type: 'string',
description: 'getcapabilities file'
})
+ .option('mode', {
+ demandOption: true,
+ alias: 'm',
+ type: 'string',
+ description: 'mode'
+ })
.epilog('Pulls GetCapabilities XML and linked metadata from configured locations')
const { argv } = options
@@ -68,7 +74,9 @@ async function getCapabilities () {
const inputFile = value.from
const outputFile = `${outputDir}/${value.to}`
+ if (argv.mode === 'verbose') console.warn(`Fetching config for ${inputFile} to ${outputFile}...`)
await fetchConfigs(inputFile, outputFile)
+ if (argv.mode === 'verbose') console.warn(`Processing capabilities for ${outputFile}...`)
await processGetCapabilities(outputFile)
}
}
@@ -83,6 +91,7 @@ async function fetchConfigs (inputFile, outputFile) {
responseType: 'stream',
timeout: 10000
}).then(async (response) => {
+ if (argv.mode === 'verbose') console.warn(`Writing ${outputFile}...`)
await response.data.pipe(writer)
return finished(writer)
})
@@ -99,10 +108,15 @@ async function handleException (error, link, dir, ext, count) {
}
async function processVectorData (layer) {
+ if (argv.mode === 'verbose') {
+ const ident = layer['ows:Identifier']._text
+ console.warn(`Processing vector data for ${ident}:`)
+ }
if (layer['ows:Metadata']) {
Object.values(layer['ows:Metadata']).forEach((item) => {
const schemaVersion = item._attributes['xlink:role']
if (schemaVersion === 'http://earthdata.nasa.gov/gibs/metadata-type/layer/1.0') {
+ if (argv.mode === 'verbose') console.warn(` Processing Metadata: ${item._attributes['xlink:href']}`)
const vectorDataLink = item._attributes['xlink:href']
const vectorDataFile = path.basename(vectorDataLink)
const vectorDataId = path.parse(vectorDataFile).name
@@ -114,11 +128,13 @@ async function processVectorData (layer) {
async function processLayer (layer) {
const ident = layer['ows:Identifier']._text
+ if (argv.mode === 'verbose') console.warn(`Processing layer ${ident}:`)
if (layer['ows:Metadata']) {
if (config.skipPalettes) {
console.warn(`${prog}: WARN: Skipping palette for ${ident} \n`)
} else {
Object.values(layer['ows:Metadata']).forEach((item) => {
+ if (argv.mode === 'verbose') console.warn(` Processing pallette: ${item._attributes['xlink:href']}`)
const schemaVersion = item._attributes['xlink:role']
if (schemaVersion === 'http://earthdata.nasa.gov/gibs/metadata-type/colormap/1.3') {
const colormapLink = item._attributes['xlink:href']
diff --git a/tasks/build-options/validateConfigs.js b/tasks/build-options/validateConfigs.js
index cb089cc0d..3381117f2 100644
--- a/tasks/build-options/validateConfigs.js
+++ b/tasks/build-options/validateConfigs.js
@@ -22,6 +22,12 @@ const options = yargs
type: 'string',
description: 'layer-config.json schema'
})
+ .option('mode', {
+ demandOption: true,
+ alias: 'm',
+ type: 'string',
+ description: 'mode'
+ })
.epilog('Validates layers using a JSON schema')
const { argv } = options
@@ -47,6 +53,7 @@ async function main () {
validateFile(filePath)
}
if (invalidJsonFiles.length) {
+ if (argv.mode === 'verbose') console.warn(`${prog}: Invalid JSON files: ${invalidJsonFiles}`)
throw new Error(`${prog}: FAILED: ${invalidJsonFiles.length} layer configs failed validation.`)
} else {
console.warn(`${prog}: PASSED: All layer configs passed validation!`)
@@ -54,6 +61,7 @@ async function main () {
}
async function validateFile (filePath) {
+ if (argv.mode === 'verbose') console.warn(`${prog}: Validating ${filePath}`)
const layerFile = fs.readFileSync(filePath)
const layer = JSON.parse(layerFile)
const valid = validate(layer)
diff --git a/tasks/buildOptions.sh b/tasks/buildOptions.sh
index f6d260b54..6f0124983 100755
--- a/tasks/buildOptions.sh
+++ b/tasks/buildOptions.sh
@@ -1,14 +1,22 @@
#!/bin/bash
-
PROG=$(basename "$0")
BASE=$(dirname "$0")/..
-
SRC_DIR="$BASE/config/default"
OPT_DIR="$BASE/config/default"
BUILD_DIR="$BASE/build/options-build"
DEST_DIR="$BASE/build/options"
SCRIPTS_DIR="$BASE/tasks/build-options"
+MODE="default"
+
+while getopts ":v" option; do
+ echo "Option -$option set"
+ case $option in
+ v)
+ echo "Verbose Mode Activated"
+ MODE="verbose";;
+ esac
+done
# If there is an active directory, use instead of defaults
if [ -d "$BASE/config/active" ]; then
@@ -50,24 +58,36 @@ mkdir -p "$BUILD_DIR/colormaps"
# If $FETCH_GC is set, make various API requests
if [ "$FETCH_GC" ] ; then
# Fetch GC files and create colormaps, vectordata and vectorstyle files
+ if (( $MODE = "verbose" )) ; then
+ echo "Fetch GC files and create colormaps, vectordata and vectorstyle files"
+ fi
rm -rf "$OPT_DIR/$OPT_SUBDIR/gc/*"
rm -rf "$OPT_DIR/$OPT_SUBDIR/colormaps/gc/*"
`node $SCRIPTS_DIR/getCapabilities.js \
--config "$OPT_DIR/$OPT_SUBDIR/config.json" \
- --getcapabilities "$OPT_DIR/$OPT_SUBDIR/gc"`
+ --getcapabilities "$OPT_DIR/$OPT_SUBDIR/gc" \
+ --mode "$MODE"`
# Get metadata for files in layerOrder.json and combine this data into 1 file
+ if (( $MODE = "verbose" )) ; then
+ echo "Get metadata for files in layerOrder.json and combine this data into 1 file"
+ fi
rm -rf "$OPT_DIR/$OPT_SUBDIR/layer-metadata"
mkdir -p "$OPT_DIR/$OPT_SUBDIR/layer-metadata"
`node $SCRIPTS_DIR/getVisMetadata.js \
--features "$BUILD_DIR/features.json" \
--layerOrder "$BUILD_DIR/config/wv.json/layerOrder.json" \
- --layerMetadata "$OPT_DIR/$OPT_SUBDIR/layer-metadata/all.json"`
+ --layerMetadata "$OPT_DIR/$OPT_SUBDIR/layer-metadata/all.json" \
+ --mode "$MODE"`
else
# Validate layers in wv.json with a JSON schema
+ if (( $MODE = "verbose" )) ; then
+ echo "Validate layers in wv.json with a JSON schema"
+ fi
`node $SCRIPTS_DIR/validateConfigs.js \
--inputDirectory "$SRC_DIR/common/config/wv.json/layers" \
- --schemaFile "$BASE/schemas/layer-config.json"`
+ --schemaFile "$BASE/schemas/layer-config.json" \
+ --mode "$MODE"`
if [ -e "$BUILD_DIR/features.json" ] ; then
cp "$BUILD_DIR/features.json" "$BUILD_DIR/config/wv.json/_features.json"
@@ -75,30 +95,45 @@ else
# Run extractConfigFromWMTS.js script with config.json
if [ -e "$BUILD_DIR/config.json" ] ; then
+ if (( $MODE = "verbose" )) ; then
+ echo "Run extractConfigFromWMTS.js script with config.json"
+ fi
`node $SCRIPTS_DIR/extractConfigFromWMTS.js \
--config "$BUILD_DIR/config.json" \
--inputDir "$BUILD_DIR/gc" \
- --outputDir "$BUILD_DIR/_wmts"`
+ --outputDir "$BUILD_DIR/_wmts" \
+ --mode "$MODE"`
fi
# Run processVectorStyles.js and move vectorstyles where we want them
if [ -e "$BUILD_DIR/gc/vectorstyles" ] ; then
+ if (( $MODE = "verbose" )) ; then
+ echo "Run processVectorStyles.js and move vectorstyles where we want them"
+ fi
mkdir -p "$BUILD_DIR/config/wv.json/vectorstyles"
`node $SCRIPTS_DIR/processVectorStyles.js \
--inputDir "$BUILD_DIR/gc/vectorstyles" \
- --outputDir "$BUILD_DIR/config/wv.json/vectorstyles"`
+ --outputDir "$BUILD_DIR/config/wv.json/vectorstyles" \
+ --mode "$MODE"`
fi
# Run processVectorData.js and move vectordata where we want them
if [ -e "$BUILD_DIR/gc/vectordata" ] ; then
+ if (( $MODE = "verbose" )) ; then
+ echo "Run processVectorData.js and move vectordata where we want them"
+ fi
mkdir -p "$BUILD_DIR/config/wv.json/vectordata"
`node $SCRIPTS_DIR/processVectorData.js \
--inputDir "$BUILD_DIR/gc/vectordata" \
- --outputDir "$BUILD_DIR/config/wv.json/vectordata"`
+ --outputDir "$BUILD_DIR/config/wv.json/vectordata" \
+ --mode "$MODE"`
fi
# Run processColormap.js and move colormaps where we want them
if [ -e "$BUILD_DIR/colormaps" ] ; then
+ if (( $MODE = "verbose" )) ; then
+ echo "Run processColormap.js and move colormaps where we want them"
+ fi
mkdir -p "$BUILD_DIR"/config/palettes
if [ -d "$BUILD_DIR"/gc/colormaps ] ; then
cp -r "$BUILD_DIR"/gc/colormaps "$BUILD_DIR"/colormaps/gc
@@ -106,7 +141,8 @@ else
`node $SCRIPTS_DIR/processColormap.js \
--config "$OPT_DIR/$OPT_SUBDIR/config.json" \
--inputDir "$BUILD_DIR/colormaps" \
- --outputDir "$BUILD_DIR/config/palettes"`
+ --outputDir "$BUILD_DIR/config/palettes" \
+ --mode "$MODE"`
fi
# Throw error if no categoryGroupOrder.json file present
@@ -114,7 +150,8 @@ else
echo "categoryGroupOrder.json not found. Generating..."
`node $SCRIPTS_DIR/generateCategoryGroupOrder.js \
--inputDir "$SRC_DIR/common/config/wv.json/categories/" \
- --outputDir "$SRC_DIR/common/config/wv.json/"`
+ --outputDir "$SRC_DIR/common/config/wv.json/" \
+ --mode "$MODE"`
fi
if [ -e "$OPT_DIR/$OPT_SUBDIR/layer-metadata/all.json" ] ; then
@@ -138,24 +175,39 @@ else
done
# Run mergeConfigWithWMTS.js to merge layer metadata from WMTS GC with worldview layer configs into wv.json
+ if (( $MODE = "verbose" )) ; then
+ echo "Run mergeConfigWithWMTS.js to merge layer metadata from WMTS GC with worldview layer configs into wv.json"
+ fi
`node $SCRIPTS_DIR/mergeConfigWithWMTS.js \
--inputDir "$BUILD_DIR/_wmts" \
- --outputFile "$DEST_DIR/config/wv.json"`
+ --outputFile "$DEST_DIR/config/wv.json" \
+ --mode "$MODE"`
# Copy brand files from build to dest
+ if (( $MODE = "verbose" )) ; then
+ echo "Copy brand files from build to dest"
+ fi
cp -r "$BUILD_DIR/brand" "$DEST_DIR"
cp "$BUILD_DIR/brand.json" "$DEST_DIR"
# Validate the options build
+ if (( $MODE = "verbose" )) ; then
+ echo "Validate the options build"
+ fi
`node $SCRIPTS_DIR/validateOptions.js \
--optionsFile "$BUILD_DIR/config.json" \
- --configDir "$DEST_DIR/config"`
+ --configDir "$DEST_DIR/config" \
+ --mode "$MODE"`
# Fetch preview images from WV Snapshots for any layers which they are missing
+ if (( $MODE = "verbose" )) ; then
+ echo "Fetch preview images from WV Snapshots for any layers which they are missing"
+ fi
`node $SCRIPTS_DIR/fetchPreviewSnapshots.js \
--wvJsonFile "$DEST_DIR/config/wv.json" \
--overridesFile "$OPT_DIR/common/previewLayerOverrides.json" \
- --featuresFile "$BUILD_DIR/features.json"`
+ --featuresFile "$BUILD_DIR/features.json" \
+ --mode "$MODE"`
fi
exit 0