Skip to content

Interactive interface for browsing global, full-resolution satellite imagery

Add profiling mode to build options scripts for performance measurement and improve performance (#5640)

Patrick Mouldendd928d84232e
diff --git a/tasks/build-options/extractConfigFromWMTS.js b/tasks/build-options/extractConfigFromWMTS.js
index d4525a77f..c3c894ca0 100644
--- a/tasks/build-options/extractConfigFromWMTS.js
+++ b/tasks/build-options/extractConfigFromWMTS.js
@@ -27,6 +27,12 @@ const options = yargs
type: 'string',
description: 'wmts output directory'
})
+ .option('mode', {
+ demandOption: true,
+ alias: 'm',
+ type: 'string',
+ description: 'mode'
+ })
.epilog('Extracts configuration information from a WMTS GetCapabilities file, converts the XML to JSON')
const { argv } = options
@@ -75,7 +81,9 @@ class SkipException extends Error {
* @throws {SkipException}
*/
async function main () {
+ if (argv.mode === 'profile') console.time('extractConfigFromWMTS')
for (const entry of entries) {
+ if (argv.mode === 'profile') console.time(`extractConfigFromWMTS-${entry.source}`)
wv.layers = {}
wv.sources = {}
wvMatrixSets = {}
@@ -92,6 +100,7 @@ async function main () {
totalErrorCount += errorCount
totalWarningCount += warningCount
totalLayerCount += layerCount
+ if (argv.mode === 'profile') console.timeEnd(`extractConfigFromWMTS-${entry.source}`)
}
console.warn(`${prog}:${totalErrorCount} errors, ${totalWarningCount} warnings, ${totalLayerCount} layers`)
@@ -99,6 +108,7 @@ async function main () {
if (totalErrorCount > 0) {
throw new Error(`${prog}: Error: ${totalErrorCount} errors occured`)
}
+ if (argv.mode === 'profile') console.timeEnd('extractConfigFromWMTS')
}
async function processEntry (entry) {
@@ -135,23 +145,22 @@ async function processEntry (entry) {
console.error(`${prog}: ERROR: [${gcId}] No layers\n`)
return { errorCount, warningCount, layerCount }
}
-
- for (const gcLayer of gcContents.Layer) {
- try {
- layerCount += 1
- await processLayer(gcLayer, wvLayers, entry)
- } catch (error) {
- if (error instanceof SkipException) {
- warningCount += 1
- console.warn(`${prog}: WARNING: [${gcId}] Skipping\n`)
- } else {
- errorCount += 1
- console.error(error.stack)
- const ident = gcLayer['ows:Identifier']._text
- console.error(`${prog}: ERROR: [${gcId}:${ident}] ${error}\n`)
- }
+ const promises = gcContents.Layer.map((gcLayer) => {
+ layerCount += 1
+ return processLayer(gcLayer, wvLayers, entry)
+ })
+ const results = await Promise.allSettled(promises)
+ const rejected = results.filter(result => result.status === 'rejected')
+ rejected.forEach((reject) => {
+ const error = reject.reason
+ if (error instanceof SkipException) {
+ warningCount += 1
+ console.warn(`${prog}: WARNING: [${gcId}] Skipping\n`)
+ } else {
+ errorCount += 1
+ console.error(`${prog}: ERROR: [${gcId}] ${error}\n`)
}
- }
+ })
if (gcContents.TileMatrixSet === 'Object') {
processMatrixSet(gcContents.TileMatrixSet, entry)
diff --git a/tasks/build-options/fetchPreviewSnapshots.js b/tasks/build-options/fetchPreviewSnapshots.js
index 7aafe8600..bab74a237 100644
--- a/tasks/build-options/fetchPreviewSnapshots.js
+++ b/tasks/build-options/fetchPreviewSnapshots.js
@@ -26,6 +26,12 @@ const options = yargs
type: 'string',
description: 'features file'
})
+ .option('mode', {
+ demandOption: true,
+ alias: 'm',
+ type: 'string',
+ description: 'mode'
+ })
.epilog('Fetch preview images from WV Snapshots for any layers which they are missing.')
const { argv } = options
@@ -80,6 +86,7 @@ const referenceLayers = {
}
async function main () {
+ if (argv.mode === 'profile') console.time('fetchPreviewSnapshots')
// Check to see if this feature is enabled in features.json before continuing
const featuresFileContent = fs.readFileSync(featuresFile, { encoding: 'utf-8' })
const featuresDict = JSON.parse(featuresFileContent)
@@ -121,6 +128,7 @@ async function main () {
if (totalFailureCount === 0 && totalSuccessCount === 0) {
console.warn(`\n${prog}: No snapshots were retrieved. All layers found in wv.json have existing preview images!`)
}
+ if (argv.mode === 'profile') console.timeEnd('fetchPreviewSnapshots')
}
async function trackBadSnapshots (layerId, projection, request, imgFile, badSnapshots) {
diff --git a/tasks/build-options/getCapabilities.js b/tasks/build-options/getCapabilities.js
index 04b7d0cbc..e704add62 100644
--- a/tasks/build-options/getCapabilities.js
+++ b/tasks/build-options/getCapabilities.js
@@ -49,6 +49,7 @@ const configData = fs.readFileSync(configFile)
const config = JSON.parse(configData)
async function main () {
+ if (argv.mode === 'profile') console.time('getCapabilities')
if (!fs.existsSync(outputDir)) {
fs.mkdirSync(outputDir)
}
@@ -64,6 +65,7 @@ async function main () {
if (vectordata) {
await gatherProcess(vectordata, 'vectordata files', vectordataDir, '.json')
}
+ if (argv.mode === 'profile') console.timeEnd('getCapabilities')
}
async function getCapabilities () {
diff --git a/tasks/build-options/getVisMetadata.js b/tasks/build-options/getVisMetadata.js
index 88f0da4b1..db208a0b9 100644
--- a/tasks/build-options/getVisMetadata.js
+++ b/tasks/build-options/getVisMetadata.js
@@ -115,15 +115,17 @@ const useKeys = [
]
async function main (url) {
+ if (argv.mode === 'profile') console.time('getVisMetadata')
layerOrder = layerOrder.layerOrder
layerOrder = layerOrder.filter(x => !skipLayers.includes(x))
console.warn(`${prog}: Fetching ${layerOrder.length} layer-metadata files`)
- for (const layerId of layerOrder) {
- if (!layerId.includes('_STD') && !layerId.includes('_NRT')) {
- await getMetadata(layerId, url)
- }
- }
+ const promises = layerOrder.map((layerId) => {
+ if (layerId.includes('_STD') && layerId.includes('_NRT')) return null
+
+ return getMetadata(layerId, url)
+ })
+ await Promise.allSettled(promises)
const layers = Object.keys(layerMetadata).sort().reduce(
(obj, key) => {
@@ -135,6 +137,7 @@ async function main (url) {
await fs.writeFileSync(outputFile, JSON.stringify({ layers }))
console.warn(`${prog}: Combined all layer-metadata files into ${path.parse(outputFile).base}`)
+ if (argv.mode === 'profile') console.timeEnd('getVisMetadata')
}
async function getDAAC (metadata) {
diff --git a/tasks/build-options/mergeConfigWithWMTS.js b/tasks/build-options/mergeConfigWithWMTS.js
index 0ce247d13..c2bb71f52 100644
--- a/tasks/build-options/mergeConfigWithWMTS.js
+++ b/tasks/build-options/mergeConfigWithWMTS.js
@@ -18,6 +18,12 @@ const options = yargs
type: 'string',
description: 'wmts output directory'
})
+ .option('mode', {
+ demandOption: true,
+ alias: 'm',
+ type: 'string',
+ description: 'mode'
+ })
.epilog('Concatenates all configuration items a directory into one configuration file.')
const { argv } = options
@@ -37,6 +43,7 @@ const outputData = JSON.parse(fs.readFileSync(outputFile, 'utf-8'))
let fileCount = 0
async function main () {
+ if (argv.mode === 'profile') console.time('mergeConfigWithWMTS')
layers = newConf.layers
delete layers.sources
delete newConf.layers
@@ -50,6 +57,7 @@ async function main () {
await fs.writeFileSync(outputFile, JSON.stringify(newConf, jsonOptions))
console.warn(`${prog}: ${fileCount} file(s) merged into ${path.basename(outputFile)}`)
+ if (argv.mode === 'profile') console.timeEnd('mergeConfigWithWMTS')
}
async function processFiles () {
diff --git a/tasks/build-options/processColormap.js b/tasks/build-options/processColormap.js
index 54efae9dc..c3064a86c 100755
--- a/tasks/build-options/processColormap.js
+++ b/tasks/build-options/processColormap.js
@@ -29,6 +29,12 @@ const options = yargs
type: 'string',
description: 'wmts output directory'
})
+ .option('mode', {
+ demandOption: true,
+ alias: 'm',
+ type: 'string',
+ description: 'mode'
+ })
.epilog('Converts colormaps to JSON files')
const { argv } = options
@@ -64,6 +70,7 @@ const walk = (dir) => {
}
async function main () {
+ if (argv.mode === 'profile') console.time('processColormap')
const files = await walk(inputDir)
for (const file of files) {
try {
@@ -82,6 +89,7 @@ async function main () {
if (errorCount > 0) {
throw new Error(`${prog}: Error: ${errorCount} errors occured`)
}
+ if (argv.mode === 'profile') console.timeEnd('processColormap')
}
function toList (v) {
diff --git a/tasks/build-options/processVectorData.js b/tasks/build-options/processVectorData.js
index 78c9f4fbc..e600a4f3f 100644
--- a/tasks/build-options/processVectorData.js
+++ b/tasks/build-options/processVectorData.js
@@ -23,6 +23,12 @@ const options = yargs
type: 'string',
description: 'wmts output directory'
})
+ .option('mode', {
+ demandOption: true,
+ alias: 'm',
+ type: 'string',
+ description: 'mode'
+ })
.epilog('Extracts vector data information from GetCapabilities')
const { argv } = options
@@ -34,6 +40,7 @@ const inputDir = argv.inputDir
const outputDir = argv.outputDir
async function main () {
+ if (argv.mode === 'profile') console.time('processVectorData')
let fileCount = 0
let errorCount = 0
@@ -52,6 +59,7 @@ async function main () {
if (errorCount > 0) {
throw new Error(`${prog}: Error: ${errorCount} errors occured`)
}
+ if (argv.mode === 'profile') console.timeEnd('processVectorData')
}
async function copyFileAsync (file) {
diff --git a/tasks/build-options/processVectorStyles.js b/tasks/build-options/processVectorStyles.js
index bfcff1e30..6c8c7014a 100644
--- a/tasks/build-options/processVectorStyles.js
+++ b/tasks/build-options/processVectorStyles.js
@@ -23,6 +23,12 @@ const options = yargs
type: 'string',
description: 'wmts output directory'
})
+ .option('mode', {
+ demandOption: true,
+ alias: 'm',
+ type: 'string',
+ description: 'mode'
+ })
.epilog('Extracts vector style information from GetCapabilities')
const { argv } = options
@@ -34,6 +40,7 @@ const inputDir = argv.inputDir
const outputDir = argv.outputDir
async function main () {
+ if (argv.mode === 'profile') console.time('processVectorStyles')
let fileCount = 0
let errorCount = 0
@@ -52,6 +59,7 @@ async function main () {
if (errorCount > 0) {
throw new Error(`${prog}: Error: ${errorCount} errors occured`)
}
+ if (argv.mode === 'profile') console.timeEnd('processVectorStyles')
}
async function copyFileAsync (file) {
diff --git a/tasks/build-options/validateConfigs.js b/tasks/build-options/validateConfigs.js
index 48dce4f72..28c2d6e29 100644
--- a/tasks/build-options/validateConfigs.js
+++ b/tasks/build-options/validateConfigs.js
@@ -52,6 +52,7 @@ const invalidJsonFiles = []
console.warn(`${prog}: Validating layer configs...`)
async function main () {
+ if (argv.mode === 'profile') console.time('validateConfigs')
let files = globSync(inputDirectory + '/**/*')
files = files.filter(file => file.endsWith('.json'))
for (const filePath of files) {
@@ -59,9 +60,11 @@ async function main () {
}
if (invalidJsonFiles.length) {
if (argv.mode === 'verbose') console.warn(`${prog}: Invalid JSON files: ${invalidJsonFiles}`)
+ if (argv.mode === 'profile') console.timeEnd('validateConfigs')
throw new Error(`${prog}: FAILED: ${invalidJsonFiles.length} layer configs failed validation.`)
} else {
console.warn(`${prog}: PASSED: All layer configs passed validation!`)
+ if (argv.mode === 'profile') console.timeEnd('validateConfigs')
}
}
diff --git a/tasks/build-options/validateOptions.js b/tasks/build-options/validateOptions.js
index 91e6eb762..f21943788 100644
--- a/tasks/build-options/validateOptions.js
+++ b/tasks/build-options/validateOptions.js
@@ -19,6 +19,12 @@ const options = yargs
type: 'string',
description: 'config directory'
})
+ .option('mode', {
+ demandOption: true,
+ alias: 'm',
+ type: 'string',
+ description: 'mode'
+ })
.epilog('Validates and corrects the configuration files.')
const { argv } = options
@@ -75,6 +81,7 @@ if (tolerant) warn('Validation enforcement disabled')
let startDate = moment.max()
async function main () {
+ if (argv.mode === 'profile') console.time('validateOptions')
for (const layerId of Object.keys(wv.layers)) {
let layer = wv.layers[layerId]
if (layerId !== layer.id) {
@@ -186,6 +193,7 @@ async function main () {
if (errorCount > 0) {
throw new Error(`${prog}: Error: ${errorCount} errors occured`)
}
+ if (argv.mode === 'profile') console.timeEnd('validateOptions')
}
main().catch((err) => {
diff --git a/tasks/buildOptions.sh b/tasks/buildOptions.sh
index 6f0124983..00295dc0b 100755
--- a/tasks/buildOptions.sh
+++ b/tasks/buildOptions.sh
@@ -9,12 +9,15 @@ SCRIPTS_DIR="$BASE/tasks/build-options"
MODE="default"
-while getopts ":v" option; do
+while getopts ":vp" option; do
echo "Option -$option set"
case $option in
v)
echo "Verbose Mode Activated"
MODE="verbose";;
+ p)
+ echo "Profile Mode Activated"
+ MODE="profile";;
esac
done
@@ -58,33 +61,33 @@ mkdir -p "$BUILD_DIR/colormaps"
# If $FETCH_GC is set, make various API requests
if [ "$FETCH_GC" ] ; then
# Fetch GC files and create colormaps, vectordata and vectorstyle files
- if (( $MODE = "verbose" )) ; then
+ if (( $MODE == "verbose" )) ; then
echo "Fetch GC files and create colormaps, vectordata and vectorstyle files"
fi
rm -rf "$OPT_DIR/$OPT_SUBDIR/gc/*"
rm -rf "$OPT_DIR/$OPT_SUBDIR/colormaps/gc/*"
- `node $SCRIPTS_DIR/getCapabilities.js \
+ echo `node $SCRIPTS_DIR/getCapabilities.js \
--config "$OPT_DIR/$OPT_SUBDIR/config.json" \
--getcapabilities "$OPT_DIR/$OPT_SUBDIR/gc" \
--mode "$MODE"`
# Get metadata for files in layerOrder.json and combine this data into 1 file
- if (( $MODE = "verbose" )) ; then
+ if (( $MODE == "verbose" )) ; then
echo "Get metadata for files in layerOrder.json and combine this data into 1 file"
fi
rm -rf "$OPT_DIR/$OPT_SUBDIR/layer-metadata"
mkdir -p "$OPT_DIR/$OPT_SUBDIR/layer-metadata"
- `node $SCRIPTS_DIR/getVisMetadata.js \
+ echo `node $SCRIPTS_DIR/getVisMetadata.js \
--features "$BUILD_DIR/features.json" \
--layerOrder "$BUILD_DIR/config/wv.json/layerOrder.json" \
--layerMetadata "$OPT_DIR/$OPT_SUBDIR/layer-metadata/all.json" \
--mode "$MODE"`
else
# Validate layers in wv.json with a JSON schema
- if (( $MODE = "verbose" )) ; then
+ if (( $MODE == "verbose" )) ; then
echo "Validate layers in wv.json with a JSON schema"
fi
- `node $SCRIPTS_DIR/validateConfigs.js \
+ echo `node $SCRIPTS_DIR/validateConfigs.js \
--inputDirectory "$SRC_DIR/common/config/wv.json/layers" \
--schemaFile "$BASE/schemas/layer-config.json" \
--mode "$MODE"`
@@ -98,7 +101,7 @@ else
if (( $MODE = "verbose" )) ; then
echo "Run extractConfigFromWMTS.js script with config.json"
fi
- `node $SCRIPTS_DIR/extractConfigFromWMTS.js \
+ echo `node $SCRIPTS_DIR/extractConfigFromWMTS.js \
--config "$BUILD_DIR/config.json" \
--inputDir "$BUILD_DIR/gc" \
--outputDir "$BUILD_DIR/_wmts" \
@@ -107,11 +110,11 @@ else
# Run processVectorStyles.js and move vectorstyles where we want them
if [ -e "$BUILD_DIR/gc/vectorstyles" ] ; then
- if (( $MODE = "verbose" )) ; then
+ if (( $MODE == "verbose" )) ; then
echo "Run processVectorStyles.js and move vectorstyles where we want them"
fi
mkdir -p "$BUILD_DIR/config/wv.json/vectorstyles"
- `node $SCRIPTS_DIR/processVectorStyles.js \
+ echo `node $SCRIPTS_DIR/processVectorStyles.js \
--inputDir "$BUILD_DIR/gc/vectorstyles" \
--outputDir "$BUILD_DIR/config/wv.json/vectorstyles" \
--mode "$MODE"`
@@ -119,11 +122,11 @@ else
# Run processVectorData.js and move vectordata where we want them
if [ -e "$BUILD_DIR/gc/vectordata" ] ; then
- if (( $MODE = "verbose" )) ; then
+ if (( $MODE == "verbose" )) ; then
echo "Run processVectorData.js and move vectordata where we want them"
fi
mkdir -p "$BUILD_DIR/config/wv.json/vectordata"
- `node $SCRIPTS_DIR/processVectorData.js \
+ echo `node $SCRIPTS_DIR/processVectorData.js \
--inputDir "$BUILD_DIR/gc/vectordata" \
--outputDir "$BUILD_DIR/config/wv.json/vectordata" \
--mode "$MODE"`
@@ -131,14 +134,14 @@ else
# Run processColormap.js and move colormaps where we want them
if [ -e "$BUILD_DIR/colormaps" ] ; then
- if (( $MODE = "verbose" )) ; then
+ if (( $MODE == "verbose" )) ; then
echo "Run processColormap.js and move colormaps where we want them"
fi
mkdir -p "$BUILD_DIR"/config/palettes
if [ -d "$BUILD_DIR"/gc/colormaps ] ; then
cp -r "$BUILD_DIR"/gc/colormaps "$BUILD_DIR"/colormaps/gc
fi
- `node $SCRIPTS_DIR/processColormap.js \
+ echo `node $SCRIPTS_DIR/processColormap.js \
--config "$OPT_DIR/$OPT_SUBDIR/config.json" \
--inputDir "$BUILD_DIR/colormaps" \
--outputDir "$BUILD_DIR/config/palettes" \
@@ -148,7 +151,7 @@ else
# Throw error if no categoryGroupOrder.json file present
if [ ! -e "$BUILD_DIR/config/wv.json/categoryGroupOrder.json" ] ; then
echo "categoryGroupOrder.json not found. Generating..."
- `node $SCRIPTS_DIR/generateCategoryGroupOrder.js \
+ echo `node $SCRIPTS_DIR/generateCategoryGroupOrder.js \
--inputDir "$SRC_DIR/common/config/wv.json/categories/" \
--outputDir "$SRC_DIR/common/config/wv.json/" \
--mode "$MODE"`
@@ -175,16 +178,16 @@ else
done
# Run mergeConfigWithWMTS.js to merge layer metadata from WMTS GC with worldview layer configs into wv.json
- if (( $MODE = "verbose" )) ; then
+ if (( $MODE == "verbose" )) ; then
echo "Run mergeConfigWithWMTS.js to merge layer metadata from WMTS GC with worldview layer configs into wv.json"
fi
- `node $SCRIPTS_DIR/mergeConfigWithWMTS.js \
+ echo `node $SCRIPTS_DIR/mergeConfigWithWMTS.js \
--inputDir "$BUILD_DIR/_wmts" \
--outputFile "$DEST_DIR/config/wv.json" \
--mode "$MODE"`
# Copy brand files from build to dest
- if (( $MODE = "verbose" )) ; then
+ if (( $MODE == "verbose" )) ; then
echo "Copy brand files from build to dest"
fi
cp -r "$BUILD_DIR/brand" "$DEST_DIR"
@@ -192,19 +195,19 @@ else
# Validate the options build
- if (( $MODE = "verbose" )) ; then
+ if (( $MODE == "verbose" )) ; then
echo "Validate the options build"
fi
- `node $SCRIPTS_DIR/validateOptions.js \
+ echo `node $SCRIPTS_DIR/validateOptions.js \
--optionsFile "$BUILD_DIR/config.json" \
--configDir "$DEST_DIR/config" \
--mode "$MODE"`
# Fetch preview images from WV Snapshots for any layers which they are missing
- if (( $MODE = "verbose" )) ; then
+ if (( $MODE == "verbose" )) ; then
echo "Fetch preview images from WV Snapshots for any layers which they are missing"
fi
- `node $SCRIPTS_DIR/fetchPreviewSnapshots.js \
+ echo `node $SCRIPTS_DIR/fetchPreviewSnapshots.js \
--wvJsonFile "$DEST_DIR/config/wv.json" \
--overridesFile "$OPT_DIR/common/previewLayerOverrides.json" \
--featuresFile "$BUILD_DIR/features.json" \