Interactive interface for browsing global, full-resolution satellite imagery
Add profiling mode to build options scripts for performance measurement and improve performance (#5640)
Patrick Mouldendd928d84232e
diff --git a/tasks/build-options/extractConfigFromWMTS.js b/tasks/build-options/extractConfigFromWMTS.jsindex d4525a77f..c3c894ca0 100644--- a/tasks/build-options/extractConfigFromWMTS.js+++ b/tasks/build-options/extractConfigFromWMTS.js@@ -27,6 +27,12 @@ const options = yargstype: 'string',description: 'wmts output directory'})+ .option('mode', {+ demandOption: true,+ alias: 'm',+ type: 'string',+ description: 'mode'+ }).epilog('Extracts configuration information from a WMTS GetCapabilities file, converts the XML to JSON')const { argv } = options@@ -75,7 +81,9 @@ class SkipException extends Error {* @throws {SkipException}*/async function main () {+ if (argv.mode === 'profile') console.time('extractConfigFromWMTS')for (const entry of entries) {+ if (argv.mode === 'profile') console.time(`extractConfigFromWMTS-${entry.source}`)wv.layers = {}wv.sources = {}wvMatrixSets = {}@@ -92,6 +100,7 @@ async function main () {totalErrorCount += errorCounttotalWarningCount += warningCounttotalLayerCount += layerCount+ if (argv.mode === 'profile') console.timeEnd(`extractConfigFromWMTS-${entry.source}`)}console.warn(`${prog}:${totalErrorCount} errors, ${totalWarningCount} warnings, ${totalLayerCount} layers`)@@ -99,6 +108,7 @@ async function main () {if (totalErrorCount > 0) {throw new Error(`${prog}: Error: ${totalErrorCount} errors occured`)}+ if (argv.mode === 'profile') console.timeEnd('extractConfigFromWMTS')}async function processEntry (entry) {@@ -135,23 +145,22 @@ async function processEntry (entry) {console.error(`${prog}: ERROR: [${gcId}] No layers\n`)return { errorCount, warningCount, layerCount }}-- for (const gcLayer of gcContents.Layer) {- try {- layerCount += 1- await processLayer(gcLayer, wvLayers, entry)- } catch (error) {- if (error instanceof SkipException) {- warningCount += 1- console.warn(`${prog}: WARNING: [${gcId}] Skipping\n`)- } else {- errorCount += 1- console.error(error.stack)- const ident = gcLayer['ows:Identifier']._text- console.error(`${prog}: ERROR: [${gcId}:${ident}] ${error}\n`)- }+ const promises = gcContents.Layer.map((gcLayer) => {+ layerCount += 1+ return processLayer(gcLayer, wvLayers, entry)+ })+ const results = await Promise.allSettled(promises)+ const rejected = results.filter(result => result.status === 'rejected')+ rejected.forEach((reject) => {+ const error = reject.reason+ if (error instanceof SkipException) {+ warningCount += 1+ console.warn(`${prog}: WARNING: [${gcId}] Skipping\n`)+ } else {+ errorCount += 1+ console.error(`${prog}: ERROR: [${gcId}] ${error}\n`)}- }+ })if (gcContents.TileMatrixSet === 'Object') {processMatrixSet(gcContents.TileMatrixSet, entry)diff --git a/tasks/build-options/fetchPreviewSnapshots.js b/tasks/build-options/fetchPreviewSnapshots.jsindex 7aafe8600..bab74a237 100644--- a/tasks/build-options/fetchPreviewSnapshots.js+++ b/tasks/build-options/fetchPreviewSnapshots.js@@ -26,6 +26,12 @@ const options = yargstype: 'string',description: 'features file'})+ .option('mode', {+ demandOption: true,+ alias: 'm',+ type: 'string',+ description: 'mode'+ }).epilog('Fetch preview images from WV Snapshots for any layers which they are missing.')const { argv } = options@@ -80,6 +86,7 @@ const referenceLayers = {}async function main () {+ if (argv.mode === 'profile') console.time('fetchPreviewSnapshots')// Check to see if this feature is enabled in features.json before continuingconst featuresFileContent = fs.readFileSync(featuresFile, { encoding: 'utf-8' })const featuresDict = JSON.parse(featuresFileContent)@@ -121,6 +128,7 @@ async function main () {if (totalFailureCount === 0 && totalSuccessCount === 0) {console.warn(`\n${prog}: No snapshots were retrieved. All layers found in wv.json have existing preview images!`)}+ if (argv.mode === 'profile') console.timeEnd('fetchPreviewSnapshots')}async function trackBadSnapshots (layerId, projection, request, imgFile, badSnapshots) {diff --git a/tasks/build-options/getCapabilities.js b/tasks/build-options/getCapabilities.jsindex 04b7d0cbc..e704add62 100644--- a/tasks/build-options/getCapabilities.js+++ b/tasks/build-options/getCapabilities.js@@ -49,6 +49,7 @@ const configData = fs.readFileSync(configFile)const config = JSON.parse(configData)async function main () {+ if (argv.mode === 'profile') console.time('getCapabilities')if (!fs.existsSync(outputDir)) {fs.mkdirSync(outputDir)}@@ -64,6 +65,7 @@ async function main () {if (vectordata) {await gatherProcess(vectordata, 'vectordata files', vectordataDir, '.json')}+ if (argv.mode === 'profile') console.timeEnd('getCapabilities')}async function getCapabilities () {diff --git a/tasks/build-options/getVisMetadata.js b/tasks/build-options/getVisMetadata.jsindex 88f0da4b1..db208a0b9 100644--- a/tasks/build-options/getVisMetadata.js+++ b/tasks/build-options/getVisMetadata.js@@ -115,15 +115,17 @@ const useKeys = []async function main (url) {+ if (argv.mode === 'profile') console.time('getVisMetadata')layerOrder = layerOrder.layerOrderlayerOrder = layerOrder.filter(x => !skipLayers.includes(x))console.warn(`${prog}: Fetching ${layerOrder.length} layer-metadata files`)- for (const layerId of layerOrder) {- if (!layerId.includes('_STD') && !layerId.includes('_NRT')) {- await getMetadata(layerId, url)- }- }+ const promises = layerOrder.map((layerId) => {+ if (layerId.includes('_STD') && layerId.includes('_NRT')) return null++ return getMetadata(layerId, url)+ })+ await Promise.allSettled(promises)const layers = Object.keys(layerMetadata).sort().reduce((obj, key) => {@@ -135,6 +137,7 @@ async function main (url) {await fs.writeFileSync(outputFile, JSON.stringify({ layers }))console.warn(`${prog}: Combined all layer-metadata files into ${path.parse(outputFile).base}`)+ if (argv.mode === 'profile') console.timeEnd('getVisMetadata')}async function getDAAC (metadata) {diff --git a/tasks/build-options/mergeConfigWithWMTS.js b/tasks/build-options/mergeConfigWithWMTS.jsindex 0ce247d13..c2bb71f52 100644--- a/tasks/build-options/mergeConfigWithWMTS.js+++ b/tasks/build-options/mergeConfigWithWMTS.js@@ -18,6 +18,12 @@ const options = yargstype: 'string',description: 'wmts output directory'})+ .option('mode', {+ demandOption: true,+ alias: 'm',+ type: 'string',+ description: 'mode'+ }).epilog('Concatenates all configuration items a directory into one configuration file.')const { argv } = options@@ -37,6 +43,7 @@ const outputData = JSON.parse(fs.readFileSync(outputFile, 'utf-8'))let fileCount = 0async function main () {+ if (argv.mode === 'profile') console.time('mergeConfigWithWMTS')layers = newConf.layersdelete layers.sourcesdelete newConf.layers@@ -50,6 +57,7 @@ async function main () {await fs.writeFileSync(outputFile, JSON.stringify(newConf, jsonOptions))console.warn(`${prog}: ${fileCount} file(s) merged into ${path.basename(outputFile)}`)+ if (argv.mode === 'profile') console.timeEnd('mergeConfigWithWMTS')}async function processFiles () {diff --git a/tasks/build-options/processColormap.js b/tasks/build-options/processColormap.jsindex 54efae9dc..c3064a86c 100755--- a/tasks/build-options/processColormap.js+++ b/tasks/build-options/processColormap.js@@ -29,6 +29,12 @@ const options = yargstype: 'string',description: 'wmts output directory'})+ .option('mode', {+ demandOption: true,+ alias: 'm',+ type: 'string',+ description: 'mode'+ }).epilog('Converts colormaps to JSON files')const { argv } = options@@ -64,6 +70,7 @@ const walk = (dir) => {}async function main () {+ if (argv.mode === 'profile') console.time('processColormap')const files = await walk(inputDir)for (const file of files) {try {@@ -82,6 +89,7 @@ async function main () {if (errorCount > 0) {throw new Error(`${prog}: Error: ${errorCount} errors occured`)}+ if (argv.mode === 'profile') console.timeEnd('processColormap')}function toList (v) {diff --git a/tasks/build-options/processVectorData.js b/tasks/build-options/processVectorData.jsindex 78c9f4fbc..e600a4f3f 100644--- a/tasks/build-options/processVectorData.js+++ b/tasks/build-options/processVectorData.js@@ -23,6 +23,12 @@ const options = yargstype: 'string',description: 'wmts output directory'})+ .option('mode', {+ demandOption: true,+ alias: 'm',+ type: 'string',+ description: 'mode'+ }).epilog('Extracts vector data information from GetCapabilities')const { argv } = options@@ -34,6 +40,7 @@ const inputDir = argv.inputDirconst outputDir = argv.outputDirasync function main () {+ if (argv.mode === 'profile') console.time('processVectorData')let fileCount = 0let errorCount = 0@@ -52,6 +59,7 @@ async function main () {if (errorCount > 0) {throw new Error(`${prog}: Error: ${errorCount} errors occured`)}+ if (argv.mode === 'profile') console.timeEnd('processVectorData')}async function copyFileAsync (file) {diff --git a/tasks/build-options/processVectorStyles.js b/tasks/build-options/processVectorStyles.jsindex bfcff1e30..6c8c7014a 100644--- a/tasks/build-options/processVectorStyles.js+++ b/tasks/build-options/processVectorStyles.js@@ -23,6 +23,12 @@ const options = yargstype: 'string',description: 'wmts output directory'})+ .option('mode', {+ demandOption: true,+ alias: 'm',+ type: 'string',+ description: 'mode'+ }).epilog('Extracts vector style information from GetCapabilities')const { argv } = options@@ -34,6 +40,7 @@ const inputDir = argv.inputDirconst outputDir = argv.outputDirasync function main () {+ if (argv.mode === 'profile') console.time('processVectorStyles')let fileCount = 0let errorCount = 0@@ -52,6 +59,7 @@ async function main () {if (errorCount > 0) {throw new Error(`${prog}: Error: ${errorCount} errors occured`)}+ if (argv.mode === 'profile') console.timeEnd('processVectorStyles')}async function copyFileAsync (file) {diff --git a/tasks/build-options/validateConfigs.js b/tasks/build-options/validateConfigs.jsindex 48dce4f72..28c2d6e29 100644--- a/tasks/build-options/validateConfigs.js+++ b/tasks/build-options/validateConfigs.js@@ -52,6 +52,7 @@ const invalidJsonFiles = []console.warn(`${prog}: Validating layer configs...`)async function main () {+ if (argv.mode === 'profile') console.time('validateConfigs')let files = globSync(inputDirectory + '/**/*')files = files.filter(file => file.endsWith('.json'))for (const filePath of files) {@@ -59,9 +60,11 @@ async function main () {}if (invalidJsonFiles.length) {if (argv.mode === 'verbose') console.warn(`${prog}: Invalid JSON files: ${invalidJsonFiles}`)+ if (argv.mode === 'profile') console.timeEnd('validateConfigs')throw new Error(`${prog}: FAILED: ${invalidJsonFiles.length} layer configs failed validation.`)} else {console.warn(`${prog}: PASSED: All layer configs passed validation!`)+ if (argv.mode === 'profile') console.timeEnd('validateConfigs')}}diff --git a/tasks/build-options/validateOptions.js b/tasks/build-options/validateOptions.jsindex 91e6eb762..f21943788 100644--- a/tasks/build-options/validateOptions.js+++ b/tasks/build-options/validateOptions.js@@ -19,6 +19,12 @@ const options = yargstype: 'string',description: 'config directory'})+ .option('mode', {+ demandOption: true,+ alias: 'm',+ type: 'string',+ description: 'mode'+ }).epilog('Validates and corrects the configuration files.')const { argv } = options@@ -75,6 +81,7 @@ if (tolerant) warn('Validation enforcement disabled')let startDate = moment.max()async function main () {+ if (argv.mode === 'profile') console.time('validateOptions')for (const layerId of Object.keys(wv.layers)) {let layer = wv.layers[layerId]if (layerId !== layer.id) {@@ -186,6 +193,7 @@ async function main () {if (errorCount > 0) {throw new Error(`${prog}: Error: ${errorCount} errors occured`)}+ if (argv.mode === 'profile') console.timeEnd('validateOptions')}main().catch((err) => {diff --git a/tasks/buildOptions.sh b/tasks/buildOptions.shindex 6f0124983..00295dc0b 100755--- a/tasks/buildOptions.sh+++ b/tasks/buildOptions.sh@@ -9,12 +9,15 @@ SCRIPTS_DIR="$BASE/tasks/build-options"MODE="default"-while getopts ":v" option; do+while getopts ":vp" option; doecho "Option -$option set"case $option inv)echo "Verbose Mode Activated"MODE="verbose";;+ p)+ echo "Profile Mode Activated"+ MODE="profile";;esacdone@@ -58,33 +61,33 @@ mkdir -p "$BUILD_DIR/colormaps"# If $FETCH_GC is set, make various API requestsif [ "$FETCH_GC" ] ; then# Fetch GC files and create colormaps, vectordata and vectorstyle files- if (( $MODE = "verbose" )) ; then+ if (( $MODE == "verbose" )) ; thenecho "Fetch GC files and create colormaps, vectordata and vectorstyle files"firm -rf "$OPT_DIR/$OPT_SUBDIR/gc/*"rm -rf "$OPT_DIR/$OPT_SUBDIR/colormaps/gc/*"- `node $SCRIPTS_DIR/getCapabilities.js \+ echo `node $SCRIPTS_DIR/getCapabilities.js \--config "$OPT_DIR/$OPT_SUBDIR/config.json" \--getcapabilities "$OPT_DIR/$OPT_SUBDIR/gc" \--mode "$MODE"`# Get metadata for files in layerOrder.json and combine this data into 1 file- if (( $MODE = "verbose" )) ; then+ if (( $MODE == "verbose" )) ; thenecho "Get metadata for files in layerOrder.json and combine this data into 1 file"firm -rf "$OPT_DIR/$OPT_SUBDIR/layer-metadata"mkdir -p "$OPT_DIR/$OPT_SUBDIR/layer-metadata"- `node $SCRIPTS_DIR/getVisMetadata.js \+ echo `node $SCRIPTS_DIR/getVisMetadata.js \--features "$BUILD_DIR/features.json" \--layerOrder "$BUILD_DIR/config/wv.json/layerOrder.json" \--layerMetadata "$OPT_DIR/$OPT_SUBDIR/layer-metadata/all.json" \--mode "$MODE"`else# Validate layers in wv.json with a JSON schema- if (( $MODE = "verbose" )) ; then+ if (( $MODE == "verbose" )) ; thenecho "Validate layers in wv.json with a JSON schema"fi- `node $SCRIPTS_DIR/validateConfigs.js \+ echo `node $SCRIPTS_DIR/validateConfigs.js \--inputDirectory "$SRC_DIR/common/config/wv.json/layers" \--schemaFile "$BASE/schemas/layer-config.json" \--mode "$MODE"`@@ -98,7 +101,7 @@ elseif (( $MODE = "verbose" )) ; thenecho "Run extractConfigFromWMTS.js script with config.json"fi- `node $SCRIPTS_DIR/extractConfigFromWMTS.js \+ echo `node $SCRIPTS_DIR/extractConfigFromWMTS.js \--config "$BUILD_DIR/config.json" \--inputDir "$BUILD_DIR/gc" \--outputDir "$BUILD_DIR/_wmts" \@@ -107,11 +110,11 @@ else# Run processVectorStyles.js and move vectorstyles where we want themif [ -e "$BUILD_DIR/gc/vectorstyles" ] ; then- if (( $MODE = "verbose" )) ; then+ if (( $MODE == "verbose" )) ; thenecho "Run processVectorStyles.js and move vectorstyles where we want them"fimkdir -p "$BUILD_DIR/config/wv.json/vectorstyles"- `node $SCRIPTS_DIR/processVectorStyles.js \+ echo `node $SCRIPTS_DIR/processVectorStyles.js \--inputDir "$BUILD_DIR/gc/vectorstyles" \--outputDir "$BUILD_DIR/config/wv.json/vectorstyles" \--mode "$MODE"`@@ -119,11 +122,11 @@ else# Run processVectorData.js and move vectordata where we want themif [ -e "$BUILD_DIR/gc/vectordata" ] ; then- if (( $MODE = "verbose" )) ; then+ if (( $MODE == "verbose" )) ; thenecho "Run processVectorData.js and move vectordata where we want them"fimkdir -p "$BUILD_DIR/config/wv.json/vectordata"- `node $SCRIPTS_DIR/processVectorData.js \+ echo `node $SCRIPTS_DIR/processVectorData.js \--inputDir "$BUILD_DIR/gc/vectordata" \--outputDir "$BUILD_DIR/config/wv.json/vectordata" \--mode "$MODE"`@@ -131,14 +134,14 @@ else# Run processColormap.js and move colormaps where we want themif [ -e "$BUILD_DIR/colormaps" ] ; then- if (( $MODE = "verbose" )) ; then+ if (( $MODE == "verbose" )) ; thenecho "Run processColormap.js and move colormaps where we want them"fimkdir -p "$BUILD_DIR"/config/palettesif [ -d "$BUILD_DIR"/gc/colormaps ] ; thencp -r "$BUILD_DIR"/gc/colormaps "$BUILD_DIR"/colormaps/gcfi- `node $SCRIPTS_DIR/processColormap.js \+ echo `node $SCRIPTS_DIR/processColormap.js \--config "$OPT_DIR/$OPT_SUBDIR/config.json" \--inputDir "$BUILD_DIR/colormaps" \--outputDir "$BUILD_DIR/config/palettes" \@@ -148,7 +151,7 @@ else# Throw error if no categoryGroupOrder.json file presentif [ ! -e "$BUILD_DIR/config/wv.json/categoryGroupOrder.json" ] ; thenecho "categoryGroupOrder.json not found. Generating..."- `node $SCRIPTS_DIR/generateCategoryGroupOrder.js \+ echo `node $SCRIPTS_DIR/generateCategoryGroupOrder.js \--inputDir "$SRC_DIR/common/config/wv.json/categories/" \--outputDir "$SRC_DIR/common/config/wv.json/" \--mode "$MODE"`@@ -175,16 +178,16 @@ elsedone# Run mergeConfigWithWMTS.js to merge layer metadata from WMTS GC with worldview layer configs into wv.json- if (( $MODE = "verbose" )) ; then+ if (( $MODE == "verbose" )) ; thenecho "Run mergeConfigWithWMTS.js to merge layer metadata from WMTS GC with worldview layer configs into wv.json"fi- `node $SCRIPTS_DIR/mergeConfigWithWMTS.js \+ echo `node $SCRIPTS_DIR/mergeConfigWithWMTS.js \--inputDir "$BUILD_DIR/_wmts" \--outputFile "$DEST_DIR/config/wv.json" \--mode "$MODE"`# Copy brand files from build to dest- if (( $MODE = "verbose" )) ; then+ if (( $MODE == "verbose" )) ; thenecho "Copy brand files from build to dest"ficp -r "$BUILD_DIR/brand" "$DEST_DIR"@@ -192,19 +195,19 @@ else# Validate the options build- if (( $MODE = "verbose" )) ; then+ if (( $MODE == "verbose" )) ; thenecho "Validate the options build"fi- `node $SCRIPTS_DIR/validateOptions.js \+ echo `node $SCRIPTS_DIR/validateOptions.js \--optionsFile "$BUILD_DIR/config.json" \--configDir "$DEST_DIR/config" \--mode "$MODE"`# Fetch preview images from WV Snapshots for any layers which they are missing- if (( $MODE = "verbose" )) ; then+ if (( $MODE == "verbose" )) ; thenecho "Fetch preview images from WV Snapshots for any layers which they are missing"fi- `node $SCRIPTS_DIR/fetchPreviewSnapshots.js \+ echo `node $SCRIPTS_DIR/fetchPreviewSnapshots.js \--wvJsonFile "$DEST_DIR/config/wv.json" \--overridesFile "$OPT_DIR/common/previewLayerOverrides.json" \--featuresFile "$BUILD_DIR/features.json" \