Interactive interface for browsing global, full-resolution satellite imagery
Revert "Add profiling mode to build options scripts for performance measureme…" (#5642)
Patrick Mouldenac91d91dd538
diff --git a/tasks/build-options/extractConfigFromWMTS.js b/tasks/build-options/extractConfigFromWMTS.jsindex c3c894ca0..d4525a77f 100644--- a/tasks/build-options/extractConfigFromWMTS.js+++ b/tasks/build-options/extractConfigFromWMTS.js@@ -27,12 +27,6 @@ const options = yargstype: 'string',description: 'wmts output directory'})- .option('mode', {- demandOption: true,- alias: 'm',- type: 'string',- description: 'mode'- }).epilog('Extracts configuration information from a WMTS GetCapabilities file, converts the XML to JSON')const { argv } = options@@ -81,9 +75,7 @@ class SkipException extends Error {* @throws {SkipException}*/async function main () {- if (argv.mode === 'profile') console.time('extractConfigFromWMTS')for (const entry of entries) {- if (argv.mode === 'profile') console.time(`extractConfigFromWMTS-${entry.source}`)wv.layers = {}wv.sources = {}wvMatrixSets = {}@@ -100,7 +92,6 @@ async function main () {totalErrorCount += errorCounttotalWarningCount += warningCounttotalLayerCount += layerCount- if (argv.mode === 'profile') console.timeEnd(`extractConfigFromWMTS-${entry.source}`)}console.warn(`${prog}:${totalErrorCount} errors, ${totalWarningCount} warnings, ${totalLayerCount} layers`)@@ -108,7 +99,6 @@ async function main () {if (totalErrorCount > 0) {throw new Error(`${prog}: Error: ${totalErrorCount} errors occured`)}- if (argv.mode === 'profile') console.timeEnd('extractConfigFromWMTS')}async function processEntry (entry) {@@ -145,22 +135,23 @@ async function processEntry (entry) {console.error(`${prog}: ERROR: [${gcId}] No layers\n`)return { errorCount, warningCount, layerCount }}- const promises = gcContents.Layer.map((gcLayer) => {- layerCount += 1- return processLayer(gcLayer, wvLayers, entry)- })- const results = await Promise.allSettled(promises)- const rejected = results.filter(result => result.status === 'rejected')- rejected.forEach((reject) => {- const error = reject.reason- if (error instanceof SkipException) {- warningCount += 1- console.warn(`${prog}: WARNING: [${gcId}] Skipping\n`)- } else {- errorCount += 1- console.error(`${prog}: ERROR: [${gcId}] ${error}\n`)++ for (const gcLayer of gcContents.Layer) {+ try {+ layerCount += 1+ await processLayer(gcLayer, wvLayers, entry)+ } catch (error) {+ if (error instanceof SkipException) {+ warningCount += 1+ console.warn(`${prog}: WARNING: [${gcId}] Skipping\n`)+ } else {+ errorCount += 1+ console.error(error.stack)+ const ident = gcLayer['ows:Identifier']._text+ console.error(`${prog}: ERROR: [${gcId}:${ident}] ${error}\n`)+ }}- })+ }if (gcContents.TileMatrixSet === 'Object') {processMatrixSet(gcContents.TileMatrixSet, entry)diff --git a/tasks/build-options/fetchPreviewSnapshots.js b/tasks/build-options/fetchPreviewSnapshots.jsindex bab74a237..7aafe8600 100644--- a/tasks/build-options/fetchPreviewSnapshots.js+++ b/tasks/build-options/fetchPreviewSnapshots.js@@ -26,12 +26,6 @@ const options = yargstype: 'string',description: 'features file'})- .option('mode', {- demandOption: true,- alias: 'm',- type: 'string',- description: 'mode'- }).epilog('Fetch preview images from WV Snapshots for any layers which they are missing.')const { argv } = options@@ -86,7 +80,6 @@ const referenceLayers = {}async function main () {- if (argv.mode === 'profile') console.time('fetchPreviewSnapshots')// Check to see if this feature is enabled in features.json before continuingconst featuresFileContent = fs.readFileSync(featuresFile, { encoding: 'utf-8' })const featuresDict = JSON.parse(featuresFileContent)@@ -128,7 +121,6 @@ async function main () {if (totalFailureCount === 0 && totalSuccessCount === 0) {console.warn(`\n${prog}: No snapshots were retrieved. All layers found in wv.json have existing preview images!`)}- if (argv.mode === 'profile') console.timeEnd('fetchPreviewSnapshots')}async function trackBadSnapshots (layerId, projection, request, imgFile, badSnapshots) {diff --git a/tasks/build-options/getCapabilities.js b/tasks/build-options/getCapabilities.jsindex e704add62..04b7d0cbc 100644--- a/tasks/build-options/getCapabilities.js+++ b/tasks/build-options/getCapabilities.js@@ -49,7 +49,6 @@ const configData = fs.readFileSync(configFile)const config = JSON.parse(configData)async function main () {- if (argv.mode === 'profile') console.time('getCapabilities')if (!fs.existsSync(outputDir)) {fs.mkdirSync(outputDir)}@@ -65,7 +64,6 @@ async function main () {if (vectordata) {await gatherProcess(vectordata, 'vectordata files', vectordataDir, '.json')}- if (argv.mode === 'profile') console.timeEnd('getCapabilities')}async function getCapabilities () {diff --git a/tasks/build-options/getVisMetadata.js b/tasks/build-options/getVisMetadata.jsindex db208a0b9..88f0da4b1 100644--- a/tasks/build-options/getVisMetadata.js+++ b/tasks/build-options/getVisMetadata.js@@ -115,17 +115,15 @@ const useKeys = []async function main (url) {- if (argv.mode === 'profile') console.time('getVisMetadata')layerOrder = layerOrder.layerOrderlayerOrder = layerOrder.filter(x => !skipLayers.includes(x))console.warn(`${prog}: Fetching ${layerOrder.length} layer-metadata files`)- const promises = layerOrder.map((layerId) => {- if (layerId.includes('_STD') && layerId.includes('_NRT')) return null-- return getMetadata(layerId, url)- })- await Promise.allSettled(promises)+ for (const layerId of layerOrder) {+ if (!layerId.includes('_STD') && !layerId.includes('_NRT')) {+ await getMetadata(layerId, url)+ }+ }const layers = Object.keys(layerMetadata).sort().reduce((obj, key) => {@@ -137,7 +135,6 @@ async function main (url) {await fs.writeFileSync(outputFile, JSON.stringify({ layers }))console.warn(`${prog}: Combined all layer-metadata files into ${path.parse(outputFile).base}`)- if (argv.mode === 'profile') console.timeEnd('getVisMetadata')}async function getDAAC (metadata) {diff --git a/tasks/build-options/mergeConfigWithWMTS.js b/tasks/build-options/mergeConfigWithWMTS.jsindex c2bb71f52..0ce247d13 100644--- a/tasks/build-options/mergeConfigWithWMTS.js+++ b/tasks/build-options/mergeConfigWithWMTS.js@@ -18,12 +18,6 @@ const options = yargstype: 'string',description: 'wmts output directory'})- .option('mode', {- demandOption: true,- alias: 'm',- type: 'string',- description: 'mode'- }).epilog('Concatenates all configuration items a directory into one configuration file.')const { argv } = options@@ -43,7 +37,6 @@ const outputData = JSON.parse(fs.readFileSync(outputFile, 'utf-8'))let fileCount = 0async function main () {- if (argv.mode === 'profile') console.time('mergeConfigWithWMTS')layers = newConf.layersdelete layers.sourcesdelete newConf.layers@@ -57,7 +50,6 @@ async function main () {await fs.writeFileSync(outputFile, JSON.stringify(newConf, jsonOptions))console.warn(`${prog}: ${fileCount} file(s) merged into ${path.basename(outputFile)}`)- if (argv.mode === 'profile') console.timeEnd('mergeConfigWithWMTS')}async function processFiles () {diff --git a/tasks/build-options/processColormap.js b/tasks/build-options/processColormap.jsindex c3064a86c..54efae9dc 100755--- a/tasks/build-options/processColormap.js+++ b/tasks/build-options/processColormap.js@@ -29,12 +29,6 @@ const options = yargstype: 'string',description: 'wmts output directory'})- .option('mode', {- demandOption: true,- alias: 'm',- type: 'string',- description: 'mode'- }).epilog('Converts colormaps to JSON files')const { argv } = options@@ -70,7 +64,6 @@ const walk = (dir) => {}async function main () {- if (argv.mode === 'profile') console.time('processColormap')const files = await walk(inputDir)for (const file of files) {try {@@ -89,7 +82,6 @@ async function main () {if (errorCount > 0) {throw new Error(`${prog}: Error: ${errorCount} errors occured`)}- if (argv.mode === 'profile') console.timeEnd('processColormap')}function toList (v) {diff --git a/tasks/build-options/processVectorData.js b/tasks/build-options/processVectorData.jsindex e600a4f3f..78c9f4fbc 100644--- a/tasks/build-options/processVectorData.js+++ b/tasks/build-options/processVectorData.js@@ -23,12 +23,6 @@ const options = yargstype: 'string',description: 'wmts output directory'})- .option('mode', {- demandOption: true,- alias: 'm',- type: 'string',- description: 'mode'- }).epilog('Extracts vector data information from GetCapabilities')const { argv } = options@@ -40,7 +34,6 @@ const inputDir = argv.inputDirconst outputDir = argv.outputDirasync function main () {- if (argv.mode === 'profile') console.time('processVectorData')let fileCount = 0let errorCount = 0@@ -59,7 +52,6 @@ async function main () {if (errorCount > 0) {throw new Error(`${prog}: Error: ${errorCount} errors occured`)}- if (argv.mode === 'profile') console.timeEnd('processVectorData')}async function copyFileAsync (file) {diff --git a/tasks/build-options/processVectorStyles.js b/tasks/build-options/processVectorStyles.jsindex 6c8c7014a..bfcff1e30 100644--- a/tasks/build-options/processVectorStyles.js+++ b/tasks/build-options/processVectorStyles.js@@ -23,12 +23,6 @@ const options = yargstype: 'string',description: 'wmts output directory'})- .option('mode', {- demandOption: true,- alias: 'm',- type: 'string',- description: 'mode'- }).epilog('Extracts vector style information from GetCapabilities')const { argv } = options@@ -40,7 +34,6 @@ const inputDir = argv.inputDirconst outputDir = argv.outputDirasync function main () {- if (argv.mode === 'profile') console.time('processVectorStyles')let fileCount = 0let errorCount = 0@@ -59,7 +52,6 @@ async function main () {if (errorCount > 0) {throw new Error(`${prog}: Error: ${errorCount} errors occured`)}- if (argv.mode === 'profile') console.timeEnd('processVectorStyles')}async function copyFileAsync (file) {diff --git a/tasks/build-options/validateConfigs.js b/tasks/build-options/validateConfigs.jsindex 28c2d6e29..48dce4f72 100644--- a/tasks/build-options/validateConfigs.js+++ b/tasks/build-options/validateConfigs.js@@ -52,7 +52,6 @@ const invalidJsonFiles = []console.warn(`${prog}: Validating layer configs...`)async function main () {- if (argv.mode === 'profile') console.time('validateConfigs')let files = globSync(inputDirectory + '/**/*')files = files.filter(file => file.endsWith('.json'))for (const filePath of files) {@@ -60,11 +59,9 @@ async function main () {}if (invalidJsonFiles.length) {if (argv.mode === 'verbose') console.warn(`${prog}: Invalid JSON files: ${invalidJsonFiles}`)- if (argv.mode === 'profile') console.timeEnd('validateConfigs')throw new Error(`${prog}: FAILED: ${invalidJsonFiles.length} layer configs failed validation.`)} else {console.warn(`${prog}: PASSED: All layer configs passed validation!`)- if (argv.mode === 'profile') console.timeEnd('validateConfigs')}}diff --git a/tasks/build-options/validateOptions.js b/tasks/build-options/validateOptions.jsindex f21943788..91e6eb762 100644--- a/tasks/build-options/validateOptions.js+++ b/tasks/build-options/validateOptions.js@@ -19,12 +19,6 @@ const options = yargstype: 'string',description: 'config directory'})- .option('mode', {- demandOption: true,- alias: 'm',- type: 'string',- description: 'mode'- }).epilog('Validates and corrects the configuration files.')const { argv } = options@@ -81,7 +75,6 @@ if (tolerant) warn('Validation enforcement disabled')let startDate = moment.max()async function main () {- if (argv.mode === 'profile') console.time('validateOptions')for (const layerId of Object.keys(wv.layers)) {let layer = wv.layers[layerId]if (layerId !== layer.id) {@@ -193,7 +186,6 @@ async function main () {if (errorCount > 0) {throw new Error(`${prog}: Error: ${errorCount} errors occured`)}- if (argv.mode === 'profile') console.timeEnd('validateOptions')}main().catch((err) => {diff --git a/tasks/buildOptions.sh b/tasks/buildOptions.shindex 00295dc0b..6f0124983 100755--- a/tasks/buildOptions.sh+++ b/tasks/buildOptions.sh@@ -9,15 +9,12 @@ SCRIPTS_DIR="$BASE/tasks/build-options"MODE="default"-while getopts ":vp" option; do+while getopts ":v" option; doecho "Option -$option set"case $option inv)echo "Verbose Mode Activated"MODE="verbose";;- p)- echo "Profile Mode Activated"- MODE="profile";;esacdone@@ -61,33 +58,33 @@ mkdir -p "$BUILD_DIR/colormaps"# If $FETCH_GC is set, make various API requestsif [ "$FETCH_GC" ] ; then# Fetch GC files and create colormaps, vectordata and vectorstyle files- if (( $MODE == "verbose" )) ; then+ if (( $MODE = "verbose" )) ; thenecho "Fetch GC files and create colormaps, vectordata and vectorstyle files"firm -rf "$OPT_DIR/$OPT_SUBDIR/gc/*"rm -rf "$OPT_DIR/$OPT_SUBDIR/colormaps/gc/*"- echo `node $SCRIPTS_DIR/getCapabilities.js \+ `node $SCRIPTS_DIR/getCapabilities.js \--config "$OPT_DIR/$OPT_SUBDIR/config.json" \--getcapabilities "$OPT_DIR/$OPT_SUBDIR/gc" \--mode "$MODE"`# Get metadata for files in layerOrder.json and combine this data into 1 file- if (( $MODE == "verbose" )) ; then+ if (( $MODE = "verbose" )) ; thenecho "Get metadata for files in layerOrder.json and combine this data into 1 file"firm -rf "$OPT_DIR/$OPT_SUBDIR/layer-metadata"mkdir -p "$OPT_DIR/$OPT_SUBDIR/layer-metadata"- echo `node $SCRIPTS_DIR/getVisMetadata.js \+ `node $SCRIPTS_DIR/getVisMetadata.js \--features "$BUILD_DIR/features.json" \--layerOrder "$BUILD_DIR/config/wv.json/layerOrder.json" \--layerMetadata "$OPT_DIR/$OPT_SUBDIR/layer-metadata/all.json" \--mode "$MODE"`else# Validate layers in wv.json with a JSON schema- if (( $MODE == "verbose" )) ; then+ if (( $MODE = "verbose" )) ; thenecho "Validate layers in wv.json with a JSON schema"fi- echo `node $SCRIPTS_DIR/validateConfigs.js \+ `node $SCRIPTS_DIR/validateConfigs.js \--inputDirectory "$SRC_DIR/common/config/wv.json/layers" \--schemaFile "$BASE/schemas/layer-config.json" \--mode "$MODE"`@@ -101,7 +98,7 @@ elseif (( $MODE = "verbose" )) ; thenecho "Run extractConfigFromWMTS.js script with config.json"fi- echo `node $SCRIPTS_DIR/extractConfigFromWMTS.js \+ `node $SCRIPTS_DIR/extractConfigFromWMTS.js \--config "$BUILD_DIR/config.json" \--inputDir "$BUILD_DIR/gc" \--outputDir "$BUILD_DIR/_wmts" \@@ -110,11 +107,11 @@ else# Run processVectorStyles.js and move vectorstyles where we want themif [ -e "$BUILD_DIR/gc/vectorstyles" ] ; then- if (( $MODE == "verbose" )) ; then+ if (( $MODE = "verbose" )) ; thenecho "Run processVectorStyles.js and move vectorstyles where we want them"fimkdir -p "$BUILD_DIR/config/wv.json/vectorstyles"- echo `node $SCRIPTS_DIR/processVectorStyles.js \+ `node $SCRIPTS_DIR/processVectorStyles.js \--inputDir "$BUILD_DIR/gc/vectorstyles" \--outputDir "$BUILD_DIR/config/wv.json/vectorstyles" \--mode "$MODE"`@@ -122,11 +119,11 @@ else# Run processVectorData.js and move vectordata where we want themif [ -e "$BUILD_DIR/gc/vectordata" ] ; then- if (( $MODE == "verbose" )) ; then+ if (( $MODE = "verbose" )) ; thenecho "Run processVectorData.js and move vectordata where we want them"fimkdir -p "$BUILD_DIR/config/wv.json/vectordata"- echo `node $SCRIPTS_DIR/processVectorData.js \+ `node $SCRIPTS_DIR/processVectorData.js \--inputDir "$BUILD_DIR/gc/vectordata" \--outputDir "$BUILD_DIR/config/wv.json/vectordata" \--mode "$MODE"`@@ -134,14 +131,14 @@ else# Run processColormap.js and move colormaps where we want themif [ -e "$BUILD_DIR/colormaps" ] ; then- if (( $MODE == "verbose" )) ; then+ if (( $MODE = "verbose" )) ; thenecho "Run processColormap.js and move colormaps where we want them"fimkdir -p "$BUILD_DIR"/config/palettesif [ -d "$BUILD_DIR"/gc/colormaps ] ; thencp -r "$BUILD_DIR"/gc/colormaps "$BUILD_DIR"/colormaps/gcfi- echo `node $SCRIPTS_DIR/processColormap.js \+ `node $SCRIPTS_DIR/processColormap.js \--config "$OPT_DIR/$OPT_SUBDIR/config.json" \--inputDir "$BUILD_DIR/colormaps" \--outputDir "$BUILD_DIR/config/palettes" \@@ -151,7 +148,7 @@ else# Throw error if no categoryGroupOrder.json file presentif [ ! -e "$BUILD_DIR/config/wv.json/categoryGroupOrder.json" ] ; thenecho "categoryGroupOrder.json not found. Generating..."- echo `node $SCRIPTS_DIR/generateCategoryGroupOrder.js \+ `node $SCRIPTS_DIR/generateCategoryGroupOrder.js \--inputDir "$SRC_DIR/common/config/wv.json/categories/" \--outputDir "$SRC_DIR/common/config/wv.json/" \--mode "$MODE"`@@ -178,16 +175,16 @@ elsedone# Run mergeConfigWithWMTS.js to merge layer metadata from WMTS GC with worldview layer configs into wv.json- if (( $MODE == "verbose" )) ; then+ if (( $MODE = "verbose" )) ; thenecho "Run mergeConfigWithWMTS.js to merge layer metadata from WMTS GC with worldview layer configs into wv.json"fi- echo `node $SCRIPTS_DIR/mergeConfigWithWMTS.js \+ `node $SCRIPTS_DIR/mergeConfigWithWMTS.js \--inputDir "$BUILD_DIR/_wmts" \--outputFile "$DEST_DIR/config/wv.json" \--mode "$MODE"`# Copy brand files from build to dest- if (( $MODE == "verbose" )) ; then+ if (( $MODE = "verbose" )) ; thenecho "Copy brand files from build to dest"ficp -r "$BUILD_DIR/brand" "$DEST_DIR"@@ -195,19 +192,19 @@ else# Validate the options build- if (( $MODE == "verbose" )) ; then+ if (( $MODE = "verbose" )) ; thenecho "Validate the options build"fi- echo `node $SCRIPTS_DIR/validateOptions.js \+ `node $SCRIPTS_DIR/validateOptions.js \--optionsFile "$BUILD_DIR/config.json" \--configDir "$DEST_DIR/config" \--mode "$MODE"`# Fetch preview images from WV Snapshots for any layers which they are missing- if (( $MODE == "verbose" )) ; then+ if (( $MODE = "verbose" )) ; thenecho "Fetch preview images from WV Snapshots for any layers which they are missing"fi- echo `node $SCRIPTS_DIR/fetchPreviewSnapshots.js \+ `node $SCRIPTS_DIR/fetchPreviewSnapshots.js \--wvJsonFile "$DEST_DIR/config/wv.json" \--overridesFile "$OPT_DIR/common/previewLayerOverrides.json" \--featuresFile "$BUILD_DIR/features.json" \