Skip to content

Interactive interface for browsing global, full-resolution satellite imagery

WV-3452 build performance improvements (#6191)

Patrick Moulden391b1d42561f
diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml
index b98f44794..cfdcce140 100644
--- a/.github/workflows/ci-cd.yml
+++ b/.github/workflows/ci-cd.yml
@@ -10,7 +10,7 @@ jobs:
fail-fast: false
matrix:
os: [macOS-latest, ubuntu-latest, windows-latest]
- node: [20]
+ node: [22]
steps:
- name: Checkout the code
diff --git a/package.json b/package.json
index 021946963..4b9212b90 100644
--- a/package.json
+++ b/package.json
@@ -28,10 +28,12 @@
"scripts": {
"analyze": "cross-env ANALYZE_MODE=true NODE_ENV=production webpack",
"build": "run-s clean getcapabilities build:config build:dev",
- "build:verbose": "run-s clean getcapabilities:verbose build:config build:dev",
+ "build:verbose": "run-s clean getcapabilities:verbose build:config:verbose build:dev",
"build:ci": "run-s getcapabilities build:config build:prod",
"build:options": "bash tasks/buildOptions.sh",
+ "build:options:verbose": "bash tasks/buildOptions.sh -v",
"build:config": "run-s build:options && node ./tasks/util/config.js",
+ "build:config:verbose": "run-s build:options:verbose && node ./tasks/util/config.js",
"build:dev": "cross-env NODE_ENV=development webpack",
"build:prod": "cross-env NODE_ENV=production webpack",
"clean": "node ./tasks/util/clean.js",
diff --git a/tasks/build-options/extractConfigFromWMTS.js b/tasks/build-options/extractConfigFromWMTS.js
index 528dc86a9..3e0d0bc32 100644
--- a/tasks/build-options/extractConfigFromWMTS.js
+++ b/tasks/build-options/extractConfigFromWMTS.js
@@ -28,6 +28,12 @@ const options = yargs(hideBin(process.argv))
type: 'string',
description: 'wmts output directory'
})
+ .option('mode', {
+ demandOption: true,
+ alias: 'm',
+ type: 'string',
+ description: 'mode'
+ })
.epilog('Extracts configuration information from a WMTS GetCapabilities file, converts the XML to JSON')
const { argv } = options
@@ -39,6 +45,7 @@ const configFile = argv.config
const config = JSON.parse(fs.readFileSync(configFile, 'utf-8'))
const inputDir = argv.inputDir
const outputDir = argv.outputDir
+const mode = argv.mode
if (!Object.prototype.hasOwnProperty.call(config, 'wv-options-wmts')) {
throw new Error(`${prog}: Error: "wv-options-wmts" not in config file`)
@@ -56,12 +63,6 @@ let totalLayerCount = 0
let totalWarningCount = 0
let totalErrorCount = 0
-const wv = {
- layers: {},
- sources: {}
-}
-let wvMatrixSets = {}
-
class SkipException extends Error {
constructor (message) {
super(message)
@@ -76,15 +77,18 @@ class SkipException extends Error {
* @throws {SkipException}
*/
async function main () {
- for (const entry of entries) {
- wv.layers = {}
- wv.sources = {}
- wvMatrixSets = {}
- const { errorCount, warningCount, layerCount } = await processEntry(entry)
- console.warn(`${prog}: ${errorCount} errors, ${warningCount} warnings, ${layerCount} layers for ${entry.source}`)
-
- const outputFile = path.join(outputDir, entry.to)
- fs.writeFile(outputFile, JSON.stringify(wv, null, 2), 'utf-8', err => {
+ console.warn(`${prog}: Processing ${entries.length} entries...`)
+
+ const promises = entries.map((entry) => processEntry(entry))
+ const results = await Promise.allSettled(promises)
+
+ for (const { status, value } of results) {
+ if (status === 'rejected' || !value) continue
+ const { errorCount, warningCount, layerCount, wv, to, source } = value
+ console.warn(`${prog}: ${errorCount} errors, ${warningCount} warnings, ${layerCount} layers for ${source}`)
+
+ const outputFile = path.join(outputDir, to)
+ await fs.writeFile(outputFile, JSON.stringify(wv, null, 2), 'utf-8', err => {
if (err) {
console.error(err)
}
@@ -95,7 +99,7 @@ async function main () {
totalLayerCount += layerCount
}
- console.warn(`${prog}:${totalErrorCount} errors, ${totalWarningCount} warnings, ${totalLayerCount} layers`)
+ console.warn(`${prog}: ${totalErrorCount} errors, ${totalWarningCount} warnings, ${totalLayerCount} layers`)
if (totalErrorCount > 0) {
throw new Error(`${prog}: Error: ${totalErrorCount} errors occured`)
@@ -103,14 +107,16 @@ async function main () {
}
async function processEntry (entry) {
+ console.warn(`${prog}: Processing ${entry.source} from ${entry.from} --> ${entry.to}...`)
+ const wv = {}
+ wv.layers = {}
+ wv.sources = {}
+ let wvMatrixSets = {}
+
let layerCount = 0
let warningCount = 0
let errorCount = 0
- wv.sources[entry.source] = {
- matrixSets: wvMatrixSets
- }
-
const inputFile = path.join(inputDir, entry.from)
const gcId = path.basename(inputFile)
let gc
@@ -137,36 +143,39 @@ async function processEntry (entry) {
return { errorCount, warningCount, layerCount }
}
- for (const gcLayer of gcContents.Layer) {
- try {
- layerCount += 1
- await processLayer(gcLayer, wvLayers, entry)
- } catch (error) {
- if (error instanceof SkipException) {
- warningCount += 1
- console.warn(`${prog}: WARNING: [${gcId}] Skipping\n`)
- } else {
- errorCount += 1
- console.error(error.stack)
- const ident = gcLayer['ows:Identifier']._text
- console.error(`${prog}: ERROR: [${gcId}:${ident}] ${error}\n`)
- }
+ console.warn(`${prog}: Processing ${gcContents.Layer.length} layers for ${entry.source}...`)
+ const promises = gcContents.Layer.map((gcLayer) => processLayer(gcLayer, wvLayers, entry))
+ const results = await Promise.allSettled(promises)
+ console.warn(`${prog}: Processed ${results.length} layers for ${entry.source}...`)
+
+ layerCount += results.length
+
+ for (const result of results) {
+ if (result.status === 'fulfilled') continue
+ if (result.reason instanceof SkipException) {
+ warningCount += 1
+ console.warn(`${prog}: WARNING: Skipping\n`)
+ } else {
+ errorCount += 1
+ console.error(result.reason.stack)
+ console.error(`${prog}: ERROR: ${result.reason}\n`)
}
}
- if (gcContents.TileMatrixSet === 'Object') {
- processMatrixSet(gcContents.TileMatrixSet, entry)
- } else {
- gcContents.TileMatrixSet.forEach(gcMatrixSet => {
- processMatrixSet(gcMatrixSet, entry)
- })
+ const gcTileMatrixSet = gcContents.TileMatrixSet
+
+ wvMatrixSets = Array.isArray(gcTileMatrixSet) ? gcTileMatrixSet.map(gcMatrixSet => processMatrixSet(gcMatrixSet, entry)) : [processMatrixSet(gcTileMatrixSet, entry)]
+
+ wv.sources[entry.source] = {
+ matrixSets: Object.fromEntries(new Map(wvMatrixSets))
}
- return { errorCount, warningCount, layerCount }
+ return { errorCount, warningCount, layerCount, wv, to: entry.to, source: entry.source }
}
async function processLayer (gcLayer, wvLayers, entry) {
const ident = gcLayer['ows:Identifier']._text
+ if (mode === 'verbose') console.warn(`${prog}: Processing layer ${ident}...`)
if (skip.includes(ident)) {
console.log(`${ident}: skipping`)
throw new SkipException(ident)
@@ -298,7 +307,7 @@ function processMatrixSet (gcMatrixSet, entry) {
})
}
- wvMatrixSets[ident] = {
+ const value = {
id: ident,
maxResolution,
resolutions,
@@ -308,6 +317,8 @@ function processMatrixSet (gcMatrixSet, entry) {
],
tileMatrices: formattedTileMatrixArr
}
+
+ return [ident, value]
}
main().catch((err) => {
diff --git a/tasks/build-options/fetchPreviewSnapshots.js b/tasks/build-options/fetchPreviewSnapshots.js
index 206f70636..89e78b9b7 100644
--- a/tasks/build-options/fetchPreviewSnapshots.js
+++ b/tasks/build-options/fetchPreviewSnapshots.js
@@ -101,9 +101,12 @@ async function main () {
throw new Error(`${prog}: Layer preview fetching disabled. Exiting.`)
}
- for (const layer of Object.values(layers)) {
- await getSnapshots(layer)
- }
+ const layerValues = Object.values(layers)
+
+ console.warn(`${prog}: Fetching snapshots for up to ${layerValues.length} layers...`)
+ const promises = layerValues.map((layer) => getSnapshots(layer))
+
+ await Promise.allSettled(promises)
if (badSnapshots.length > 0) {
console.warn(`\n${prog}: WARNING: ${badSnapshots.length} snapshots returned no content. See below for details: `)
@@ -191,29 +194,32 @@ async function getBestDate (projection, period, dateRanges) {
return endDate
}
-async function getTimeParam (projection, layerId, layer, params) {
+async function getTimeParam (projection, layerId, layer) {
// Only include TIME param for temporal layers
+ const out = {}
const dateRanges = layer.dateRanges
const startDate = layer.startDate
const period = layer.period
if (dateRanges) {
- params.TIME = await getBestDate(projection, period, dateRanges)
+ out.TIME = await getBestDate(projection, period, dateRanges)
} else if (startDate) {
- params.TIME = startDate
+ out.TIME = startDate
}
// Use any configured override dates
if (overrideDatesDict[layerId]) {
- params.TIME = overrideDatesDict[layerId]
+ out.TIME = overrideDatesDict[layerId]
}
+
+ return out
}
async function getSnapshots (layer) {
for (const projection of Object.keys(layer.projections)) {
const projDict = layer.projections[projection]
const referenceLayer = referenceLayers[projection]
- const params = { ...paramDict.base, ...paramDict[projection] }
+ let params = { ...paramDict.base, ...paramDict[projection] }
// Sometimes a layer id is provided per projection (e.g. Land Mask layers)
// We need to use this layer id to request the layer from WVS/GIBS
@@ -234,7 +240,8 @@ async function getSnapshots (layer) {
.catch(() => false)
if (fileExists) continue
- await getTimeParam(projection, wvLayerId, layer, params)
+ const timeParam = await getTimeParam(projection, wvLayerId, layer)
+ params = Object.assign(params, timeParam)
if (gibsLayerId !== referenceLayer && !standaloneLayers.includes(gibsLayerId)) {
params.LAYERS = `${referenceLayer},${gibsLayerId}`
diff --git a/tasks/build-options/getCapabilities.js b/tasks/build-options/getCapabilities.js
index 8b6ccbbd7..232af7b92 100644
--- a/tasks/build-options/getCapabilities.js
+++ b/tasks/build-options/getCapabilities.js
@@ -71,28 +71,24 @@ async function getCapabilities () {
// Download each GC xml using the "from" attribute and put it in the "to" location
if (Object.prototype.hasOwnProperty.call(config, 'wv-options-fetch')) {
const fetchValues = config['wv-options-fetch']
- for (const value of fetchValues) {
- const inputFile = value.from
- const outputFile = `${outputDir}/${value.to}`
-
- if (argv.mode === 'verbose') console.warn(`Fetching config for ${inputFile} to ${outputFile}...`)
- await fetchConfigs(inputFile, outputFile)
- if (argv.mode === 'verbose') console.warn(`Processing capabilities for ${outputFile}...`)
- await processGetCapabilities(outputFile)
- }
+
+ const files = fetchValues.map(({ from, to }) => [from, `${outputDir}/${to}`])
+ await Promise.allSettled(files.map(([from, to]) => fetchConfigs(from, to)))
+ await Promise.allSettled(files.map(([, to]) => processGetCapabilities(to)))
}
}
// convert to superagent and use promises
async function fetchConfigs (inputFile, outputFile) {
const writer = await fs.createWriteStream(outputFile)
+ if (argv.mode === 'verbose') console.warn(`${prog}: Fetching ${inputFile}...`)
return axios({
method: 'get',
url: inputFile,
responseType: 'stream',
timeout: 100000
}).then(async (response) => {
- if (argv.mode === 'verbose') console.warn(`Writing ${outputFile}...`)
+ if (argv.mode === 'verbose') console.warn(`${prog}: Writing ${outputFile}...`)
await response.data.pipe(writer)
return finished(writer)
})
@@ -129,7 +125,7 @@ async function processVectorData (layer) {
async function processLayer (layer) {
const ident = layer['ows:Identifier']._text
- if (argv.mode === 'verbose') console.warn(`Processing layer ${ident}:`)
+ if (argv.mode === 'verbose') console.warn(`Processing layer ${ident}...`)
if (layer['ows:Metadata']) {
if (config.skipPalettes) {
console.warn(`${prog}: WARN: Skipping palette for ${ident} \n`)
@@ -190,9 +186,8 @@ async function gatherProcess (type, typeStr, dir, ext) {
if (!fs.existsSync(dir)) {
fs.mkdirSync(dir)
}
- Object.values(type).forEach(async (link) => {
- await processMetadata(link, dir, ext)
- })
+ const promises = Object.values(type).map((link) => processMetadata(link, dir, ext))
+ await Promise.allSettled(promises)
}
main().catch((err) => {
diff --git a/tasks/build-options/getVisMetadata.js b/tasks/build-options/getVisMetadata.js
index 865418162..6e64f50d5 100644
--- a/tasks/build-options/getVisMetadata.js
+++ b/tasks/build-options/getVisMetadata.js
@@ -59,7 +59,6 @@ const outputFile = argv.layerMetadata
const metadataConfig = features.features.vismetadata
const url = metadataConfig.url
const daacMap = metadataConfig.daacMap || {}
-const layerMetadata = {}
// These are alias or otherwise layers that don't exist in GIBS
const skipLayers = [
@@ -122,11 +121,18 @@ async function main (url) {
layerOrder = layerOrder.filter(x => !skipLayers.includes(x))
console.warn(`${prog}: Fetching ${layerOrder.length} layer-metadata files`)
- for (const layerId of layerOrder) {
- if (!layerId.includes('_STD') && !layerId.includes('_NRT')) {
- await getMetadata(layerId, url)
- }
- }
+ const promises = layerOrder.map((layerId) => {
+ if (!layerId.includes('_STD') && !layerId.includes('_NRT')) return getMetadata(layerId, url)
+ return Promise.reject(new Error(`Skipped layer: ${layerId}`))
+ })
+ const results = await Promise.allSettled(promises)
+ const { fulfilled = [], rejected = [] } = Object.groupBy(results, (item) => item.status)
+
+ rejected.forEach(({ reason }) => {
+ console.error(`${prog}: ERROR: ${reason}\n`)
+ })
+ const metadataMap = new Map(fulfilled.map(({ value }) => value))
+ const layerMetadata = Object.fromEntries(metadataMap)
const layers = Object.keys(layerMetadata).sort().reduce(
(obj, key) => {
@@ -136,11 +142,11 @@ async function main (url) {
{}
)
- await fs.writeFileSync(outputFile, JSON.stringify({ layers }))
- console.warn(`${prog}: Combined all layer-metadata files into ${path.parse(outputFile).base}`)
+ fs.writeFileSync(outputFile, JSON.stringify({ layers }))
+ console.warn(`${prog}: Combined all ${Object.keys(layerMetadata).length} layer-metadata files into ${path.parse(outputFile).base}`)
}
-async function getDAAC (metadata) {
+function getDAAC (metadata) {
if (!Array.isArray(metadata.conceptIds) || !metadata.conceptIds.length) {
return metadata
}
@@ -150,7 +156,7 @@ async function getDAAC (metadata) {
if (!dataCenter) {
continue
}
- await delete collection.dataCenter
+ delete collection.dataCenter
if (!metadata.dataCenter) {
metadata.dataCenter = [dataCenter]
} else if (!metadata.dataCenter.includes(dataCenter)) {
@@ -162,31 +168,34 @@ async function getDAAC (metadata) {
async function getMetadata (layerId, baseUrl, count) {
if (count) console.warn(`retry #${count} for ${layerId}`)
- return axios({
- method: 'get',
- url: `${baseUrl}${layerId}.json`,
- responseType: 'json',
- timeout: 10000
- }).then(async (response) => {
+ try {
+ const response = await axios({
+ method: 'get',
+ url: `${baseUrl}${layerId}.json`,
+ responseType: 'json',
+ timeout: 10000
+ })
const metadata = response.data
- layerMetadata[layerId] = await getDAAC(metadata)
- let metadataKeys = Object.keys(layerMetadata[layerId])
+ const daac = getDAAC(metadata)
+ let metadataKeys = Object.keys(daac)
metadataKeys = metadataKeys.filter(x => !useKeys.includes(x))
for (const key of metadataKeys) {
- delete layerMetadata[layerId][key]
+ delete daac[key]
}
- }).catch((error) => {
- handleException(error, layerId, url, count)
- })
+ return [layerId, daac]
+ } catch (error) {
+ return await handleException(error, layerId, url, count)
+ }
}
async function handleException (error, layerId, url, count) {
if (!count) count = 0
count++
if (count <= 5) {
- await getMetadata(layerId, url, count)
+ return getMetadata(layerId, url, count)
} else {
console.warn(`\n ${prog} WARN: Unable to fetch ${layerId} ${error}`)
+ return Promise.reject(new Error(`Failed to fetch layer ${layerId}: ${error}`))
}
}
diff --git a/tasks/build-options/mergeConfig.js b/tasks/build-options/mergeConfig.js
index 8bb66ddfd..166969af0 100644
--- a/tasks/build-options/mergeConfig.js
+++ b/tasks/build-options/mergeConfig.js
@@ -41,20 +41,22 @@ async function main () {
// inputDir refers to build\options-build\config\wv.json\*.*
async function mergeFiles (inputDir) {
const files = fs.readdirSync(inputDir)
- for (const file of files) {
+ const promises = files.map((file) => {
try {
if (file.endsWith('.json')) {
fileCount += 1
const data = JSON.parse(fs.readFileSync(path.join(inputDir, file), 'utf-8'))
- await dictMerge(conf, data)
+ return dictMerge(conf, data)
} else if (fs.existsSync(path.join(inputDir, file)) && fs.lstatSync(path.join(inputDir, file)).isDirectory()) {
const subDir = path.join(inputDir, file)
- await mergeFiles(subDir)
+ return mergeFiles(subDir)
}
+ return Promise.resolve()
} catch (error) {
throw new Error(`ERROR: ${path.join(inputDir, file)}: ${error.message}`)
}
- }
+ })
+ return Promise.allSettled(promises)
}
main().catch((err) => {
diff --git a/tasks/build-options/processColormap.js b/tasks/build-options/processColormap.js
index 04eeef4f8..d6b875937 100755
--- a/tasks/build-options/processColormap.js
+++ b/tasks/build-options/processColormap.js
@@ -75,17 +75,17 @@ const walk = (dir) => {
async function main () {
const files = await walk(inputDir)
- for (const file of files) {
- try {
- const { id, xml } = await readFileAsync(file)
- await processFile(id, xml)
- fileCount += 1
- } catch (error) {
- console.error(error)
- console.error(`${prog}: ERROR: ${error}`)
- errorCount += 1
- }
- }
+ const promises = files.map(async (file) => {
+ const { id, xml } = await readFileAsync(file)
+ return processFile(id, xml)
+ })
+ const results = await Promise.allSettled(promises)
+ const { fulfilled = [], rejected = [] } = Object.groupBy(results, (item) => item.status)
+
+ rejected.forEach(({ reason }) => console.error(`${prog}: ERROR: ${reason}`))
+
+ fileCount += fulfilled.length
+ errorCount += rejected.length
try {
const wmtsFiles = fs.readdirSync(layersDir)
diff --git a/tasks/build-options/processVectorData.js b/tasks/build-options/processVectorData.js
index d57c07353..292d1ca6e 100644
--- a/tasks/build-options/processVectorData.js
+++ b/tasks/build-options/processVectorData.js
@@ -38,15 +38,14 @@ async function main () {
let fileCount = 0
let errorCount = 0
- for (const file of fs.readdirSync(inputDir)) {
- try {
- fileCount += 1
- await copyFileAsync(file)
- } catch (error) {
- errorCount += 1
- console.error(`${prog}: ERROR: [${file}] ${error}\n.`)
- }
- }
+ const promises = fs.readdirSync(inputDir).map((file) => copyFileAsync(file))
+ const results = await Promise.allSettled(promises)
+ const { fulfilled = [], rejected = [] } = Object.groupBy(results, (item) => item.status)
+
+ rejected.forEach(({ reason }) => console.error(`${prog}: ERROR: ${reason}`))
+
+ fileCount += fulfilled.length
+ errorCount += rejected.length
console.error(`${prog}: ${errorCount} error(s), ${fileCount} file(s)`)
diff --git a/tasks/build-options/processVectorStyles.js b/tasks/build-options/processVectorStyles.js
index d10c228e3..c0d6b949b 100644
--- a/tasks/build-options/processVectorStyles.js
+++ b/tasks/build-options/processVectorStyles.js
@@ -47,15 +47,13 @@ async function main () {
const files = [...fs.readdirSync(inputDir), ...fs.readdirSync(wvStylesDir)]
- for (const file of files) {
- try {
- fileCount += 1
- await copyFileAsync(file)
- } catch (error) {
- errorCount += 1
- console.error(`${prog}: ERROR: [${file}] ${error}\n.`)
- }
- }
+ const results = await Promise.allSettled(files.map((file) => copyFileAsync(file)))
+ const { fulfilled = [], rejected = [] } = Object.groupBy(results, (item) => item.status)
+
+ rejected.forEach(({ reason }) => console.error(`${prog}: ERROR: ${reason}\n`))
+
+ fileCount += fulfilled.length
+ errorCount += rejected.length
console.error(`${prog}: ${errorCount} error(s), ${fileCount} file(s)`)
diff --git a/tasks/buildOptions.sh b/tasks/buildOptions.sh
index 6e6321a42..675116d49 100755
--- a/tasks/buildOptions.sh
+++ b/tasks/buildOptions.sh
@@ -58,7 +58,7 @@ mkdir -p "$BUILD_DIR/colormaps"
# If $FETCH_GC is set, make various API requests
if [ "$FETCH_GC" ] ; then
# Fetch GC files and create colormaps, vectordata and vectorstyle files
- if (( $MODE = "verbose" )) ; then
+ if [ "$MODE" == "verbose" ] ; then
echo "Fetch GC files and create colormaps, vectordata and vectorstyle files"
fi
rm -rf "$OPT_DIR/$OPT_SUBDIR/gc/*"
@@ -69,7 +69,7 @@ if [ "$FETCH_GC" ] ; then
--mode "$MODE"`
# Get metadata for files in layerOrder.json and combine this data into 1 file
- if (( $MODE = "verbose" )) ; then
+ if [ "$MODE" == "verbose" ] ; then
echo "Get metadata for files in layerOrder.json and combine this data into 1 file"
fi
rm -rf "$OPT_DIR/$OPT_SUBDIR/layer-metadata"
@@ -81,7 +81,7 @@ if [ "$FETCH_GC" ] ; then
--mode "$MODE"`
else
# Validate layers in wv.json with a JSON schema
- if (( $MODE = "verbose" )) ; then
+ if [ "$MODE" == "verbose" ] ; then
echo "Validate layers in wv.json with a JSON schema"
fi
`node $SCRIPTS_DIR/validateConfigs.js \
@@ -90,12 +90,15 @@ else
--mode "$MODE"`
if [ -e "$BUILD_DIR/features.json" ] ; then
- cp "$BUILD_DIR/features.json" "$BUILD_DIR/config/wv.json/_features.json"
+ if [ "$MODE" == "verbose" ] ; then
+ echo "Copying features.json from $BUILD_DIR/features.json --> $BUILD_DIR/config/wv.json/_features.json"
+ fi
+ cp "$BUILD_DIR/features.json" "$BUILD_DIR/config/wv.json/_features.json"
fi
# Run extractConfigFromWMTS.js script with config.json
if [ -e "$BUILD_DIR/config.json" ] ; then
- if (( $MODE = "verbose" )) ; then
+ if [ "$MODE" == "verbose" ] ; then
echo "Run extractConfigFromWMTS.js script with config.json"
fi
`node $SCRIPTS_DIR/extractConfigFromWMTS.js \
@@ -107,7 +110,7 @@ else
# Run processVectorStyles.js and move vectorstyles where we want them
if [ -e "$BUILD_DIR/gc/vectorstyles" ] ; then
- if (( $MODE = "verbose" )) ; then
+ if [ "$MODE" == "verbose" ] ; then
echo "Run processVectorStyles.js and move vectorstyles where we want them"
fi
mkdir -p "$BUILD_DIR/config/wv.json/vectorstyles"
@@ -120,7 +123,7 @@ else
# Run processVectorData.js and move vectordata where we want them
if [ -e "$BUILD_DIR/gc/vectordata" ] ; then
- if (( $MODE = "verbose" )) ; then
+ if [ "$MODE" == "verbose" ] ; then
echo "Run processVectorData.js and move vectordata where we want them"
fi
mkdir -p "$BUILD_DIR/config/wv.json/vectordata"
@@ -132,7 +135,7 @@ else
# Run processColormap.js and move colormaps where we want them
if [ -e "$BUILD_DIR/colormaps" ] ; then
- if (( $MODE = "verbose" )) ; then
+ if [ "$MODE" == "verbose" ] ; then
echo "Run processColormap.js and move colormaps where we want them"
fi
mkdir -p "$BUILD_DIR"/config/palettes
@@ -177,7 +180,7 @@ else
done
# Run mergeConfigWithWMTS.js to merge layer metadata from WMTS GC with worldview layer configs into wv.json
- if (( $MODE = "verbose" )) ; then
+ if [ "$MODE" == "verbose" ] ; then
echo "Run mergeConfigWithWMTS.js to merge layer metadata from WMTS GC with worldview layer configs into wv.json"
fi
`node $SCRIPTS_DIR/mergeConfigWithWMTS.js \
@@ -186,7 +189,7 @@ else
--mode "$MODE"`
# Copy brand files from build to dest
- if (( $MODE = "verbose" )) ; then
+ if [ "$MODE" == "verbose" ] ; then
echo "Copy brand files from build to dest"
fi
cp -r "$BUILD_DIR/brand" "$DEST_DIR"
@@ -194,7 +197,7 @@ else
# Validate the options build
- if (( $MODE = "verbose" )) ; then
+ if [ "$MODE" == "verbose" ] ; then
echo "Validate the options build"
fi
`node $SCRIPTS_DIR/validateOptions.js \
@@ -203,7 +206,7 @@ else
--mode "$MODE"`
# Fetch preview images from WV Snapshots for any layers which they are missing
- if (( $MODE = "verbose" )) ; then
+ if [ "$MODE" == "verbose" ] ; then
echo "Fetch preview images from WV Snapshots for any layers which they are missing"
fi
`node $SCRIPTS_DIR/fetchPreviewSnapshots.js \
diff --git a/tasks/link-check/natural-event-url-extract.js b/tasks/link-check/natural-event-url-extract.js
index b2b5b8818..a9ebaf8d8 100644
--- a/tasks/link-check/natural-event-url-extract.js
+++ b/tasks/link-check/natural-event-url-extract.js
@@ -1,8 +1,7 @@
/* eslint-disable no-restricted-syntax */
const fetch = require('node-fetch')
-
-// delay helper to prevent too many requests at once
-const sleeper = (ms) => (x) => new Promise((resolve) => setTimeout(() => resolve(x), ms))
+const path = require('path')
+const prog = path.basename(__filename)
// helper function to find target key in nested object
const findProp = async (obj, keys, out) => {
@@ -35,27 +34,26 @@ const scrapeLinks = async (htmlLinks) => {
const trackDoubles = {}
const addedUrls = []
- for (let i = 0; i < htmlLinks.length; i += 1) {
- const htmlLink = htmlLinks[i]
- // eslint-disable-next-line no-await-in-loop
- await fetch(htmlLink)
- .then(async (res) => {
- const status = await res.json()
- const urls = await findProp(status, ['url', 'link', 'source'])
- for (let j = 0; j < urls.length; j += 1) {
- const url = urls[j]
- const linkRel = Object.keys(url)[0]
- const linkHref = Object.values(url)[0]
+ try {
+ console.warn(`${prog}: Scraping ${htmlLinks.length} links from EONET...`)
+ const results = await Promise.allSettled(htmlLinks.map((htmlLink, i) => fetch(htmlLink)))
+ const statuses = await Promise.allSettled(results.map(({ status, value }) => status === 'fulfilled' && value.json()))
+ const urls = await Promise.allSettled(statuses.map(({ status, value }) => status === 'fulfilled' && findProp(value, ['url', 'link', 'source'])))
+ urls.forEach(({ status, value }) => {
+ if (status === 'fulfilled' && Array.isArray(value)) {
+ value.forEach(obj => {
+ const linkRel = Object.keys(obj)[0]
+ const linkHref = Object.values(obj)[0]
if (trackDoubles[linkRel] !== linkHref) {
addedUrls.push({ [linkRel]: linkHref })
trackDoubles[linkRel] = linkHref
}
- }
- }).then(sleeper(500))
- .catch((err) => {
- console.log(htmlLink, err)
- })
+ })
+ }
+ })
+ } catch (err) {
+ console.log(`${prog}: ERROR: ${err}`)
}
return addedUrls
}
diff --git a/tasks/link-check/url-check.js b/tasks/link-check/url-check.js
index f446fe4d1..c15bd3c3c 100644
--- a/tasks/link-check/url-check.js
+++ b/tasks/link-check/url-check.js
@@ -1,36 +1,33 @@
const fetch = require('node-fetch')
+const path = require('path')
+const prog = path.basename(__filename)
// URL status check function makes GET requests to provided list of URLs to get status codes / errors
-// Delay helper to prevent too many requests at once
-const sleeper = (ms) => (x) => new Promise((resolve) => setTimeout(() => resolve(x), ms))
-
// Checks status code of provided links and return object organized by errors / statuscodes
const requestCheck = async (urls) => {
const parsedUrls = {
ERROR: [],
STATUSCODE: {}
}
- for (let i = 0; i < urls.length; i += 1) {
- const linkName = Object.keys(urls[i])[0]
- const url = Object.values(urls[i])[0]
- // Skip for mailto email links
- if (url[0] === 'h') {
- // eslint-disable-next-line no-await-in-loop
- await fetch(url, { timeout: 10000 })
- .then(async (res) => {
- const statusCode = await res.status
- if (!parsedUrls.STATUSCODE[statusCode]) {
- parsedUrls.STATUSCODE[statusCode] = []
- }
- parsedUrls.STATUSCODE[statusCode].push({ [linkName]: url })
- })
- .then(sleeper(500))
- .catch((err) => { // eslint-disable-line n/handle-callback-err
- parsedUrls.ERROR.push({ [linkName]: url })
- })
- }
- }
+ console.warn(`${prog}: Checking ${urls.length} URLs for errors/status codes...`)
+ const promises = urls.map((link) => {
+ const linkName = Object.keys(link)[0]
+ const url = Object.values(link)[0]
+ if (url[0] !== 'h') return Promise.resolve() // Skip for mailto email links
+ return fetch(url, { timeout: 10000 })
+ .then(async (res) => {
+ const statusCode = await res.status
+ if (!parsedUrls.STATUSCODE[statusCode]) {
+ parsedUrls.STATUSCODE[statusCode] = []
+ }
+ parsedUrls.STATUSCODE[statusCode].push({ [linkName]: url })
+ })
+ .catch((err) => { // eslint-disable-line n/handle-callback-err
+ parsedUrls.ERROR.push({ [linkName]: url })
+ })
+ })
+ await Promise.allSettled(promises)
return parsedUrls
}