Interactive interface for browsing global, full-resolution satellite imagery
WV-3452 build performance improvements (#6191)
Patrick Moulden391b1d42561f
diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.ymlindex b98f44794..cfdcce140 100644--- a/.github/workflows/ci-cd.yml+++ b/.github/workflows/ci-cd.yml@@ -10,7 +10,7 @@ jobs:fail-fast: falsematrix:os: [macOS-latest, ubuntu-latest, windows-latest]- node: [20]+ node: [22]steps:- name: Checkout the codediff --git a/package.json b/package.jsonindex 021946963..4b9212b90 100644--- a/package.json+++ b/package.json@@ -28,10 +28,12 @@"scripts": {"analyze": "cross-env ANALYZE_MODE=true NODE_ENV=production webpack","build": "run-s clean getcapabilities build:config build:dev",- "build:verbose": "run-s clean getcapabilities:verbose build:config build:dev",+ "build:verbose": "run-s clean getcapabilities:verbose build:config:verbose build:dev","build:ci": "run-s getcapabilities build:config build:prod","build:options": "bash tasks/buildOptions.sh",+ "build:options:verbose": "bash tasks/buildOptions.sh -v","build:config": "run-s build:options && node ./tasks/util/config.js",+ "build:config:verbose": "run-s build:options:verbose && node ./tasks/util/config.js","build:dev": "cross-env NODE_ENV=development webpack","build:prod": "cross-env NODE_ENV=production webpack","clean": "node ./tasks/util/clean.js",diff --git a/tasks/build-options/extractConfigFromWMTS.js b/tasks/build-options/extractConfigFromWMTS.jsindex 528dc86a9..3e0d0bc32 100644--- a/tasks/build-options/extractConfigFromWMTS.js+++ b/tasks/build-options/extractConfigFromWMTS.js@@ -28,6 +28,12 @@ const options = yargs(hideBin(process.argv))type: 'string',description: 'wmts output directory'})+ .option('mode', {+ demandOption: true,+ alias: 'm',+ type: 'string',+ description: 'mode'+ }).epilog('Extracts configuration information from a WMTS GetCapabilities file, converts the XML to JSON')const { argv } = options@@ -39,6 +45,7 @@ const configFile = argv.configconst config = JSON.parse(fs.readFileSync(configFile, 'utf-8'))const inputDir = argv.inputDirconst outputDir = argv.outputDir+const mode = argv.modeif (!Object.prototype.hasOwnProperty.call(config, 'wv-options-wmts')) {throw new Error(`${prog}: Error: "wv-options-wmts" not in config file`)@@ -56,12 +63,6 @@ let totalLayerCount = 0let totalWarningCount = 0let totalErrorCount = 0-const wv = {- layers: {},- sources: {}-}-let wvMatrixSets = {}-class SkipException extends Error {constructor (message) {super(message)@@ -76,15 +77,18 @@ class SkipException extends Error {* @throws {SkipException}*/async function main () {- for (const entry of entries) {- wv.layers = {}- wv.sources = {}- wvMatrixSets = {}- const { errorCount, warningCount, layerCount } = await processEntry(entry)- console.warn(`${prog}: ${errorCount} errors, ${warningCount} warnings, ${layerCount} layers for ${entry.source}`)-- const outputFile = path.join(outputDir, entry.to)- fs.writeFile(outputFile, JSON.stringify(wv, null, 2), 'utf-8', err => {+ console.warn(`${prog}: Processing ${entries.length} entries...`)++ const promises = entries.map((entry) => processEntry(entry))+ const results = await Promise.allSettled(promises)++ for (const { status, value } of results) {+ if (status === 'rejected' || !value) continue+ const { errorCount, warningCount, layerCount, wv, to, source } = value+ console.warn(`${prog}: ${errorCount} errors, ${warningCount} warnings, ${layerCount} layers for ${source}`)++ const outputFile = path.join(outputDir, to)+ await fs.writeFile(outputFile, JSON.stringify(wv, null, 2), 'utf-8', err => {if (err) {console.error(err)}@@ -95,7 +99,7 @@ async function main () {totalLayerCount += layerCount}- console.warn(`${prog}:${totalErrorCount} errors, ${totalWarningCount} warnings, ${totalLayerCount} layers`)+ console.warn(`${prog}: ${totalErrorCount} errors, ${totalWarningCount} warnings, ${totalLayerCount} layers`)if (totalErrorCount > 0) {throw new Error(`${prog}: Error: ${totalErrorCount} errors occured`)@@ -103,14 +107,16 @@ async function main () {}async function processEntry (entry) {+ console.warn(`${prog}: Processing ${entry.source} from ${entry.from} --> ${entry.to}...`)+ const wv = {}+ wv.layers = {}+ wv.sources = {}+ let wvMatrixSets = {}+let layerCount = 0let warningCount = 0let errorCount = 0- wv.sources[entry.source] = {- matrixSets: wvMatrixSets- }-const inputFile = path.join(inputDir, entry.from)const gcId = path.basename(inputFile)let gc@@ -137,36 +143,39 @@ async function processEntry (entry) {return { errorCount, warningCount, layerCount }}- for (const gcLayer of gcContents.Layer) {- try {- layerCount += 1- await processLayer(gcLayer, wvLayers, entry)- } catch (error) {- if (error instanceof SkipException) {- warningCount += 1- console.warn(`${prog}: WARNING: [${gcId}] Skipping\n`)- } else {- errorCount += 1- console.error(error.stack)- const ident = gcLayer['ows:Identifier']._text- console.error(`${prog}: ERROR: [${gcId}:${ident}] ${error}\n`)- }+ console.warn(`${prog}: Processing ${gcContents.Layer.length} layers for ${entry.source}...`)+ const promises = gcContents.Layer.map((gcLayer) => processLayer(gcLayer, wvLayers, entry))+ const results = await Promise.allSettled(promises)+ console.warn(`${prog}: Processed ${results.length} layers for ${entry.source}...`)++ layerCount += results.length++ for (const result of results) {+ if (result.status === 'fulfilled') continue+ if (result.reason instanceof SkipException) {+ warningCount += 1+ console.warn(`${prog}: WARNING: Skipping\n`)+ } else {+ errorCount += 1+ console.error(result.reason.stack)+ console.error(`${prog}: ERROR: ${result.reason}\n`)}}- if (gcContents.TileMatrixSet === 'Object') {- processMatrixSet(gcContents.TileMatrixSet, entry)- } else {- gcContents.TileMatrixSet.forEach(gcMatrixSet => {- processMatrixSet(gcMatrixSet, entry)- })+ const gcTileMatrixSet = gcContents.TileMatrixSet++ wvMatrixSets = Array.isArray(gcTileMatrixSet) ? gcTileMatrixSet.map(gcMatrixSet => processMatrixSet(gcMatrixSet, entry)) : [processMatrixSet(gcTileMatrixSet, entry)]++ wv.sources[entry.source] = {+ matrixSets: Object.fromEntries(new Map(wvMatrixSets))}- return { errorCount, warningCount, layerCount }+ return { errorCount, warningCount, layerCount, wv, to: entry.to, source: entry.source }}async function processLayer (gcLayer, wvLayers, entry) {const ident = gcLayer['ows:Identifier']._text+ if (mode === 'verbose') console.warn(`${prog}: Processing layer ${ident}...`)if (skip.includes(ident)) {console.log(`${ident}: skipping`)throw new SkipException(ident)@@ -298,7 +307,7 @@ function processMatrixSet (gcMatrixSet, entry) {})}- wvMatrixSets[ident] = {+ const value = {id: ident,maxResolution,resolutions,@@ -308,6 +317,8 @@ function processMatrixSet (gcMatrixSet, entry) {],tileMatrices: formattedTileMatrixArr}++ return [ident, value]}main().catch((err) => {diff --git a/tasks/build-options/fetchPreviewSnapshots.js b/tasks/build-options/fetchPreviewSnapshots.jsindex 206f70636..89e78b9b7 100644--- a/tasks/build-options/fetchPreviewSnapshots.js+++ b/tasks/build-options/fetchPreviewSnapshots.js@@ -101,9 +101,12 @@ async function main () {throw new Error(`${prog}: Layer preview fetching disabled. Exiting.`)}- for (const layer of Object.values(layers)) {- await getSnapshots(layer)- }+ const layerValues = Object.values(layers)++ console.warn(`${prog}: Fetching snapshots for up to ${layerValues.length} layers...`)+ const promises = layerValues.map((layer) => getSnapshots(layer))++ await Promise.allSettled(promises)if (badSnapshots.length > 0) {console.warn(`\n${prog}: WARNING: ${badSnapshots.length} snapshots returned no content. See below for details: `)@@ -191,29 +194,32 @@ async function getBestDate (projection, period, dateRanges) {return endDate}-async function getTimeParam (projection, layerId, layer, params) {+async function getTimeParam (projection, layerId, layer) {// Only include TIME param for temporal layers+ const out = {}const dateRanges = layer.dateRangesconst startDate = layer.startDateconst period = layer.periodif (dateRanges) {- params.TIME = await getBestDate(projection, period, dateRanges)+ out.TIME = await getBestDate(projection, period, dateRanges)} else if (startDate) {- params.TIME = startDate+ out.TIME = startDate}// Use any configured override datesif (overrideDatesDict[layerId]) {- params.TIME = overrideDatesDict[layerId]+ out.TIME = overrideDatesDict[layerId]}++ return out}async function getSnapshots (layer) {for (const projection of Object.keys(layer.projections)) {const projDict = layer.projections[projection]const referenceLayer = referenceLayers[projection]- const params = { ...paramDict.base, ...paramDict[projection] }+ let params = { ...paramDict.base, ...paramDict[projection] }// Sometimes a layer id is provided per projection (e.g. Land Mask layers)// We need to use this layer id to request the layer from WVS/GIBS@@ -234,7 +240,8 @@ async function getSnapshots (layer) {.catch(() => false)if (fileExists) continue- await getTimeParam(projection, wvLayerId, layer, params)+ const timeParam = await getTimeParam(projection, wvLayerId, layer)+ params = Object.assign(params, timeParam)if (gibsLayerId !== referenceLayer && !standaloneLayers.includes(gibsLayerId)) {params.LAYERS = `${referenceLayer},${gibsLayerId}`diff --git a/tasks/build-options/getCapabilities.js b/tasks/build-options/getCapabilities.jsindex 8b6ccbbd7..232af7b92 100644--- a/tasks/build-options/getCapabilities.js+++ b/tasks/build-options/getCapabilities.js@@ -71,28 +71,24 @@ async function getCapabilities () {// Download each GC xml using the "from" attribute and put it in the "to" locationif (Object.prototype.hasOwnProperty.call(config, 'wv-options-fetch')) {const fetchValues = config['wv-options-fetch']- for (const value of fetchValues) {- const inputFile = value.from- const outputFile = `${outputDir}/${value.to}`-- if (argv.mode === 'verbose') console.warn(`Fetching config for ${inputFile} to ${outputFile}...`)- await fetchConfigs(inputFile, outputFile)- if (argv.mode === 'verbose') console.warn(`Processing capabilities for ${outputFile}...`)- await processGetCapabilities(outputFile)- }++ const files = fetchValues.map(({ from, to }) => [from, `${outputDir}/${to}`])+ await Promise.allSettled(files.map(([from, to]) => fetchConfigs(from, to)))+ await Promise.allSettled(files.map(([, to]) => processGetCapabilities(to)))}}// convert to superagent and use promisesasync function fetchConfigs (inputFile, outputFile) {const writer = await fs.createWriteStream(outputFile)+ if (argv.mode === 'verbose') console.warn(`${prog}: Fetching ${inputFile}...`)return axios({method: 'get',url: inputFile,responseType: 'stream',timeout: 100000}).then(async (response) => {- if (argv.mode === 'verbose') console.warn(`Writing ${outputFile}...`)+ if (argv.mode === 'verbose') console.warn(`${prog}: Writing ${outputFile}...`)await response.data.pipe(writer)return finished(writer)})@@ -129,7 +125,7 @@ async function processVectorData (layer) {async function processLayer (layer) {const ident = layer['ows:Identifier']._text- if (argv.mode === 'verbose') console.warn(`Processing layer ${ident}:`)+ if (argv.mode === 'verbose') console.warn(`Processing layer ${ident}...`)if (layer['ows:Metadata']) {if (config.skipPalettes) {console.warn(`${prog}: WARN: Skipping palette for ${ident} \n`)@@ -190,9 +186,8 @@ async function gatherProcess (type, typeStr, dir, ext) {if (!fs.existsSync(dir)) {fs.mkdirSync(dir)}- Object.values(type).forEach(async (link) => {- await processMetadata(link, dir, ext)- })+ const promises = Object.values(type).map((link) => processMetadata(link, dir, ext))+ await Promise.allSettled(promises)}main().catch((err) => {diff --git a/tasks/build-options/getVisMetadata.js b/tasks/build-options/getVisMetadata.jsindex 865418162..6e64f50d5 100644--- a/tasks/build-options/getVisMetadata.js+++ b/tasks/build-options/getVisMetadata.js@@ -59,7 +59,6 @@ const outputFile = argv.layerMetadataconst metadataConfig = features.features.vismetadataconst url = metadataConfig.urlconst daacMap = metadataConfig.daacMap || {}-const layerMetadata = {}// These are alias or otherwise layers that don't exist in GIBSconst skipLayers = [@@ -122,11 +121,18 @@ async function main (url) {layerOrder = layerOrder.filter(x => !skipLayers.includes(x))console.warn(`${prog}: Fetching ${layerOrder.length} layer-metadata files`)- for (const layerId of layerOrder) {- if (!layerId.includes('_STD') && !layerId.includes('_NRT')) {- await getMetadata(layerId, url)- }- }+ const promises = layerOrder.map((layerId) => {+ if (!layerId.includes('_STD') && !layerId.includes('_NRT')) return getMetadata(layerId, url)+ return Promise.reject(new Error(`Skipped layer: ${layerId}`))+ })+ const results = await Promise.allSettled(promises)+ const { fulfilled = [], rejected = [] } = Object.groupBy(results, (item) => item.status)++ rejected.forEach(({ reason }) => {+ console.error(`${prog}: ERROR: ${reason}\n`)+ })+ const metadataMap = new Map(fulfilled.map(({ value }) => value))+ const layerMetadata = Object.fromEntries(metadataMap)const layers = Object.keys(layerMetadata).sort().reduce((obj, key) => {@@ -136,11 +142,11 @@ async function main (url) {{})- await fs.writeFileSync(outputFile, JSON.stringify({ layers }))- console.warn(`${prog}: Combined all layer-metadata files into ${path.parse(outputFile).base}`)+ fs.writeFileSync(outputFile, JSON.stringify({ layers }))+ console.warn(`${prog}: Combined all ${Object.keys(layerMetadata).length} layer-metadata files into ${path.parse(outputFile).base}`)}-async function getDAAC (metadata) {+function getDAAC (metadata) {if (!Array.isArray(metadata.conceptIds) || !metadata.conceptIds.length) {return metadata}@@ -150,7 +156,7 @@ async function getDAAC (metadata) {if (!dataCenter) {continue}- await delete collection.dataCenter+ delete collection.dataCenterif (!metadata.dataCenter) {metadata.dataCenter = [dataCenter]} else if (!metadata.dataCenter.includes(dataCenter)) {@@ -162,31 +168,34 @@ async function getDAAC (metadata) {async function getMetadata (layerId, baseUrl, count) {if (count) console.warn(`retry #${count} for ${layerId}`)- return axios({- method: 'get',- url: `${baseUrl}${layerId}.json`,- responseType: 'json',- timeout: 10000- }).then(async (response) => {+ try {+ const response = await axios({+ method: 'get',+ url: `${baseUrl}${layerId}.json`,+ responseType: 'json',+ timeout: 10000+ })const metadata = response.data- layerMetadata[layerId] = await getDAAC(metadata)- let metadataKeys = Object.keys(layerMetadata[layerId])+ const daac = getDAAC(metadata)+ let metadataKeys = Object.keys(daac)metadataKeys = metadataKeys.filter(x => !useKeys.includes(x))for (const key of metadataKeys) {- delete layerMetadata[layerId][key]+ delete daac[key]}- }).catch((error) => {- handleException(error, layerId, url, count)- })+ return [layerId, daac]+ } catch (error) {+ return await handleException(error, layerId, url, count)+ }}async function handleException (error, layerId, url, count) {if (!count) count = 0count++if (count <= 5) {- await getMetadata(layerId, url, count)+ return getMetadata(layerId, url, count)} else {console.warn(`\n ${prog} WARN: Unable to fetch ${layerId} ${error}`)+ return Promise.reject(new Error(`Failed to fetch layer ${layerId}: ${error}`))}}diff --git a/tasks/build-options/mergeConfig.js b/tasks/build-options/mergeConfig.jsindex 8bb66ddfd..166969af0 100644--- a/tasks/build-options/mergeConfig.js+++ b/tasks/build-options/mergeConfig.js@@ -41,20 +41,22 @@ async function main () {// inputDir refers to build\options-build\config\wv.json\*.*async function mergeFiles (inputDir) {const files = fs.readdirSync(inputDir)- for (const file of files) {+ const promises = files.map((file) => {try {if (file.endsWith('.json')) {fileCount += 1const data = JSON.parse(fs.readFileSync(path.join(inputDir, file), 'utf-8'))- await dictMerge(conf, data)+ return dictMerge(conf, data)} else if (fs.existsSync(path.join(inputDir, file)) && fs.lstatSync(path.join(inputDir, file)).isDirectory()) {const subDir = path.join(inputDir, file)- await mergeFiles(subDir)+ return mergeFiles(subDir)}+ return Promise.resolve()} catch (error) {throw new Error(`ERROR: ${path.join(inputDir, file)}: ${error.message}`)}- }+ })+ return Promise.allSettled(promises)}main().catch((err) => {diff --git a/tasks/build-options/processColormap.js b/tasks/build-options/processColormap.jsindex 04eeef4f8..d6b875937 100755--- a/tasks/build-options/processColormap.js+++ b/tasks/build-options/processColormap.js@@ -75,17 +75,17 @@ const walk = (dir) => {async function main () {const files = await walk(inputDir)- for (const file of files) {- try {- const { id, xml } = await readFileAsync(file)- await processFile(id, xml)- fileCount += 1- } catch (error) {- console.error(error)- console.error(`${prog}: ERROR: ${error}`)- errorCount += 1- }- }+ const promises = files.map(async (file) => {+ const { id, xml } = await readFileAsync(file)+ return processFile(id, xml)+ })+ const results = await Promise.allSettled(promises)+ const { fulfilled = [], rejected = [] } = Object.groupBy(results, (item) => item.status)++ rejected.forEach(({ reason }) => console.error(`${prog}: ERROR: ${reason}`))++ fileCount += fulfilled.length+ errorCount += rejected.lengthtry {const wmtsFiles = fs.readdirSync(layersDir)diff --git a/tasks/build-options/processVectorData.js b/tasks/build-options/processVectorData.jsindex d57c07353..292d1ca6e 100644--- a/tasks/build-options/processVectorData.js+++ b/tasks/build-options/processVectorData.js@@ -38,15 +38,14 @@ async function main () {let fileCount = 0let errorCount = 0- for (const file of fs.readdirSync(inputDir)) {- try {- fileCount += 1- await copyFileAsync(file)- } catch (error) {- errorCount += 1- console.error(`${prog}: ERROR: [${file}] ${error}\n.`)- }- }+ const promises = fs.readdirSync(inputDir).map((file) => copyFileAsync(file))+ const results = await Promise.allSettled(promises)+ const { fulfilled = [], rejected = [] } = Object.groupBy(results, (item) => item.status)++ rejected.forEach(({ reason }) => console.error(`${prog}: ERROR: ${reason}`))++ fileCount += fulfilled.length+ errorCount += rejected.lengthconsole.error(`${prog}: ${errorCount} error(s), ${fileCount} file(s)`)diff --git a/tasks/build-options/processVectorStyles.js b/tasks/build-options/processVectorStyles.jsindex d10c228e3..c0d6b949b 100644--- a/tasks/build-options/processVectorStyles.js+++ b/tasks/build-options/processVectorStyles.js@@ -47,15 +47,13 @@ async function main () {const files = [...fs.readdirSync(inputDir), ...fs.readdirSync(wvStylesDir)]- for (const file of files) {- try {- fileCount += 1- await copyFileAsync(file)- } catch (error) {- errorCount += 1- console.error(`${prog}: ERROR: [${file}] ${error}\n.`)- }- }+ const results = await Promise.allSettled(files.map((file) => copyFileAsync(file)))+ const { fulfilled = [], rejected = [] } = Object.groupBy(results, (item) => item.status)++ rejected.forEach(({ reason }) => console.error(`${prog}: ERROR: ${reason}\n`))++ fileCount += fulfilled.length+ errorCount += rejected.lengthconsole.error(`${prog}: ${errorCount} error(s), ${fileCount} file(s)`)diff --git a/tasks/buildOptions.sh b/tasks/buildOptions.shindex 6e6321a42..675116d49 100755--- a/tasks/buildOptions.sh+++ b/tasks/buildOptions.sh@@ -58,7 +58,7 @@ mkdir -p "$BUILD_DIR/colormaps"# If $FETCH_GC is set, make various API requestsif [ "$FETCH_GC" ] ; then# Fetch GC files and create colormaps, vectordata and vectorstyle files- if (( $MODE = "verbose" )) ; then+ if [ "$MODE" == "verbose" ] ; thenecho "Fetch GC files and create colormaps, vectordata and vectorstyle files"firm -rf "$OPT_DIR/$OPT_SUBDIR/gc/*"@@ -69,7 +69,7 @@ if [ "$FETCH_GC" ] ; then--mode "$MODE"`# Get metadata for files in layerOrder.json and combine this data into 1 file- if (( $MODE = "verbose" )) ; then+ if [ "$MODE" == "verbose" ] ; thenecho "Get metadata for files in layerOrder.json and combine this data into 1 file"firm -rf "$OPT_DIR/$OPT_SUBDIR/layer-metadata"@@ -81,7 +81,7 @@ if [ "$FETCH_GC" ] ; then--mode "$MODE"`else# Validate layers in wv.json with a JSON schema- if (( $MODE = "verbose" )) ; then+ if [ "$MODE" == "verbose" ] ; thenecho "Validate layers in wv.json with a JSON schema"fi`node $SCRIPTS_DIR/validateConfigs.js \@@ -90,12 +90,15 @@ else--mode "$MODE"`if [ -e "$BUILD_DIR/features.json" ] ; then- cp "$BUILD_DIR/features.json" "$BUILD_DIR/config/wv.json/_features.json"+ if [ "$MODE" == "verbose" ] ; then+ echo "Copying features.json from $BUILD_DIR/features.json --> $BUILD_DIR/config/wv.json/_features.json"+ fi+ cp "$BUILD_DIR/features.json" "$BUILD_DIR/config/wv.json/_features.json"fi# Run extractConfigFromWMTS.js script with config.jsonif [ -e "$BUILD_DIR/config.json" ] ; then- if (( $MODE = "verbose" )) ; then+ if [ "$MODE" == "verbose" ] ; thenecho "Run extractConfigFromWMTS.js script with config.json"fi`node $SCRIPTS_DIR/extractConfigFromWMTS.js \@@ -107,7 +110,7 @@ else# Run processVectorStyles.js and move vectorstyles where we want themif [ -e "$BUILD_DIR/gc/vectorstyles" ] ; then- if (( $MODE = "verbose" )) ; then+ if [ "$MODE" == "verbose" ] ; thenecho "Run processVectorStyles.js and move vectorstyles where we want them"fimkdir -p "$BUILD_DIR/config/wv.json/vectorstyles"@@ -120,7 +123,7 @@ else# Run processVectorData.js and move vectordata where we want themif [ -e "$BUILD_DIR/gc/vectordata" ] ; then- if (( $MODE = "verbose" )) ; then+ if [ "$MODE" == "verbose" ] ; thenecho "Run processVectorData.js and move vectordata where we want them"fimkdir -p "$BUILD_DIR/config/wv.json/vectordata"@@ -132,7 +135,7 @@ else# Run processColormap.js and move colormaps where we want themif [ -e "$BUILD_DIR/colormaps" ] ; then- if (( $MODE = "verbose" )) ; then+ if [ "$MODE" == "verbose" ] ; thenecho "Run processColormap.js and move colormaps where we want them"fimkdir -p "$BUILD_DIR"/config/palettes@@ -177,7 +180,7 @@ elsedone# Run mergeConfigWithWMTS.js to merge layer metadata from WMTS GC with worldview layer configs into wv.json- if (( $MODE = "verbose" )) ; then+ if [ "$MODE" == "verbose" ] ; thenecho "Run mergeConfigWithWMTS.js to merge layer metadata from WMTS GC with worldview layer configs into wv.json"fi`node $SCRIPTS_DIR/mergeConfigWithWMTS.js \@@ -186,7 +189,7 @@ else--mode "$MODE"`# Copy brand files from build to dest- if (( $MODE = "verbose" )) ; then+ if [ "$MODE" == "verbose" ] ; thenecho "Copy brand files from build to dest"ficp -r "$BUILD_DIR/brand" "$DEST_DIR"@@ -194,7 +197,7 @@ else# Validate the options build- if (( $MODE = "verbose" )) ; then+ if [ "$MODE" == "verbose" ] ; thenecho "Validate the options build"fi`node $SCRIPTS_DIR/validateOptions.js \@@ -203,7 +206,7 @@ else--mode "$MODE"`# Fetch preview images from WV Snapshots for any layers which they are missing- if (( $MODE = "verbose" )) ; then+ if [ "$MODE" == "verbose" ] ; thenecho "Fetch preview images from WV Snapshots for any layers which they are missing"fi`node $SCRIPTS_DIR/fetchPreviewSnapshots.js \diff --git a/tasks/link-check/natural-event-url-extract.js b/tasks/link-check/natural-event-url-extract.jsindex b2b5b8818..a9ebaf8d8 100644--- a/tasks/link-check/natural-event-url-extract.js+++ b/tasks/link-check/natural-event-url-extract.js@@ -1,8 +1,7 @@/* eslint-disable no-restricted-syntax */const fetch = require('node-fetch')--// delay helper to prevent too many requests at once-const sleeper = (ms) => (x) => new Promise((resolve) => setTimeout(() => resolve(x), ms))+const path = require('path')+const prog = path.basename(__filename)// helper function to find target key in nested objectconst findProp = async (obj, keys, out) => {@@ -35,27 +34,26 @@ const scrapeLinks = async (htmlLinks) => {const trackDoubles = {}const addedUrls = []- for (let i = 0; i < htmlLinks.length; i += 1) {- const htmlLink = htmlLinks[i]- // eslint-disable-next-line no-await-in-loop- await fetch(htmlLink)- .then(async (res) => {- const status = await res.json()- const urls = await findProp(status, ['url', 'link', 'source'])- for (let j = 0; j < urls.length; j += 1) {- const url = urls[j]- const linkRel = Object.keys(url)[0]- const linkHref = Object.values(url)[0]+ try {+ console.warn(`${prog}: Scraping ${htmlLinks.length} links from EONET...`)+ const results = await Promise.allSettled(htmlLinks.map((htmlLink, i) => fetch(htmlLink)))+ const statuses = await Promise.allSettled(results.map(({ status, value }) => status === 'fulfilled' && value.json()))+ const urls = await Promise.allSettled(statuses.map(({ status, value }) => status === 'fulfilled' && findProp(value, ['url', 'link', 'source'])))+ urls.forEach(({ status, value }) => {+ if (status === 'fulfilled' && Array.isArray(value)) {+ value.forEach(obj => {+ const linkRel = Object.keys(obj)[0]+ const linkHref = Object.values(obj)[0]if (trackDoubles[linkRel] !== linkHref) {addedUrls.push({ [linkRel]: linkHref })trackDoubles[linkRel] = linkHref}- }- }).then(sleeper(500))- .catch((err) => {- console.log(htmlLink, err)- })+ })+ }+ })+ } catch (err) {+ console.log(`${prog}: ERROR: ${err}`)}return addedUrls}diff --git a/tasks/link-check/url-check.js b/tasks/link-check/url-check.jsindex f446fe4d1..c15bd3c3c 100644--- a/tasks/link-check/url-check.js+++ b/tasks/link-check/url-check.js@@ -1,36 +1,33 @@const fetch = require('node-fetch')+const path = require('path')+const prog = path.basename(__filename)// URL status check function makes GET requests to provided list of URLs to get status codes / errors-// Delay helper to prevent too many requests at once-const sleeper = (ms) => (x) => new Promise((resolve) => setTimeout(() => resolve(x), ms))-// Checks status code of provided links and return object organized by errors / statuscodesconst requestCheck = async (urls) => {const parsedUrls = {ERROR: [],STATUSCODE: {}}- for (let i = 0; i < urls.length; i += 1) {- const linkName = Object.keys(urls[i])[0]- const url = Object.values(urls[i])[0]- // Skip for mailto email links- if (url[0] === 'h') {- // eslint-disable-next-line no-await-in-loop- await fetch(url, { timeout: 10000 })- .then(async (res) => {- const statusCode = await res.status- if (!parsedUrls.STATUSCODE[statusCode]) {- parsedUrls.STATUSCODE[statusCode] = []- }- parsedUrls.STATUSCODE[statusCode].push({ [linkName]: url })- })- .then(sleeper(500))- .catch((err) => { // eslint-disable-line n/handle-callback-err- parsedUrls.ERROR.push({ [linkName]: url })- })- }- }+ console.warn(`${prog}: Checking ${urls.length} URLs for errors/status codes...`)+ const promises = urls.map((link) => {+ const linkName = Object.keys(link)[0]+ const url = Object.values(link)[0]+ if (url[0] !== 'h') return Promise.resolve() // Skip for mailto email links+ return fetch(url, { timeout: 10000 })+ .then(async (res) => {+ const statusCode = await res.status+ if (!parsedUrls.STATUSCODE[statusCode]) {+ parsedUrls.STATUSCODE[statusCode] = []+ }+ parsedUrls.STATUSCODE[statusCode].push({ [linkName]: url })+ })+ .catch((err) => { // eslint-disable-line n/handle-callback-err+ parsedUrls.ERROR.push({ [linkName]: url })+ })+ })+ await Promise.allSettled(promises)return parsedUrls}