Skip to content

Interactive interface for browsing global, full-resolution satellite imagery

Revert "WV-3452 build time 2 (#6189)" (#6190)

Patrick Moulden0126f07b1549
diff --git a/package.json b/package.json
index 4b9212b90..021946963 100644
--- a/package.json
+++ b/package.json
@@ -28,12 +28,10 @@
"scripts": {
"analyze": "cross-env ANALYZE_MODE=true NODE_ENV=production webpack",
"build": "run-s clean getcapabilities build:config build:dev",
- "build:verbose": "run-s clean getcapabilities:verbose build:config:verbose build:dev",
+ "build:verbose": "run-s clean getcapabilities:verbose build:config build:dev",
"build:ci": "run-s getcapabilities build:config build:prod",
"build:options": "bash tasks/buildOptions.sh",
- "build:options:verbose": "bash tasks/buildOptions.sh -v",
"build:config": "run-s build:options && node ./tasks/util/config.js",
- "build:config:verbose": "run-s build:options:verbose && node ./tasks/util/config.js",
"build:dev": "cross-env NODE_ENV=development webpack",
"build:prod": "cross-env NODE_ENV=production webpack",
"clean": "node ./tasks/util/clean.js",
diff --git a/tasks/build-options/extractConfigFromWMTS.js b/tasks/build-options/extractConfigFromWMTS.js
index 5c8fe94a8..528dc86a9 100644
--- a/tasks/build-options/extractConfigFromWMTS.js
+++ b/tasks/build-options/extractConfigFromWMTS.js
@@ -28,12 +28,6 @@ const options = yargs(hideBin(process.argv))
type: 'string',
description: 'wmts output directory'
})
- .option('mode', {
- demandOption: true,
- alias: 'm',
- type: 'string',
- description: 'mode'
- })
.epilog('Extracts configuration information from a WMTS GetCapabilities file, converts the XML to JSON')
const { argv } = options
@@ -45,7 +39,6 @@ const configFile = argv.config
const config = JSON.parse(fs.readFileSync(configFile, 'utf-8'))
const inputDir = argv.inputDir
const outputDir = argv.outputDir
-const mode = argv.mode
if (!Object.prototype.hasOwnProperty.call(config, 'wv-options-wmts')) {
throw new Error(`${prog}: Error: "wv-options-wmts" not in config file`)
@@ -63,6 +56,12 @@ let totalLayerCount = 0
let totalWarningCount = 0
let totalErrorCount = 0
+const wv = {
+ layers: {},
+ sources: {}
+}
+let wvMatrixSets = {}
+
class SkipException extends Error {
constructor (message) {
super(message)
@@ -77,16 +76,14 @@ class SkipException extends Error {
* @throws {SkipException}
*/
async function main () {
- console.warn(`${prog}: Processing ${entries.length} entries...`)
-
- const promises = entries.map((entry) => processEntry(entry))
- const results = await Promise.allSettled(promises)
-
- results.forEach(({ value }) => {
- const { errorCount, warningCount, layerCount, wv, to, source } = value
- console.warn(`${prog}: ${errorCount} errors, ${warningCount} warnings, ${layerCount} layers for ${source}`)
-
- const outputFile = path.join(outputDir, to)
+ for (const entry of entries) {
+ wv.layers = {}
+ wv.sources = {}
+ wvMatrixSets = {}
+ const { errorCount, warningCount, layerCount } = await processEntry(entry)
+ console.warn(`${prog}: ${errorCount} errors, ${warningCount} warnings, ${layerCount} layers for ${entry.source}`)
+
+ const outputFile = path.join(outputDir, entry.to)
fs.writeFile(outputFile, JSON.stringify(wv, null, 2), 'utf-8', err => {
if (err) {
console.error(err)
@@ -96,9 +93,9 @@ async function main () {
totalErrorCount += errorCount
totalWarningCount += warningCount
totalLayerCount += layerCount
- })
+ }
- console.warn(`${prog}: ${totalErrorCount} errors, ${totalWarningCount} warnings, ${totalLayerCount} layers`)
+ console.warn(`${prog}:${totalErrorCount} errors, ${totalWarningCount} warnings, ${totalLayerCount} layers`)
if (totalErrorCount > 0) {
throw new Error(`${prog}: Error: ${totalErrorCount} errors occured`)
@@ -106,16 +103,14 @@ async function main () {
}
async function processEntry (entry) {
- console.warn(`${prog}: Processing ${entry.source} from ${entry.from} --> ${entry.to}...`)
- const wv = {}
- wv.layers = {}
- wv.sources = {}
- let wvMatrixSets = {}
-
let layerCount = 0
let warningCount = 0
let errorCount = 0
+ wv.sources[entry.source] = {
+ matrixSets: wvMatrixSets
+ }
+
const inputFile = path.join(inputDir, entry.from)
const gcId = path.basename(inputFile)
let gc
@@ -142,40 +137,36 @@ async function processEntry (entry) {
return { errorCount, warningCount, layerCount }
}
- console.warn(`${prog}: Processing ${gcContents.Layer.length} layers for ${entry.source}...`)
- const promises = gcContents.Layer.map((gcLayer) => processLayer(gcLayer, wvLayers, entry))
- const results = await Promise.allSettled(promises)
- console.warn(`${prog}: Processed ${results.length} layers for ${entry.source}...`)
-
- layerCount += results.length
-
- results.forEach((result) => {
- if (result.status === 'rejected') {
- if (result.reason instanceof SkipException) {
+ for (const gcLayer of gcContents.Layer) {
+ try {
+ layerCount += 1
+ await processLayer(gcLayer, wvLayers, entry)
+ } catch (error) {
+ if (error instanceof SkipException) {
warningCount += 1
- console.warn(`${prog}: WARNING: Skipping\n`)
+ console.warn(`${prog}: WARNING: [${gcId}] Skipping\n`)
} else {
errorCount += 1
- console.error(result.reason.stack)
- console.error(`${prog}: ERROR: ${result.reason}\n`)
+ console.error(error.stack)
+ const ident = gcLayer['ows:Identifier']._text
+ console.error(`${prog}: ERROR: [${gcId}:${ident}] ${error}\n`)
}
}
- })
-
- const gcTileMatrixSet = gcContents.TileMatrixSet
-
- wvMatrixSets = Array.isArray(gcTileMatrixSet) ? gcTileMatrixSet.map(gcMatrixSet => processMatrixSet(gcMatrixSet, entry)) : [processMatrixSet(gcTileMatrixSet, entry)]
+ }
- wv.sources[entry.source] = {
- matrixSets: Object.fromEntries(new Map(wvMatrixSets))
+ if (gcContents.TileMatrixSet === 'Object') {
+ processMatrixSet(gcContents.TileMatrixSet, entry)
+ } else {
+ gcContents.TileMatrixSet.forEach(gcMatrixSet => {
+ processMatrixSet(gcMatrixSet, entry)
+ })
}
- return { errorCount, warningCount, layerCount, wv, to: entry.to, source: entry.source }
+ return { errorCount, warningCount, layerCount }
}
async function processLayer (gcLayer, wvLayers, entry) {
const ident = gcLayer['ows:Identifier']._text
- if (mode === 'verbose') console.warn(`${prog}: Processing layer ${ident}...`)
if (skip.includes(ident)) {
console.log(`${ident}: skipping`)
throw new SkipException(ident)
@@ -307,7 +298,7 @@ function processMatrixSet (gcMatrixSet, entry) {
})
}
- const value = {
+ wvMatrixSets[ident] = {
id: ident,
maxResolution,
resolutions,
@@ -317,8 +308,6 @@ function processMatrixSet (gcMatrixSet, entry) {
],
tileMatrices: formattedTileMatrixArr
}
-
- return [ident, value]
}
main().catch((err) => {
diff --git a/tasks/build-options/fetchPreviewSnapshots.js b/tasks/build-options/fetchPreviewSnapshots.js
index 89e78b9b7..206f70636 100644
--- a/tasks/build-options/fetchPreviewSnapshots.js
+++ b/tasks/build-options/fetchPreviewSnapshots.js
@@ -101,12 +101,9 @@ async function main () {
throw new Error(`${prog}: Layer preview fetching disabled. Exiting.`)
}
- const layerValues = Object.values(layers)
-
- console.warn(`${prog}: Fetching snapshots for up to ${layerValues.length} layers...`)
- const promises = layerValues.map((layer) => getSnapshots(layer))
-
- await Promise.allSettled(promises)
+ for (const layer of Object.values(layers)) {
+ await getSnapshots(layer)
+ }
if (badSnapshots.length > 0) {
console.warn(`\n${prog}: WARNING: ${badSnapshots.length} snapshots returned no content. See below for details: `)
@@ -194,32 +191,29 @@ async function getBestDate (projection, period, dateRanges) {
return endDate
}
-async function getTimeParam (projection, layerId, layer) {
+async function getTimeParam (projection, layerId, layer, params) {
// Only include TIME param for temporal layers
- const out = {}
const dateRanges = layer.dateRanges
const startDate = layer.startDate
const period = layer.period
if (dateRanges) {
- out.TIME = await getBestDate(projection, period, dateRanges)
+ params.TIME = await getBestDate(projection, period, dateRanges)
} else if (startDate) {
- out.TIME = startDate
+ params.TIME = startDate
}
// Use any configured override dates
if (overrideDatesDict[layerId]) {
- out.TIME = overrideDatesDict[layerId]
+ params.TIME = overrideDatesDict[layerId]
}
-
- return out
}
async function getSnapshots (layer) {
for (const projection of Object.keys(layer.projections)) {
const projDict = layer.projections[projection]
const referenceLayer = referenceLayers[projection]
- let params = { ...paramDict.base, ...paramDict[projection] }
+ const params = { ...paramDict.base, ...paramDict[projection] }
// Sometimes a layer id is provided per projection (e.g. Land Mask layers)
// We need to use this layer id to request the layer from WVS/GIBS
@@ -240,8 +234,7 @@ async function getSnapshots (layer) {
.catch(() => false)
if (fileExists) continue
- const timeParam = await getTimeParam(projection, wvLayerId, layer)
- params = Object.assign(params, timeParam)
+ await getTimeParam(projection, wvLayerId, layer, params)
if (gibsLayerId !== referenceLayer && !standaloneLayers.includes(gibsLayerId)) {
params.LAYERS = `${referenceLayer},${gibsLayerId}`
diff --git a/tasks/build-options/getCapabilities.js b/tasks/build-options/getCapabilities.js
index 232af7b92..8b6ccbbd7 100644
--- a/tasks/build-options/getCapabilities.js
+++ b/tasks/build-options/getCapabilities.js
@@ -71,24 +71,28 @@ async function getCapabilities () {
// Download each GC xml using the "from" attribute and put it in the "to" location
if (Object.prototype.hasOwnProperty.call(config, 'wv-options-fetch')) {
const fetchValues = config['wv-options-fetch']
-
- const files = fetchValues.map(({ from, to }) => [from, `${outputDir}/${to}`])
- await Promise.allSettled(files.map(([from, to]) => fetchConfigs(from, to)))
- await Promise.allSettled(files.map(([, to]) => processGetCapabilities(to)))
+ for (const value of fetchValues) {
+ const inputFile = value.from
+ const outputFile = `${outputDir}/${value.to}`
+
+ if (argv.mode === 'verbose') console.warn(`Fetching config for ${inputFile} to ${outputFile}...`)
+ await fetchConfigs(inputFile, outputFile)
+ if (argv.mode === 'verbose') console.warn(`Processing capabilities for ${outputFile}...`)
+ await processGetCapabilities(outputFile)
+ }
}
}
// convert to superagent and use promises
async function fetchConfigs (inputFile, outputFile) {
const writer = await fs.createWriteStream(outputFile)
- if (argv.mode === 'verbose') console.warn(`${prog}: Fetching ${inputFile}...`)
return axios({
method: 'get',
url: inputFile,
responseType: 'stream',
timeout: 100000
}).then(async (response) => {
- if (argv.mode === 'verbose') console.warn(`${prog}: Writing ${outputFile}...`)
+ if (argv.mode === 'verbose') console.warn(`Writing ${outputFile}...`)
await response.data.pipe(writer)
return finished(writer)
})
@@ -125,7 +129,7 @@ async function processVectorData (layer) {
async function processLayer (layer) {
const ident = layer['ows:Identifier']._text
- if (argv.mode === 'verbose') console.warn(`Processing layer ${ident}...`)
+ if (argv.mode === 'verbose') console.warn(`Processing layer ${ident}:`)
if (layer['ows:Metadata']) {
if (config.skipPalettes) {
console.warn(`${prog}: WARN: Skipping palette for ${ident} \n`)
@@ -186,8 +190,9 @@ async function gatherProcess (type, typeStr, dir, ext) {
if (!fs.existsSync(dir)) {
fs.mkdirSync(dir)
}
- const promises = Object.values(type).map((link) => processMetadata(link, dir, ext))
- await Promise.allSettled(promises)
+ Object.values(type).forEach(async (link) => {
+ await processMetadata(link, dir, ext)
+ })
}
main().catch((err) => {
diff --git a/tasks/build-options/getVisMetadata.js b/tasks/build-options/getVisMetadata.js
index 3c1bb6315..865418162 100644
--- a/tasks/build-options/getVisMetadata.js
+++ b/tasks/build-options/getVisMetadata.js
@@ -59,6 +59,7 @@ const outputFile = argv.layerMetadata
const metadataConfig = features.features.vismetadata
const url = metadataConfig.url
const daacMap = metadataConfig.daacMap || {}
+const layerMetadata = {}
// These are alias or otherwise layers that don't exist in GIBS
const skipLayers = [
@@ -121,18 +122,11 @@ async function main (url) {
layerOrder = layerOrder.filter(x => !skipLayers.includes(x))
console.warn(`${prog}: Fetching ${layerOrder.length} layer-metadata files`)
- const promises = layerOrder.map((layerId) => {
- if (!layerId.includes('_STD') && !layerId.includes('_NRT')) return getMetadata(layerId, url)
- return Promise.reject(new Error(`Skipped layer: ${layerId}`))
- })
- const results = await Promise.allSettled(promises)
- const { fulfilled = [], rejected = [] } = Object.groupBy(results, (item) => item.status)
-
- rejected.forEach(({ reason }) => {
- console.error(`${prog}: ERROR: ${reason}\n`)
- })
- const metadataMap = new Map(fulfilled.map(({ value }) => value))
- const layerMetadata = Object.fromEntries(metadataMap.entries())
+ for (const layerId of layerOrder) {
+ if (!layerId.includes('_STD') && !layerId.includes('_NRT')) {
+ await getMetadata(layerId, url)
+ }
+ }
const layers = Object.keys(layerMetadata).sort().reduce(
(obj, key) => {
@@ -142,11 +136,11 @@ async function main (url) {
{}
)
- fs.writeFileSync(outputFile, JSON.stringify({ layers }))
- console.warn(`${prog}: Combined all ${Object.keys(layerMetadata).length} layer-metadata files into ${path.parse(outputFile).base}`)
+ await fs.writeFileSync(outputFile, JSON.stringify({ layers }))
+ console.warn(`${prog}: Combined all layer-metadata files into ${path.parse(outputFile).base}`)
}
-function getDAAC (metadata) {
+async function getDAAC (metadata) {
if (!Array.isArray(metadata.conceptIds) || !metadata.conceptIds.length) {
return metadata
}
@@ -156,7 +150,7 @@ function getDAAC (metadata) {
if (!dataCenter) {
continue
}
- delete collection.dataCenter
+ await delete collection.dataCenter
if (!metadata.dataCenter) {
metadata.dataCenter = [dataCenter]
} else if (!metadata.dataCenter.includes(dataCenter)) {
@@ -168,34 +162,31 @@ function getDAAC (metadata) {
async function getMetadata (layerId, baseUrl, count) {
if (count) console.warn(`retry #${count} for ${layerId}`)
- try {
- const response = await axios({
- method: 'get',
- url: `${baseUrl}${layerId}.json`,
- responseType: 'json',
- timeout: 10000
- })
+ return axios({
+ method: 'get',
+ url: `${baseUrl}${layerId}.json`,
+ responseType: 'json',
+ timeout: 10000
+ }).then(async (response) => {
const metadata = response.data
- const daac = getDAAC(metadata)
- let metadataKeys = Object.keys(daac)
+ layerMetadata[layerId] = await getDAAC(metadata)
+ let metadataKeys = Object.keys(layerMetadata[layerId])
metadataKeys = metadataKeys.filter(x => !useKeys.includes(x))
for (const key of metadataKeys) {
- delete daac[key]
+ delete layerMetadata[layerId][key]
}
- return [layerId, daac]
- } catch (error) {
- return await handleException(error, layerId, url, count)
- }
+ }).catch((error) => {
+ handleException(error, layerId, url, count)
+ })
}
async function handleException (error, layerId, url, count) {
if (!count) count = 0
count++
if (count <= 5) {
- return getMetadata(layerId, url, count)
+ await getMetadata(layerId, url, count)
} else {
console.warn(`\n ${prog} WARN: Unable to fetch ${layerId} ${error}`)
- return Promise.reject(new Error(`Failed to fetch layer ${layerId}: ${error}`))
}
}
diff --git a/tasks/build-options/mergeConfig.js b/tasks/build-options/mergeConfig.js
index 166969af0..8bb66ddfd 100644
--- a/tasks/build-options/mergeConfig.js
+++ b/tasks/build-options/mergeConfig.js
@@ -41,22 +41,20 @@ async function main () {
// inputDir refers to build\options-build\config\wv.json\*.*
async function mergeFiles (inputDir) {
const files = fs.readdirSync(inputDir)
- const promises = files.map((file) => {
+ for (const file of files) {
try {
if (file.endsWith('.json')) {
fileCount += 1
const data = JSON.parse(fs.readFileSync(path.join(inputDir, file), 'utf-8'))
- return dictMerge(conf, data)
+ await dictMerge(conf, data)
} else if (fs.existsSync(path.join(inputDir, file)) && fs.lstatSync(path.join(inputDir, file)).isDirectory()) {
const subDir = path.join(inputDir, file)
- return mergeFiles(subDir)
+ await mergeFiles(subDir)
}
- return Promise.resolve()
} catch (error) {
throw new Error(`ERROR: ${path.join(inputDir, file)}: ${error.message}`)
}
- })
- return Promise.allSettled(promises)
+ }
}
main().catch((err) => {
diff --git a/tasks/build-options/processColormap.js b/tasks/build-options/processColormap.js
index d6b875937..04eeef4f8 100755
--- a/tasks/build-options/processColormap.js
+++ b/tasks/build-options/processColormap.js
@@ -75,17 +75,17 @@ const walk = (dir) => {
async function main () {
const files = await walk(inputDir)
- const promises = files.map(async (file) => {
- const { id, xml } = await readFileAsync(file)
- return processFile(id, xml)
- })
- const results = await Promise.allSettled(promises)
- const { fulfilled = [], rejected = [] } = Object.groupBy(results, (item) => item.status)
-
- rejected.forEach(({ reason }) => console.error(`${prog}: ERROR: ${reason}`))
-
- fileCount += fulfilled.length
- errorCount += rejected.length
+ for (const file of files) {
+ try {
+ const { id, xml } = await readFileAsync(file)
+ await processFile(id, xml)
+ fileCount += 1
+ } catch (error) {
+ console.error(error)
+ console.error(`${prog}: ERROR: ${error}`)
+ errorCount += 1
+ }
+ }
try {
const wmtsFiles = fs.readdirSync(layersDir)
diff --git a/tasks/build-options/processVectorData.js b/tasks/build-options/processVectorData.js
index 292d1ca6e..d57c07353 100644
--- a/tasks/build-options/processVectorData.js
+++ b/tasks/build-options/processVectorData.js
@@ -38,14 +38,15 @@ async function main () {
let fileCount = 0
let errorCount = 0
- const promises = fs.readdirSync(inputDir).map((file) => copyFileAsync(file))
- const results = await Promise.allSettled(promises)
- const { fulfilled = [], rejected = [] } = Object.groupBy(results, (item) => item.status)
-
- rejected.forEach(({ reason }) => console.error(`${prog}: ERROR: ${reason}`))
-
- fileCount += fulfilled.length
- errorCount += rejected.length
+ for (const file of fs.readdirSync(inputDir)) {
+ try {
+ fileCount += 1
+ await copyFileAsync(file)
+ } catch (error) {
+ errorCount += 1
+ console.error(`${prog}: ERROR: [${file}] ${error}\n.`)
+ }
+ }
console.error(`${prog}: ${errorCount} error(s), ${fileCount} file(s)`)
diff --git a/tasks/build-options/processVectorStyles.js b/tasks/build-options/processVectorStyles.js
index c0d6b949b..d10c228e3 100644
--- a/tasks/build-options/processVectorStyles.js
+++ b/tasks/build-options/processVectorStyles.js
@@ -47,13 +47,15 @@ async function main () {
const files = [...fs.readdirSync(inputDir), ...fs.readdirSync(wvStylesDir)]
- const results = await Promise.allSettled(files.map((file) => copyFileAsync(file)))
- const { fulfilled = [], rejected = [] } = Object.groupBy(results, (item) => item.status)
-
- rejected.forEach(({ reason }) => console.error(`${prog}: ERROR: ${reason}\n`))
-
- fileCount += fulfilled.length
- errorCount += rejected.length
+ for (const file of files) {
+ try {
+ fileCount += 1
+ await copyFileAsync(file)
+ } catch (error) {
+ errorCount += 1
+ console.error(`${prog}: ERROR: [${file}] ${error}\n.`)
+ }
+ }
console.error(`${prog}: ${errorCount} error(s), ${fileCount} file(s)`)
diff --git a/tasks/buildOptions.sh b/tasks/buildOptions.sh
index 675116d49..6e6321a42 100755
--- a/tasks/buildOptions.sh
+++ b/tasks/buildOptions.sh
@@ -58,7 +58,7 @@ mkdir -p "$BUILD_DIR/colormaps"
# If $FETCH_GC is set, make various API requests
if [ "$FETCH_GC" ] ; then
# Fetch GC files and create colormaps, vectordata and vectorstyle files
- if [ "$MODE" == "verbose" ] ; then
+ if (( $MODE = "verbose" )) ; then
echo "Fetch GC files and create colormaps, vectordata and vectorstyle files"
fi
rm -rf "$OPT_DIR/$OPT_SUBDIR/gc/*"
@@ -69,7 +69,7 @@ if [ "$FETCH_GC" ] ; then
--mode "$MODE"`
# Get metadata for files in layerOrder.json and combine this data into 1 file
- if [ "$MODE" == "verbose" ] ; then
+ if (( $MODE = "verbose" )) ; then
echo "Get metadata for files in layerOrder.json and combine this data into 1 file"
fi
rm -rf "$OPT_DIR/$OPT_SUBDIR/layer-metadata"
@@ -81,7 +81,7 @@ if [ "$FETCH_GC" ] ; then
--mode "$MODE"`
else
# Validate layers in wv.json with a JSON schema
- if [ "$MODE" == "verbose" ] ; then
+ if (( $MODE = "verbose" )) ; then
echo "Validate layers in wv.json with a JSON schema"
fi
`node $SCRIPTS_DIR/validateConfigs.js \
@@ -90,15 +90,12 @@ else
--mode "$MODE"`
if [ -e "$BUILD_DIR/features.json" ] ; then
- if [ "$MODE" == "verbose" ] ; then
- echo "Copying features.json from $BUILD_DIR/features.json --> $BUILD_DIR/config/wv.json/_features.json"
- fi
- cp "$BUILD_DIR/features.json" "$BUILD_DIR/config/wv.json/_features.json"
+ cp "$BUILD_DIR/features.json" "$BUILD_DIR/config/wv.json/_features.json"
fi
# Run extractConfigFromWMTS.js script with config.json
if [ -e "$BUILD_DIR/config.json" ] ; then
- if [ "$MODE" == "verbose" ] ; then
+ if (( $MODE = "verbose" )) ; then
echo "Run extractConfigFromWMTS.js script with config.json"
fi
`node $SCRIPTS_DIR/extractConfigFromWMTS.js \
@@ -110,7 +107,7 @@ else
# Run processVectorStyles.js and move vectorstyles where we want them
if [ -e "$BUILD_DIR/gc/vectorstyles" ] ; then
- if [ "$MODE" == "verbose" ] ; then
+ if (( $MODE = "verbose" )) ; then
echo "Run processVectorStyles.js and move vectorstyles where we want them"
fi
mkdir -p "$BUILD_DIR/config/wv.json/vectorstyles"
@@ -123,7 +120,7 @@ else
# Run processVectorData.js and move vectordata where we want them
if [ -e "$BUILD_DIR/gc/vectordata" ] ; then
- if [ "$MODE" == "verbose" ] ; then
+ if (( $MODE = "verbose" )) ; then
echo "Run processVectorData.js and move vectordata where we want them"
fi
mkdir -p "$BUILD_DIR/config/wv.json/vectordata"
@@ -135,7 +132,7 @@ else
# Run processColormap.js and move colormaps where we want them
if [ -e "$BUILD_DIR/colormaps" ] ; then
- if [ "$MODE" == "verbose" ] ; then
+ if (( $MODE = "verbose" )) ; then
echo "Run processColormap.js and move colormaps where we want them"
fi
mkdir -p "$BUILD_DIR"/config/palettes
@@ -180,7 +177,7 @@ else
done
# Run mergeConfigWithWMTS.js to merge layer metadata from WMTS GC with worldview layer configs into wv.json
- if [ "$MODE" == "verbose" ] ; then
+ if (( $MODE = "verbose" )) ; then
echo "Run mergeConfigWithWMTS.js to merge layer metadata from WMTS GC with worldview layer configs into wv.json"
fi
`node $SCRIPTS_DIR/mergeConfigWithWMTS.js \
@@ -189,7 +186,7 @@ else
--mode "$MODE"`
# Copy brand files from build to dest
- if [ "$MODE" == "verbose" ] ; then
+ if (( $MODE = "verbose" )) ; then
echo "Copy brand files from build to dest"
fi
cp -r "$BUILD_DIR/brand" "$DEST_DIR"
@@ -197,7 +194,7 @@ else
# Validate the options build
- if [ "$MODE" == "verbose" ] ; then
+ if (( $MODE = "verbose" )) ; then
echo "Validate the options build"
fi
`node $SCRIPTS_DIR/validateOptions.js \
@@ -206,7 +203,7 @@ else
--mode "$MODE"`
# Fetch preview images from WV Snapshots for any layers which they are missing
- if [ "$MODE" == "verbose" ] ; then
+ if (( $MODE = "verbose" )) ; then
echo "Fetch preview images from WV Snapshots for any layers which they are missing"
fi
`node $SCRIPTS_DIR/fetchPreviewSnapshots.js \
diff --git a/tasks/link-check/natural-event-url-extract.js b/tasks/link-check/natural-event-url-extract.js
index a9ebaf8d8..b2b5b8818 100644
--- a/tasks/link-check/natural-event-url-extract.js
+++ b/tasks/link-check/natural-event-url-extract.js
@@ -1,7 +1,8 @@
/* eslint-disable no-restricted-syntax */
const fetch = require('node-fetch')
-const path = require('path')
-const prog = path.basename(__filename)
+
+// delay helper to prevent too many requests at once
+const sleeper = (ms) => (x) => new Promise((resolve) => setTimeout(() => resolve(x), ms))
// helper function to find target key in nested object
const findProp = async (obj, keys, out) => {
@@ -34,26 +35,27 @@ const scrapeLinks = async (htmlLinks) => {
const trackDoubles = {}
const addedUrls = []
- try {
- console.warn(`${prog}: Scraping ${htmlLinks.length} links from EONET...`)
- const results = await Promise.allSettled(htmlLinks.map((htmlLink, i) => fetch(htmlLink)))
- const statuses = await Promise.allSettled(results.map(({ status, value }) => status === 'fulfilled' && value.json()))
- const urls = await Promise.allSettled(statuses.map(({ status, value }) => status === 'fulfilled' && findProp(value, ['url', 'link', 'source'])))
- urls.forEach(({ status, value }) => {
- if (status === 'fulfilled' && Array.isArray(value)) {
- value.forEach(obj => {
- const linkRel = Object.keys(obj)[0]
- const linkHref = Object.values(obj)[0]
+ for (let i = 0; i < htmlLinks.length; i += 1) {
+ const htmlLink = htmlLinks[i]
+ // eslint-disable-next-line no-await-in-loop
+ await fetch(htmlLink)
+ .then(async (res) => {
+ const status = await res.json()
+ const urls = await findProp(status, ['url', 'link', 'source'])
+ for (let j = 0; j < urls.length; j += 1) {
+ const url = urls[j]
+ const linkRel = Object.keys(url)[0]
+ const linkHref = Object.values(url)[0]
if (trackDoubles[linkRel] !== linkHref) {
addedUrls.push({ [linkRel]: linkHref })
trackDoubles[linkRel] = linkHref
}
- })
- }
- })
- } catch (err) {
- console.log(`${prog}: ERROR: ${err}`)
+ }
+ }).then(sleeper(500))
+ .catch((err) => {
+ console.log(htmlLink, err)
+ })
}
return addedUrls
}
diff --git a/tasks/link-check/url-check.js b/tasks/link-check/url-check.js
index c15bd3c3c..f446fe4d1 100644
--- a/tasks/link-check/url-check.js
+++ b/tasks/link-check/url-check.js
@@ -1,33 +1,36 @@
const fetch = require('node-fetch')
-const path = require('path')
-const prog = path.basename(__filename)
// URL status check function makes GET requests to provided list of URLs to get status codes / errors
+// Delay helper to prevent too many requests at once
+const sleeper = (ms) => (x) => new Promise((resolve) => setTimeout(() => resolve(x), ms))
+
// Checks status code of provided links and return object organized by errors / statuscodes
const requestCheck = async (urls) => {
const parsedUrls = {
ERROR: [],
STATUSCODE: {}
}
+ for (let i = 0; i < urls.length; i += 1) {
+ const linkName = Object.keys(urls[i])[0]
+ const url = Object.values(urls[i])[0]
- console.warn(`${prog}: Checking ${urls.length} URLs for errors/status codes...`)
- const promises = urls.map((link) => {
- const linkName = Object.keys(link)[0]
- const url = Object.values(link)[0]
- if (url[0] !== 'h') return Promise.resolve() // Skip for mailto email links
- return fetch(url, { timeout: 10000 })
- .then(async (res) => {
- const statusCode = await res.status
- if (!parsedUrls.STATUSCODE[statusCode]) {
- parsedUrls.STATUSCODE[statusCode] = []
- }
- parsedUrls.STATUSCODE[statusCode].push({ [linkName]: url })
- })
- .catch((err) => { // eslint-disable-line n/handle-callback-err
- parsedUrls.ERROR.push({ [linkName]: url })
- })
- })
- await Promise.allSettled(promises)
+ // Skip for mailto email links
+ if (url[0] === 'h') {
+ // eslint-disable-next-line no-await-in-loop
+ await fetch(url, { timeout: 10000 })
+ .then(async (res) => {
+ const statusCode = await res.status
+ if (!parsedUrls.STATUSCODE[statusCode]) {
+ parsedUrls.STATUSCODE[statusCode] = []
+ }
+ parsedUrls.STATUSCODE[statusCode].push({ [linkName]: url })
+ })
+ .then(sleeper(500))
+ .catch((err) => { // eslint-disable-line n/handle-callback-err
+ parsedUrls.ERROR.push({ [linkName]: url })
+ })
+ }
+ }
return parsedUrls
}
diff --git a/web/images/layers/previews/antarctic/OrbitTracks_Sentinel-1C_Ascending.jpg b/web/images/layers/previews/antarctic/OrbitTracks_Sentinel-1C_Ascending.jpg
deleted file mode 100644
index f2646fec4..000000000
Binary files a/web/images/layers/previews/antarctic/OrbitTracks_Sentinel-1C_Ascending.jpg and /dev/null differ
diff --git a/web/images/layers/previews/antarctic/OrbitTracks_Sentinel-1C_Descending.jpg b/web/images/layers/previews/antarctic/OrbitTracks_Sentinel-1C_Descending.jpg
deleted file mode 100644
index 2e8cadd6c..000000000
Binary files a/web/images/layers/previews/antarctic/OrbitTracks_Sentinel-1C_Descending.jpg and /dev/null differ
diff --git a/web/images/layers/previews/arctic/OrbitTracks_Sentinel-1C_Ascending.jpg b/web/images/layers/previews/arctic/OrbitTracks_Sentinel-1C_Ascending.jpg
deleted file mode 100644
index b2fdd8fa3..000000000
Binary files a/web/images/layers/previews/arctic/OrbitTracks_Sentinel-1C_Ascending.jpg and /dev/null differ
diff --git a/web/images/layers/previews/arctic/OrbitTracks_Sentinel-1C_Descending.jpg b/web/images/layers/previews/arctic/OrbitTracks_Sentinel-1C_Descending.jpg
deleted file mode 100644
index de53d7209..000000000
Binary files a/web/images/layers/previews/arctic/OrbitTracks_Sentinel-1C_Descending.jpg and /dev/null differ