Interactive interface for browsing global, full-resolution satellite imagery
Revert "WV-3452 build time 2 (#6189)" (#6190)
Patrick Moulden0126f07b1549
diff --git a/package.json b/package.jsonindex 4b9212b90..021946963 100644--- a/package.json+++ b/package.json@@ -28,12 +28,10 @@"scripts": {"analyze": "cross-env ANALYZE_MODE=true NODE_ENV=production webpack","build": "run-s clean getcapabilities build:config build:dev",- "build:verbose": "run-s clean getcapabilities:verbose build:config:verbose build:dev",+ "build:verbose": "run-s clean getcapabilities:verbose build:config build:dev","build:ci": "run-s getcapabilities build:config build:prod","build:options": "bash tasks/buildOptions.sh",- "build:options:verbose": "bash tasks/buildOptions.sh -v","build:config": "run-s build:options && node ./tasks/util/config.js",- "build:config:verbose": "run-s build:options:verbose && node ./tasks/util/config.js","build:dev": "cross-env NODE_ENV=development webpack","build:prod": "cross-env NODE_ENV=production webpack","clean": "node ./tasks/util/clean.js",diff --git a/tasks/build-options/extractConfigFromWMTS.js b/tasks/build-options/extractConfigFromWMTS.jsindex 5c8fe94a8..528dc86a9 100644--- a/tasks/build-options/extractConfigFromWMTS.js+++ b/tasks/build-options/extractConfigFromWMTS.js@@ -28,12 +28,6 @@ const options = yargs(hideBin(process.argv))type: 'string',description: 'wmts output directory'})- .option('mode', {- demandOption: true,- alias: 'm',- type: 'string',- description: 'mode'- }).epilog('Extracts configuration information from a WMTS GetCapabilities file, converts the XML to JSON')const { argv } = options@@ -45,7 +39,6 @@ const configFile = argv.configconst config = JSON.parse(fs.readFileSync(configFile, 'utf-8'))const inputDir = argv.inputDirconst outputDir = argv.outputDir-const mode = argv.modeif (!Object.prototype.hasOwnProperty.call(config, 'wv-options-wmts')) {throw new Error(`${prog}: Error: "wv-options-wmts" not in config file`)@@ -63,6 +56,12 @@ let totalLayerCount = 0let totalWarningCount = 0let totalErrorCount = 0+const wv = {+ layers: {},+ sources: {}+}+let wvMatrixSets = {}+class SkipException extends Error {constructor (message) {super(message)@@ -77,16 +76,14 @@ class SkipException extends Error {* @throws {SkipException}*/async function main () {- console.warn(`${prog}: Processing ${entries.length} entries...`)-- const promises = entries.map((entry) => processEntry(entry))- const results = await Promise.allSettled(promises)-- results.forEach(({ value }) => {- const { errorCount, warningCount, layerCount, wv, to, source } = value- console.warn(`${prog}: ${errorCount} errors, ${warningCount} warnings, ${layerCount} layers for ${source}`)-- const outputFile = path.join(outputDir, to)+ for (const entry of entries) {+ wv.layers = {}+ wv.sources = {}+ wvMatrixSets = {}+ const { errorCount, warningCount, layerCount } = await processEntry(entry)+ console.warn(`${prog}: ${errorCount} errors, ${warningCount} warnings, ${layerCount} layers for ${entry.source}`)++ const outputFile = path.join(outputDir, entry.to)fs.writeFile(outputFile, JSON.stringify(wv, null, 2), 'utf-8', err => {if (err) {console.error(err)@@ -96,9 +93,9 @@ async function main () {totalErrorCount += errorCounttotalWarningCount += warningCounttotalLayerCount += layerCount- })+ }- console.warn(`${prog}: ${totalErrorCount} errors, ${totalWarningCount} warnings, ${totalLayerCount} layers`)+ console.warn(`${prog}:${totalErrorCount} errors, ${totalWarningCount} warnings, ${totalLayerCount} layers`)if (totalErrorCount > 0) {throw new Error(`${prog}: Error: ${totalErrorCount} errors occured`)@@ -106,16 +103,14 @@ async function main () {}async function processEntry (entry) {- console.warn(`${prog}: Processing ${entry.source} from ${entry.from} --> ${entry.to}...`)- const wv = {}- wv.layers = {}- wv.sources = {}- let wvMatrixSets = {}-let layerCount = 0let warningCount = 0let errorCount = 0+ wv.sources[entry.source] = {+ matrixSets: wvMatrixSets+ }+const inputFile = path.join(inputDir, entry.from)const gcId = path.basename(inputFile)let gc@@ -142,40 +137,36 @@ async function processEntry (entry) {return { errorCount, warningCount, layerCount }}- console.warn(`${prog}: Processing ${gcContents.Layer.length} layers for ${entry.source}...`)- const promises = gcContents.Layer.map((gcLayer) => processLayer(gcLayer, wvLayers, entry))- const results = await Promise.allSettled(promises)- console.warn(`${prog}: Processed ${results.length} layers for ${entry.source}...`)-- layerCount += results.length-- results.forEach((result) => {- if (result.status === 'rejected') {- if (result.reason instanceof SkipException) {+ for (const gcLayer of gcContents.Layer) {+ try {+ layerCount += 1+ await processLayer(gcLayer, wvLayers, entry)+ } catch (error) {+ if (error instanceof SkipException) {warningCount += 1- console.warn(`${prog}: WARNING: Skipping\n`)+ console.warn(`${prog}: WARNING: [${gcId}] Skipping\n`)} else {errorCount += 1- console.error(result.reason.stack)- console.error(`${prog}: ERROR: ${result.reason}\n`)+ console.error(error.stack)+ const ident = gcLayer['ows:Identifier']._text+ console.error(`${prog}: ERROR: [${gcId}:${ident}] ${error}\n`)}}- })-- const gcTileMatrixSet = gcContents.TileMatrixSet-- wvMatrixSets = Array.isArray(gcTileMatrixSet) ? gcTileMatrixSet.map(gcMatrixSet => processMatrixSet(gcMatrixSet, entry)) : [processMatrixSet(gcTileMatrixSet, entry)]+ }- wv.sources[entry.source] = {- matrixSets: Object.fromEntries(new Map(wvMatrixSets))+ if (gcContents.TileMatrixSet === 'Object') {+ processMatrixSet(gcContents.TileMatrixSet, entry)+ } else {+ gcContents.TileMatrixSet.forEach(gcMatrixSet => {+ processMatrixSet(gcMatrixSet, entry)+ })}- return { errorCount, warningCount, layerCount, wv, to: entry.to, source: entry.source }+ return { errorCount, warningCount, layerCount }}async function processLayer (gcLayer, wvLayers, entry) {const ident = gcLayer['ows:Identifier']._text- if (mode === 'verbose') console.warn(`${prog}: Processing layer ${ident}...`)if (skip.includes(ident)) {console.log(`${ident}: skipping`)throw new SkipException(ident)@@ -307,7 +298,7 @@ function processMatrixSet (gcMatrixSet, entry) {})}- const value = {+ wvMatrixSets[ident] = {id: ident,maxResolution,resolutions,@@ -317,8 +308,6 @@ function processMatrixSet (gcMatrixSet, entry) {],tileMatrices: formattedTileMatrixArr}-- return [ident, value]}main().catch((err) => {diff --git a/tasks/build-options/fetchPreviewSnapshots.js b/tasks/build-options/fetchPreviewSnapshots.jsindex 89e78b9b7..206f70636 100644--- a/tasks/build-options/fetchPreviewSnapshots.js+++ b/tasks/build-options/fetchPreviewSnapshots.js@@ -101,12 +101,9 @@ async function main () {throw new Error(`${prog}: Layer preview fetching disabled. Exiting.`)}- const layerValues = Object.values(layers)-- console.warn(`${prog}: Fetching snapshots for up to ${layerValues.length} layers...`)- const promises = layerValues.map((layer) => getSnapshots(layer))-- await Promise.allSettled(promises)+ for (const layer of Object.values(layers)) {+ await getSnapshots(layer)+ }if (badSnapshots.length > 0) {console.warn(`\n${prog}: WARNING: ${badSnapshots.length} snapshots returned no content. See below for details: `)@@ -194,32 +191,29 @@ async function getBestDate (projection, period, dateRanges) {return endDate}-async function getTimeParam (projection, layerId, layer) {+async function getTimeParam (projection, layerId, layer, params) {// Only include TIME param for temporal layers- const out = {}const dateRanges = layer.dateRangesconst startDate = layer.startDateconst period = layer.periodif (dateRanges) {- out.TIME = await getBestDate(projection, period, dateRanges)+ params.TIME = await getBestDate(projection, period, dateRanges)} else if (startDate) {- out.TIME = startDate+ params.TIME = startDate}// Use any configured override datesif (overrideDatesDict[layerId]) {- out.TIME = overrideDatesDict[layerId]+ params.TIME = overrideDatesDict[layerId]}-- return out}async function getSnapshots (layer) {for (const projection of Object.keys(layer.projections)) {const projDict = layer.projections[projection]const referenceLayer = referenceLayers[projection]- let params = { ...paramDict.base, ...paramDict[projection] }+ const params = { ...paramDict.base, ...paramDict[projection] }// Sometimes a layer id is provided per projection (e.g. Land Mask layers)// We need to use this layer id to request the layer from WVS/GIBS@@ -240,8 +234,7 @@ async function getSnapshots (layer) {.catch(() => false)if (fileExists) continue- const timeParam = await getTimeParam(projection, wvLayerId, layer)- params = Object.assign(params, timeParam)+ await getTimeParam(projection, wvLayerId, layer, params)if (gibsLayerId !== referenceLayer && !standaloneLayers.includes(gibsLayerId)) {params.LAYERS = `${referenceLayer},${gibsLayerId}`diff --git a/tasks/build-options/getCapabilities.js b/tasks/build-options/getCapabilities.jsindex 232af7b92..8b6ccbbd7 100644--- a/tasks/build-options/getCapabilities.js+++ b/tasks/build-options/getCapabilities.js@@ -71,24 +71,28 @@ async function getCapabilities () {// Download each GC xml using the "from" attribute and put it in the "to" locationif (Object.prototype.hasOwnProperty.call(config, 'wv-options-fetch')) {const fetchValues = config['wv-options-fetch']-- const files = fetchValues.map(({ from, to }) => [from, `${outputDir}/${to}`])- await Promise.allSettled(files.map(([from, to]) => fetchConfigs(from, to)))- await Promise.allSettled(files.map(([, to]) => processGetCapabilities(to)))+ for (const value of fetchValues) {+ const inputFile = value.from+ const outputFile = `${outputDir}/${value.to}`++ if (argv.mode === 'verbose') console.warn(`Fetching config for ${inputFile} to ${outputFile}...`)+ await fetchConfigs(inputFile, outputFile)+ if (argv.mode === 'verbose') console.warn(`Processing capabilities for ${outputFile}...`)+ await processGetCapabilities(outputFile)+ }}}// convert to superagent and use promisesasync function fetchConfigs (inputFile, outputFile) {const writer = await fs.createWriteStream(outputFile)- if (argv.mode === 'verbose') console.warn(`${prog}: Fetching ${inputFile}...`)return axios({method: 'get',url: inputFile,responseType: 'stream',timeout: 100000}).then(async (response) => {- if (argv.mode === 'verbose') console.warn(`${prog}: Writing ${outputFile}...`)+ if (argv.mode === 'verbose') console.warn(`Writing ${outputFile}...`)await response.data.pipe(writer)return finished(writer)})@@ -125,7 +129,7 @@ async function processVectorData (layer) {async function processLayer (layer) {const ident = layer['ows:Identifier']._text- if (argv.mode === 'verbose') console.warn(`Processing layer ${ident}...`)+ if (argv.mode === 'verbose') console.warn(`Processing layer ${ident}:`)if (layer['ows:Metadata']) {if (config.skipPalettes) {console.warn(`${prog}: WARN: Skipping palette for ${ident} \n`)@@ -186,8 +190,9 @@ async function gatherProcess (type, typeStr, dir, ext) {if (!fs.existsSync(dir)) {fs.mkdirSync(dir)}- const promises = Object.values(type).map((link) => processMetadata(link, dir, ext))- await Promise.allSettled(promises)+ Object.values(type).forEach(async (link) => {+ await processMetadata(link, dir, ext)+ })}main().catch((err) => {diff --git a/tasks/build-options/getVisMetadata.js b/tasks/build-options/getVisMetadata.jsindex 3c1bb6315..865418162 100644--- a/tasks/build-options/getVisMetadata.js+++ b/tasks/build-options/getVisMetadata.js@@ -59,6 +59,7 @@ const outputFile = argv.layerMetadataconst metadataConfig = features.features.vismetadataconst url = metadataConfig.urlconst daacMap = metadataConfig.daacMap || {}+const layerMetadata = {}// These are alias or otherwise layers that don't exist in GIBSconst skipLayers = [@@ -121,18 +122,11 @@ async function main (url) {layerOrder = layerOrder.filter(x => !skipLayers.includes(x))console.warn(`${prog}: Fetching ${layerOrder.length} layer-metadata files`)- const promises = layerOrder.map((layerId) => {- if (!layerId.includes('_STD') && !layerId.includes('_NRT')) return getMetadata(layerId, url)- return Promise.reject(new Error(`Skipped layer: ${layerId}`))- })- const results = await Promise.allSettled(promises)- const { fulfilled = [], rejected = [] } = Object.groupBy(results, (item) => item.status)-- rejected.forEach(({ reason }) => {- console.error(`${prog}: ERROR: ${reason}\n`)- })- const metadataMap = new Map(fulfilled.map(({ value }) => value))- const layerMetadata = Object.fromEntries(metadataMap.entries())+ for (const layerId of layerOrder) {+ if (!layerId.includes('_STD') && !layerId.includes('_NRT')) {+ await getMetadata(layerId, url)+ }+ }const layers = Object.keys(layerMetadata).sort().reduce((obj, key) => {@@ -142,11 +136,11 @@ async function main (url) {{})- fs.writeFileSync(outputFile, JSON.stringify({ layers }))- console.warn(`${prog}: Combined all ${Object.keys(layerMetadata).length} layer-metadata files into ${path.parse(outputFile).base}`)+ await fs.writeFileSync(outputFile, JSON.stringify({ layers }))+ console.warn(`${prog}: Combined all layer-metadata files into ${path.parse(outputFile).base}`)}-function getDAAC (metadata) {+async function getDAAC (metadata) {if (!Array.isArray(metadata.conceptIds) || !metadata.conceptIds.length) {return metadata}@@ -156,7 +150,7 @@ function getDAAC (metadata) {if (!dataCenter) {continue}- delete collection.dataCenter+ await delete collection.dataCenterif (!metadata.dataCenter) {metadata.dataCenter = [dataCenter]} else if (!metadata.dataCenter.includes(dataCenter)) {@@ -168,34 +162,31 @@ function getDAAC (metadata) {async function getMetadata (layerId, baseUrl, count) {if (count) console.warn(`retry #${count} for ${layerId}`)- try {- const response = await axios({- method: 'get',- url: `${baseUrl}${layerId}.json`,- responseType: 'json',- timeout: 10000- })+ return axios({+ method: 'get',+ url: `${baseUrl}${layerId}.json`,+ responseType: 'json',+ timeout: 10000+ }).then(async (response) => {const metadata = response.data- const daac = getDAAC(metadata)- let metadataKeys = Object.keys(daac)+ layerMetadata[layerId] = await getDAAC(metadata)+ let metadataKeys = Object.keys(layerMetadata[layerId])metadataKeys = metadataKeys.filter(x => !useKeys.includes(x))for (const key of metadataKeys) {- delete daac[key]+ delete layerMetadata[layerId][key]}- return [layerId, daac]- } catch (error) {- return await handleException(error, layerId, url, count)- }+ }).catch((error) => {+ handleException(error, layerId, url, count)+ })}async function handleException (error, layerId, url, count) {if (!count) count = 0count++if (count <= 5) {- return getMetadata(layerId, url, count)+ await getMetadata(layerId, url, count)} else {console.warn(`\n ${prog} WARN: Unable to fetch ${layerId} ${error}`)- return Promise.reject(new Error(`Failed to fetch layer ${layerId}: ${error}`))}}diff --git a/tasks/build-options/mergeConfig.js b/tasks/build-options/mergeConfig.jsindex 166969af0..8bb66ddfd 100644--- a/tasks/build-options/mergeConfig.js+++ b/tasks/build-options/mergeConfig.js@@ -41,22 +41,20 @@ async function main () {// inputDir refers to build\options-build\config\wv.json\*.*async function mergeFiles (inputDir) {const files = fs.readdirSync(inputDir)- const promises = files.map((file) => {+ for (const file of files) {try {if (file.endsWith('.json')) {fileCount += 1const data = JSON.parse(fs.readFileSync(path.join(inputDir, file), 'utf-8'))- return dictMerge(conf, data)+ await dictMerge(conf, data)} else if (fs.existsSync(path.join(inputDir, file)) && fs.lstatSync(path.join(inputDir, file)).isDirectory()) {const subDir = path.join(inputDir, file)- return mergeFiles(subDir)+ await mergeFiles(subDir)}- return Promise.resolve()} catch (error) {throw new Error(`ERROR: ${path.join(inputDir, file)}: ${error.message}`)}- })- return Promise.allSettled(promises)+ }}main().catch((err) => {diff --git a/tasks/build-options/processColormap.js b/tasks/build-options/processColormap.jsindex d6b875937..04eeef4f8 100755--- a/tasks/build-options/processColormap.js+++ b/tasks/build-options/processColormap.js@@ -75,17 +75,17 @@ const walk = (dir) => {async function main () {const files = await walk(inputDir)- const promises = files.map(async (file) => {- const { id, xml } = await readFileAsync(file)- return processFile(id, xml)- })- const results = await Promise.allSettled(promises)- const { fulfilled = [], rejected = [] } = Object.groupBy(results, (item) => item.status)-- rejected.forEach(({ reason }) => console.error(`${prog}: ERROR: ${reason}`))-- fileCount += fulfilled.length- errorCount += rejected.length+ for (const file of files) {+ try {+ const { id, xml } = await readFileAsync(file)+ await processFile(id, xml)+ fileCount += 1+ } catch (error) {+ console.error(error)+ console.error(`${prog}: ERROR: ${error}`)+ errorCount += 1+ }+ }try {const wmtsFiles = fs.readdirSync(layersDir)diff --git a/tasks/build-options/processVectorData.js b/tasks/build-options/processVectorData.jsindex 292d1ca6e..d57c07353 100644--- a/tasks/build-options/processVectorData.js+++ b/tasks/build-options/processVectorData.js@@ -38,14 +38,15 @@ async function main () {let fileCount = 0let errorCount = 0- const promises = fs.readdirSync(inputDir).map((file) => copyFileAsync(file))- const results = await Promise.allSettled(promises)- const { fulfilled = [], rejected = [] } = Object.groupBy(results, (item) => item.status)-- rejected.forEach(({ reason }) => console.error(`${prog}: ERROR: ${reason}`))-- fileCount += fulfilled.length- errorCount += rejected.length+ for (const file of fs.readdirSync(inputDir)) {+ try {+ fileCount += 1+ await copyFileAsync(file)+ } catch (error) {+ errorCount += 1+ console.error(`${prog}: ERROR: [${file}] ${error}\n.`)+ }+ }console.error(`${prog}: ${errorCount} error(s), ${fileCount} file(s)`)diff --git a/tasks/build-options/processVectorStyles.js b/tasks/build-options/processVectorStyles.jsindex c0d6b949b..d10c228e3 100644--- a/tasks/build-options/processVectorStyles.js+++ b/tasks/build-options/processVectorStyles.js@@ -47,13 +47,15 @@ async function main () {const files = [...fs.readdirSync(inputDir), ...fs.readdirSync(wvStylesDir)]- const results = await Promise.allSettled(files.map((file) => copyFileAsync(file)))- const { fulfilled = [], rejected = [] } = Object.groupBy(results, (item) => item.status)-- rejected.forEach(({ reason }) => console.error(`${prog}: ERROR: ${reason}\n`))-- fileCount += fulfilled.length- errorCount += rejected.length+ for (const file of files) {+ try {+ fileCount += 1+ await copyFileAsync(file)+ } catch (error) {+ errorCount += 1+ console.error(`${prog}: ERROR: [${file}] ${error}\n.`)+ }+ }console.error(`${prog}: ${errorCount} error(s), ${fileCount} file(s)`)diff --git a/tasks/buildOptions.sh b/tasks/buildOptions.shindex 675116d49..6e6321a42 100755--- a/tasks/buildOptions.sh+++ b/tasks/buildOptions.sh@@ -58,7 +58,7 @@ mkdir -p "$BUILD_DIR/colormaps"# If $FETCH_GC is set, make various API requestsif [ "$FETCH_GC" ] ; then# Fetch GC files and create colormaps, vectordata and vectorstyle files- if [ "$MODE" == "verbose" ] ; then+ if (( $MODE = "verbose" )) ; thenecho "Fetch GC files and create colormaps, vectordata and vectorstyle files"firm -rf "$OPT_DIR/$OPT_SUBDIR/gc/*"@@ -69,7 +69,7 @@ if [ "$FETCH_GC" ] ; then--mode "$MODE"`# Get metadata for files in layerOrder.json and combine this data into 1 file- if [ "$MODE" == "verbose" ] ; then+ if (( $MODE = "verbose" )) ; thenecho "Get metadata for files in layerOrder.json and combine this data into 1 file"firm -rf "$OPT_DIR/$OPT_SUBDIR/layer-metadata"@@ -81,7 +81,7 @@ if [ "$FETCH_GC" ] ; then--mode "$MODE"`else# Validate layers in wv.json with a JSON schema- if [ "$MODE" == "verbose" ] ; then+ if (( $MODE = "verbose" )) ; thenecho "Validate layers in wv.json with a JSON schema"fi`node $SCRIPTS_DIR/validateConfigs.js \@@ -90,15 +90,12 @@ else--mode "$MODE"`if [ -e "$BUILD_DIR/features.json" ] ; then- if [ "$MODE" == "verbose" ] ; then- echo "Copying features.json from $BUILD_DIR/features.json --> $BUILD_DIR/config/wv.json/_features.json"- fi- cp "$BUILD_DIR/features.json" "$BUILD_DIR/config/wv.json/_features.json"+ cp "$BUILD_DIR/features.json" "$BUILD_DIR/config/wv.json/_features.json"fi# Run extractConfigFromWMTS.js script with config.jsonif [ -e "$BUILD_DIR/config.json" ] ; then- if [ "$MODE" == "verbose" ] ; then+ if (( $MODE = "verbose" )) ; thenecho "Run extractConfigFromWMTS.js script with config.json"fi`node $SCRIPTS_DIR/extractConfigFromWMTS.js \@@ -110,7 +107,7 @@ else# Run processVectorStyles.js and move vectorstyles where we want themif [ -e "$BUILD_DIR/gc/vectorstyles" ] ; then- if [ "$MODE" == "verbose" ] ; then+ if (( $MODE = "verbose" )) ; thenecho "Run processVectorStyles.js and move vectorstyles where we want them"fimkdir -p "$BUILD_DIR/config/wv.json/vectorstyles"@@ -123,7 +120,7 @@ else# Run processVectorData.js and move vectordata where we want themif [ -e "$BUILD_DIR/gc/vectordata" ] ; then- if [ "$MODE" == "verbose" ] ; then+ if (( $MODE = "verbose" )) ; thenecho "Run processVectorData.js and move vectordata where we want them"fimkdir -p "$BUILD_DIR/config/wv.json/vectordata"@@ -135,7 +132,7 @@ else# Run processColormap.js and move colormaps where we want themif [ -e "$BUILD_DIR/colormaps" ] ; then- if [ "$MODE" == "verbose" ] ; then+ if (( $MODE = "verbose" )) ; thenecho "Run processColormap.js and move colormaps where we want them"fimkdir -p "$BUILD_DIR"/config/palettes@@ -180,7 +177,7 @@ elsedone# Run mergeConfigWithWMTS.js to merge layer metadata from WMTS GC with worldview layer configs into wv.json- if [ "$MODE" == "verbose" ] ; then+ if (( $MODE = "verbose" )) ; thenecho "Run mergeConfigWithWMTS.js to merge layer metadata from WMTS GC with worldview layer configs into wv.json"fi`node $SCRIPTS_DIR/mergeConfigWithWMTS.js \@@ -189,7 +186,7 @@ else--mode "$MODE"`# Copy brand files from build to dest- if [ "$MODE" == "verbose" ] ; then+ if (( $MODE = "verbose" )) ; thenecho "Copy brand files from build to dest"ficp -r "$BUILD_DIR/brand" "$DEST_DIR"@@ -197,7 +194,7 @@ else# Validate the options build- if [ "$MODE" == "verbose" ] ; then+ if (( $MODE = "verbose" )) ; thenecho "Validate the options build"fi`node $SCRIPTS_DIR/validateOptions.js \@@ -206,7 +203,7 @@ else--mode "$MODE"`# Fetch preview images from WV Snapshots for any layers which they are missing- if [ "$MODE" == "verbose" ] ; then+ if (( $MODE = "verbose" )) ; thenecho "Fetch preview images from WV Snapshots for any layers which they are missing"fi`node $SCRIPTS_DIR/fetchPreviewSnapshots.js \diff --git a/tasks/link-check/natural-event-url-extract.js b/tasks/link-check/natural-event-url-extract.jsindex a9ebaf8d8..b2b5b8818 100644--- a/tasks/link-check/natural-event-url-extract.js+++ b/tasks/link-check/natural-event-url-extract.js@@ -1,7 +1,8 @@/* eslint-disable no-restricted-syntax */const fetch = require('node-fetch')-const path = require('path')-const prog = path.basename(__filename)++// delay helper to prevent too many requests at once+const sleeper = (ms) => (x) => new Promise((resolve) => setTimeout(() => resolve(x), ms))// helper function to find target key in nested objectconst findProp = async (obj, keys, out) => {@@ -34,26 +35,27 @@ const scrapeLinks = async (htmlLinks) => {const trackDoubles = {}const addedUrls = []- try {- console.warn(`${prog}: Scraping ${htmlLinks.length} links from EONET...`)- const results = await Promise.allSettled(htmlLinks.map((htmlLink, i) => fetch(htmlLink)))- const statuses = await Promise.allSettled(results.map(({ status, value }) => status === 'fulfilled' && value.json()))- const urls = await Promise.allSettled(statuses.map(({ status, value }) => status === 'fulfilled' && findProp(value, ['url', 'link', 'source'])))- urls.forEach(({ status, value }) => {- if (status === 'fulfilled' && Array.isArray(value)) {- value.forEach(obj => {- const linkRel = Object.keys(obj)[0]- const linkHref = Object.values(obj)[0]+ for (let i = 0; i < htmlLinks.length; i += 1) {+ const htmlLink = htmlLinks[i]+ // eslint-disable-next-line no-await-in-loop+ await fetch(htmlLink)+ .then(async (res) => {+ const status = await res.json()+ const urls = await findProp(status, ['url', 'link', 'source'])+ for (let j = 0; j < urls.length; j += 1) {+ const url = urls[j]+ const linkRel = Object.keys(url)[0]+ const linkHref = Object.values(url)[0]if (trackDoubles[linkRel] !== linkHref) {addedUrls.push({ [linkRel]: linkHref })trackDoubles[linkRel] = linkHref}- })- }- })- } catch (err) {- console.log(`${prog}: ERROR: ${err}`)+ }+ }).then(sleeper(500))+ .catch((err) => {+ console.log(htmlLink, err)+ })}return addedUrls}diff --git a/tasks/link-check/url-check.js b/tasks/link-check/url-check.jsindex c15bd3c3c..f446fe4d1 100644--- a/tasks/link-check/url-check.js+++ b/tasks/link-check/url-check.js@@ -1,33 +1,36 @@const fetch = require('node-fetch')-const path = require('path')-const prog = path.basename(__filename)// URL status check function makes GET requests to provided list of URLs to get status codes / errors+// Delay helper to prevent too many requests at once+const sleeper = (ms) => (x) => new Promise((resolve) => setTimeout(() => resolve(x), ms))+// Checks status code of provided links and return object organized by errors / statuscodesconst requestCheck = async (urls) => {const parsedUrls = {ERROR: [],STATUSCODE: {}}+ for (let i = 0; i < urls.length; i += 1) {+ const linkName = Object.keys(urls[i])[0]+ const url = Object.values(urls[i])[0]- console.warn(`${prog}: Checking ${urls.length} URLs for errors/status codes...`)- const promises = urls.map((link) => {- const linkName = Object.keys(link)[0]- const url = Object.values(link)[0]- if (url[0] !== 'h') return Promise.resolve() // Skip for mailto email links- return fetch(url, { timeout: 10000 })- .then(async (res) => {- const statusCode = await res.status- if (!parsedUrls.STATUSCODE[statusCode]) {- parsedUrls.STATUSCODE[statusCode] = []- }- parsedUrls.STATUSCODE[statusCode].push({ [linkName]: url })- })- .catch((err) => { // eslint-disable-line n/handle-callback-err- parsedUrls.ERROR.push({ [linkName]: url })- })- })- await Promise.allSettled(promises)+ // Skip for mailto email links+ if (url[0] === 'h') {+ // eslint-disable-next-line no-await-in-loop+ await fetch(url, { timeout: 10000 })+ .then(async (res) => {+ const statusCode = await res.status+ if (!parsedUrls.STATUSCODE[statusCode]) {+ parsedUrls.STATUSCODE[statusCode] = []+ }+ parsedUrls.STATUSCODE[statusCode].push({ [linkName]: url })+ })+ .then(sleeper(500))+ .catch((err) => { // eslint-disable-line n/handle-callback-err+ parsedUrls.ERROR.push({ [linkName]: url })+ })+ }+ }return parsedUrls}diff --git a/web/images/layers/previews/antarctic/OrbitTracks_Sentinel-1C_Ascending.jpg b/web/images/layers/previews/antarctic/OrbitTracks_Sentinel-1C_Ascending.jpgdeleted file mode 100644index f2646fec4..000000000Binary files a/web/images/layers/previews/antarctic/OrbitTracks_Sentinel-1C_Ascending.jpg and /dev/null differdiff --git a/web/images/layers/previews/antarctic/OrbitTracks_Sentinel-1C_Descending.jpg b/web/images/layers/previews/antarctic/OrbitTracks_Sentinel-1C_Descending.jpgdeleted file mode 100644index 2e8cadd6c..000000000Binary files a/web/images/layers/previews/antarctic/OrbitTracks_Sentinel-1C_Descending.jpg and /dev/null differdiff --git a/web/images/layers/previews/arctic/OrbitTracks_Sentinel-1C_Ascending.jpg b/web/images/layers/previews/arctic/OrbitTracks_Sentinel-1C_Ascending.jpgdeleted file mode 100644index b2fdd8fa3..000000000Binary files a/web/images/layers/previews/arctic/OrbitTracks_Sentinel-1C_Ascending.jpg and /dev/null differdiff --git a/web/images/layers/previews/arctic/OrbitTracks_Sentinel-1C_Descending.jpg b/web/images/layers/previews/arctic/OrbitTracks_Sentinel-1C_Descending.jpgdeleted file mode 100644index de53d7209..000000000Binary files a/web/images/layers/previews/arctic/OrbitTracks_Sentinel-1C_Descending.jpg and /dev/null differ