Interactive interface for browsing global, full-resolution satellite imagery
WV-3750 ignore cache flag (#6194)
Patrick Moulden29d1ffe87fb1
diff --git a/package.json b/package.jsonindex ec364bfbd..538b2bff8 100644--- a/package.json+++ b/package.json@@ -29,11 +29,14 @@"analyze": "cross-env ANALYZE_MODE=true NODE_ENV=production webpack","build": "run-s clean getcapabilities build:config build:dev","build:verbose": "run-s clean getcapabilities:verbose build:config:verbose build:dev",+ "build:debug": "run-s clean getcapabilities:debug build:config:debug build:dev",+ "build:nocache": "run-s clean getcapabilities:nocache build:config:nocache build:dev","build:ci": "run-s getcapabilities build:config build:prod","build:options": "bash tasks/buildOptions.sh",- "build:options:verbose": "bash tasks/buildOptions.sh -v","build:config": "run-s build:options && node ./tasks/util/config.js",- "build:config:verbose": "run-s build:options:verbose && node ./tasks/util/config.js",+ "build:config:verbose": "run-s build:options -- -v && node ./tasks/util/config.js",+ "build:config:debug": "run-s build:options -- -d && node ./tasks/util/config.js",+ "build:config:nocache": "run-s build:options -- -f && node ./tasks/util/config.js","build:dev": "cross-env NODE_ENV=development webpack","build:prod": "cross-env NODE_ENV=production webpack","clean": "node ./tasks/util/clean.js",@@ -51,6 +54,8 @@"e2e": "npm run docker:restart && docker exec -it worldview /bin/bash /build/e2e/docker-ci.sh ; npm run docker:stop","getcapabilities": "bash -c 'FETCH_GC=1 npm run build:options'","getcapabilities:verbose": "bash -c 'FETCH_GC=1 npm run build:options -- -v'",+ "getcapabilities:debug": "bash -c 'FETCH_GC=1 npm run build:options -- -d'",+ "getcapabilities:nocache": "bash -c 'FETCH_GC=1 npm run build:options -- -f'","generatePreviews": "node ./tasks/build-options/fetchPreviewSnapshots.py","linkcheck": "node ./tasks/link-check","lint": "run-s lint:js lint:scss",diff --git a/tasks/build-options/extractConfigFromWMTS.js b/tasks/build-options/extractConfigFromWMTS.jsindex 3e0d0bc32..a89efe2f0 100644--- a/tasks/build-options/extractConfigFromWMTS.js+++ b/tasks/build-options/extractConfigFromWMTS.js@@ -34,6 +34,12 @@ const options = yargs(hideBin(process.argv))type: 'string',description: 'mode'})+ .option('cacheMode', {+ demandOption: false,+ alias: 'cm',+ type: 'string',+ description: 'Cache mode for fetching data'+ }).epilog('Extracts configuration information from a WMTS GetCapabilities file, converts the XML to JSON')const { argv } = options@@ -46,6 +52,7 @@ const config = JSON.parse(fs.readFileSync(configFile, 'utf-8'))const inputDir = argv.inputDirconst outputDir = argv.outputDirconst mode = argv.mode+const cacheMode = argv.cacheModeif (!Object.prototype.hasOwnProperty.call(config, 'wv-options-wmts')) {throw new Error(`${prog}: Error: "wv-options-wmts" not in config file`)@@ -175,7 +182,7 @@ async function processEntry (entry) {async function processLayer (gcLayer, wvLayers, entry) {const ident = gcLayer['ows:Identifier']._text- if (mode === 'verbose') console.warn(`${prog}: Processing layer ${ident}...`)+ if (mode === 'verbose') console.trace(`Processing layer ${ident}...`)if (skip.includes(ident)) {console.log(`${ident}: skipping`)throw new SkipException(ident)@@ -194,7 +201,7 @@ async function processLayer (gcLayer, wvLayers, entry) {const dimension = gcLayer.Dimensionif (dimension['ows:Identifier']._text === 'Time') {try {- wvLayer = await processTemporalLayer(wvLayer, dimension.Value, entry.source)+ wvLayer = await processTemporalLayer(wvLayer, dimension.Value, entry.source, cacheMode)} catch (e) {console.error(e)console.error(`${prog}: ERROR: [${ident}] Error processing time values.`)diff --git a/tasks/build-options/fetchPreviewSnapshots.js b/tasks/build-options/fetchPreviewSnapshots.jsindex 89e78b9b7..7c9dcd346 100644--- a/tasks/build-options/fetchPreviewSnapshots.js+++ b/tasks/build-options/fetchPreviewSnapshots.js@@ -27,6 +27,12 @@ const options = yargs(hideBin(process.argv))type: 'string',description: 'features file'})+ .option('cacheMode', {+ demandOption: false,+ alias: 'c',+ type: 'string',+ description: 'Cache mode for fetching data'+ }).epilog('Fetch preview images from WV Snapshots for any layers which they are missing.')const { argv } = options@@ -37,6 +43,7 @@ if (!argv.wvJsonFile && !argv.overridesFile && !argv.featuresFile) {const wvJsonFile = argv.wvJsonFileconst overridesFile = argv.overridesFileconst featuresFile = argv.featuresFile+const cacheMode = argv.cacheModelet overrideDatesDict = {}const badSnapshots = []@@ -250,12 +257,24 @@ async function getSnapshots (layer) {params.LAYERS = gibsLayerId}+ let headers = {}++ if (cacheMode === 'no-store') {+ const noCacheHeaders = {+ 'Cache-Control': 'no-cache no-store',+ Pragma: 'no-cache',+ Expires: '0'+ }+ headers = Object.assign(headers, noCacheHeaders)+ }+try {const imageReq = await axios({method: 'get',url: snapshotsUrl,params,- responseType: 'stream'+ responseType: 'stream',+ headers})let statusTextif (imageReq.status === 200) {diff --git a/tasks/build-options/getCapabilities.js b/tasks/build-options/getCapabilities.jsindex 232af7b92..8edd9f916 100644--- a/tasks/build-options/getCapabilities.js+++ b/tasks/build-options/getCapabilities.js@@ -31,6 +31,12 @@ const options = yargs(hideBin(process.argv))type: 'string',description: 'mode'})+ .option('cacheMode', {+ demandOption: false,+ alias: 'f',+ type: 'string',+ description: 'Cache mode for fetching data'+ }).epilog('Pulls GetCapabilities XML and linked metadata from configured locations')const { argv } = options@@ -39,6 +45,7 @@ if (!argv.config && !argv.getcapabilities) {}const configFile = argv.configconst outputDir = argv.getcapabilities+const cacheMode = argv.cacheModeconst colormaps = {}const vectorstyles = {}const vectordata = {}@@ -49,6 +56,17 @@ const vectordataDir = path.join(outputDir, 'vectordata')const configData = fs.readFileSync(configFile)const config = JSON.parse(configData)+let headers = {}++if (cacheMode === 'no-store') {+ const noCacheHeaders = {+ 'Cache-Control': 'no-cache no-store',+ Pragma: 'no-cache',+ Expires: '0'+ }+ headers = Object.assign(headers, noCacheHeaders)+}+async function main () {if (!fs.existsSync(outputDir)) {fs.mkdirSync(outputDir)@@ -81,17 +99,17 @@ async function getCapabilities () {// convert to superagent and use promisesasync function fetchConfigs (inputFile, outputFile) {const writer = await fs.createWriteStream(outputFile)- if (argv.mode === 'verbose') console.warn(`${prog}: Fetching ${inputFile}...`)- return axios({+ if (argv.mode === 'verbose') console.trace(`Fetching ${inputFile}...`)+ const response = await axios({method: 'get',url: inputFile,responseType: 'stream',- timeout: 100000- }).then(async (response) => {- if (argv.mode === 'verbose') console.warn(`${prog}: Writing ${outputFile}...`)- await response.data.pipe(writer)- return finished(writer)+ timeout: 100000,+ headers})+ if (argv.mode === 'verbose') console.trace(`Writing ${outputFile}...`)+ await response.data.pipe(writer)+ return finished(writer)}async function handleException (error, link, dir, ext, count) {@@ -113,7 +131,7 @@ async function processVectorData (layer) {Object.values(layer['ows:Metadata']).forEach((item) => {const schemaVersion = item._attributes['xlink:role']if (schemaVersion === 'http://earthdata.nasa.gov/gibs/metadata-type/layer/1.0') {- if (argv.mode === 'verbose') console.warn(` Processing Metadata: ${item._attributes['xlink:href']}`)+ if (argv.mode === 'verbose') console.trace(` Processing Metadata: ${item._attributes['xlink:href']}`)const vectorDataLink = item._attributes['xlink:href']const vectorDataFile = path.basename(vectorDataLink)const vectorDataId = path.parse(vectorDataFile).name@@ -125,13 +143,13 @@ async function processVectorData (layer) {async function processLayer (layer) {const ident = layer['ows:Identifier']._text- if (argv.mode === 'verbose') console.warn(`Processing layer ${ident}...`)+ if (argv.mode === 'verbose') console.trace(`Processing layer ${ident}...`)if (layer['ows:Metadata']) {if (config.skipPalettes) {console.warn(`${prog}: WARN: Skipping palette for ${ident} \n`)} else {Object.values(layer['ows:Metadata']).forEach((item) => {- if (argv.mode === 'verbose') console.warn(` Processing pallette: ${item._attributes['xlink:href']}`)+ if (argv.mode === 'verbose') console.trace(` Processing pallette: ${item._attributes['xlink:href']}`)const schemaVersion = item._attributes['xlink:role']if (schemaVersion === 'http://earthdata.nasa.gov/gibs/metadata-type/colormap/1.3') {const colormapLink = item._attributes['xlink:href']diff --git a/tasks/build-options/getVisMetadata.js b/tasks/build-options/getVisMetadata.jsindex 79de39d06..c66dba301 100644--- a/tasks/build-options/getVisMetadata.js+++ b/tasks/build-options/getVisMetadata.js@@ -27,6 +27,12 @@ const options = yargs(hideBin(process.argv))type: 'string',description: 'layer-metadata/all.json file'})+ .option('cacheMode', {+ demandOption: false,+ alias: 'c',+ type: 'string',+ description: 'Cache mode for fetching data'+ }).epilog('Creates a layer-metadata file containing all layers')const { argv } = options@@ -34,6 +40,8 @@ if (!argv.features && !argv.layerOrder && !argv.layerMetadata) {throw new Error('Invalid number of arguments')}+const cacheMode = argv.cacheMode+const featuresFile = argv.featureslet featuresData = fs.readFileSync(featuresFile)let features = JSON.parse(featuresData)@@ -171,6 +179,17 @@ function getDAAC (metadata) {return metadata}+let headers = {}++if (cacheMode === 'no-store') {+ const noCacheHeaders = {+ 'Cache-Control': 'no-cache no-store',+ Pragma: 'no-cache',+ Expires: '0'+ }+ headers = Object.assign(headers, noCacheHeaders)+}+async function getMetadata (layerId, baseUrl, count) {if (count) console.warn(`retry #${count} for ${layerId}`)try {@@ -178,7 +197,8 @@ async function getMetadata (layerId, baseUrl, count) {method: 'get',url: `${baseUrl}${layerId}.json`,responseType: 'json',- timeout: 10000+ timeout: 10000,+ headers})const metadata = response.dataconst daac = getDAAC(metadata)diff --git a/tasks/build-options/processTemporalLayer.js b/tasks/build-options/processTemporalLayer.jsindex de8860a9f..310428348 100644--- a/tasks/build-options/processTemporalLayer.js+++ b/tasks/build-options/processTemporalLayer.js@@ -11,14 +11,15 @@ function toList (val) {return val instanceof Array ? val : [val]}-async function processTemporalLayer (wvLayer, value, source = 'GIBS:geographic') {+async function processTemporalLayer (wvLayer, value, source = 'GIBS:geographic', cacheMode) {const dateFormat = 'YYYY-MM-DD'const dateTimeFormat = 'YYYY-MM-DD HH:mm:ss'+ const fetchOpts = cacheMode === 'no-store' ? { cache: cacheMode } : undefinedtry {let ranges = toList(value)const describeDomainsUrl = `https://gibs.earthdata.nasa.gov/wmts/${projDict[source]}/best/1.0.0/${wvLayer.id}/default/250m/all/all.xml`try {- const describeDomainsResponse = await fetch(describeDomainsUrl)+ const describeDomainsResponse = await fetch(describeDomainsUrl, fetchOpts)if (describeDomainsResponse?.ok) {const describeDomainsText = await describeDomainsResponse?.text?.() || ''const parser = new xml2js.Parser()diff --git a/tasks/build-options/validateConfigs.js b/tasks/build-options/validateConfigs.jsindex 2d85f39a2..9b2bea872 100644--- a/tasks/build-options/validateConfigs.js+++ b/tasks/build-options/validateConfigs.js@@ -67,7 +67,7 @@ async function main () {}async function validateFile (filePath) {- if (argv.mode === 'verbose') console.warn(`${prog}: Validating ${filePath}`)+ if (argv.mode === 'verbose') console.trace(`Validating ${filePath}`)const layerFile = fs.readFileSync(filePath)const layer = JSON.parse(layerFile)const valid = validate(layer)diff --git a/tasks/build-options/validateOptions.js b/tasks/build-options/validateOptions.jsindex dde6f808b..99d84bbd3 100644--- a/tasks/build-options/validateOptions.js+++ b/tasks/build-options/validateOptions.js@@ -20,6 +20,12 @@ const options = yargs(hideBin(process.argv))type: 'string',description: 'config directory'})+ .option('cacheMode', {+ demandOption: false,+ alias: 'cm',+ type: 'string',+ description: 'Cache mode for fetching data'+ }).epilog('Validates and corrects the configuration files.')const { argv } = options@@ -29,6 +35,7 @@ if (!argv.optionsFile && !argv.configDir) {const optionsFile = argv.optionsFileconst configDir = argv.configDir+const cacheMode = argv.cacheModelet errorCount = 0let warningCount = 0@@ -110,7 +117,7 @@ async function main () {}if ('temporal' in layer) {warn(`[${layerId}] GC Layer temporal values overwritten by Options`)- layer = await processTemporalLayer(layer, layer.temporal)+ layer = await processTemporalLayer(layer, layer.temporal, undefined, cacheMode)}if (layer.futureTime) {if ('endDate' in layer) delete layer.endDatediff --git a/tasks/buildOptions.sh b/tasks/buildOptions.shindex 675116d49..4760e2f07 100755--- a/tasks/buildOptions.sh+++ b/tasks/buildOptions.sh@@ -8,13 +8,25 @@ DEST_DIR="$BASE/build/options"SCRIPTS_DIR="$BASE/tasks/build-options"MODE="default"+CACHE_MODE="default"-while getopts ":v" option; do+while getopts ":vfd" option; doecho "Option -$option set"case $option inv)- echo "Verbose Mode Activated"- MODE="verbose";;+ echo "Verbose Mode Activated"+ MODE="verbose"+ ;;+ f)+ echo "Force Mode Activated"+ CACHE_MODE="no-store"+ ;;+ d)+ echo "Debug Mode Activated"+ export NODE_OPTIONS='--inspect'+ export NODE_DEBUG=http,http2+ echo "Open debugger to view results (change '--inspect' to '--inspect-brk' to break on start)"+ ;;esacdone@@ -66,7 +78,8 @@ if [ "$FETCH_GC" ] ; then`node $SCRIPTS_DIR/getCapabilities.js \--config "$OPT_DIR/$OPT_SUBDIR/config.json" \--getcapabilities "$OPT_DIR/$OPT_SUBDIR/gc" \- --mode "$MODE"`+ --mode "$MODE" \+ --cacheMode "$CACHE_MODE"`# Get metadata for files in layerOrder.json and combine this data into 1 fileif [ "$MODE" == "verbose" ] ; then@@ -78,7 +91,8 @@ if [ "$FETCH_GC" ] ; then--features "$BUILD_DIR/features.json" \--layerOrder "$BUILD_DIR/config/wv.json/layerOrder.json" \--layerMetadata "$OPT_DIR/$OPT_SUBDIR/layer-metadata/all.json" \- --mode "$MODE"`+ --mode "$MODE" \+ --cacheMode "$CACHE_MODE"`else# Validate layers in wv.json with a JSON schemaif [ "$MODE" == "verbose" ] ; then@@ -87,7 +101,8 @@ else`node $SCRIPTS_DIR/validateConfigs.js \--inputDirectory "$SRC_DIR/common/config/wv.json/layers" \--schemaFile "$BASE/schemas/layer-config.json" \- --mode "$MODE"`+ --mode "$MODE" \+ --cacheMode "$CACHE_MODE"`if [ -e "$BUILD_DIR/features.json" ] ; thenif [ "$MODE" == "verbose" ] ; then@@ -105,7 +120,8 @@ else--config "$BUILD_DIR/config.json" \--inputDir "$BUILD_DIR/gc" \--outputDir "$BUILD_DIR/_wmts" \- --mode "$MODE"`+ --mode "$MODE" \+ --cacheMode "$CACHE_MODE"`fi# Run processVectorStyles.js and move vectorstyles where we want them@@ -118,7 +134,8 @@ else--inputDir "$BUILD_DIR/gc/vectorstyles" \--wvStylesDir "$SRC_DIR/common/vectorstyles" \--outputDir "$BUILD_DIR/config/wv.json/vectorstyles" \- --mode "$MODE"`+ --mode "$MODE" \+ --cacheMode "$CACHE_MODE"`fi# Run processVectorData.js and move vectordata where we want them@@ -130,7 +147,8 @@ else`node $SCRIPTS_DIR/processVectorData.js \--inputDir "$BUILD_DIR/gc/vectordata" \--outputDir "$BUILD_DIR/config/wv.json/vectordata" \- --mode "$MODE"`+ --mode "$MODE" \+ --cacheMode "$CACHE_MODE"`fi# Run processColormap.js and move colormaps where we want them@@ -147,7 +165,8 @@ else--inputDir "$BUILD_DIR/colormaps" \--outputDir "$BUILD_DIR/config/palettes" \--layersDir "$BUILD_DIR/_wmts" \- --mode "$MODE"`+ --mode "$MODE" \+ --cacheMode "$CACHE_MODE"`fi# Throw error if no categoryGroupOrder.json file present@@ -156,7 +175,8 @@ else`node $SCRIPTS_DIR/generateCategoryGroupOrder.js \--inputDir "$SRC_DIR/common/config/wv.json/categories/" \--outputDir "$SRC_DIR/common/config/wv.json/" \- --mode "$MODE"`+ --mode "$MODE" \+ --cacheMode "$CACHE_MODE"`fiif [ -e "$OPT_DIR/$OPT_SUBDIR/layer-metadata/all.json" ] ; then@@ -186,7 +206,8 @@ else`node $SCRIPTS_DIR/mergeConfigWithWMTS.js \--inputDir "$BUILD_DIR/_wmts" \--outputFile "$DEST_DIR/config/wv.json" \- --mode "$MODE"`+ --mode "$MODE" \+ --cacheMode "$CACHE_MODE"`# Copy brand files from build to destif [ "$MODE" == "verbose" ] ; then@@ -203,7 +224,8 @@ else`node $SCRIPTS_DIR/validateOptions.js \--optionsFile "$BUILD_DIR/config.json" \--configDir "$DEST_DIR/config" \- --mode "$MODE"`+ --mode "$MODE" \+ --cacheMode "$CACHE_MODE"`# Fetch preview images from WV Snapshots for any layers which they are missingif [ "$MODE" == "verbose" ] ; then@@ -213,6 +235,7 @@ else--wvJsonFile "$DEST_DIR/config/wv.json" \--overridesFile "$OPT_DIR/common/previewLayerOverrides.json" \--featuresFile "$BUILD_DIR/features.json" \- --mode "$MODE"`+ --mode "$MODE" \+ --cacheMode "$CACHE_MODE"`fiexit 0diff --git a/web/images/layers/previews/antarctic/OrbitTracks_Sentinel-1C_Ascending.jpg b/web/images/layers/previews/antarctic/OrbitTracks_Sentinel-1C_Ascending.jpgdeleted file mode 100644index 7d9c05ee4..000000000--- a/web/images/layers/previews/antarctic/OrbitTracks_Sentinel-1C_Ascending.jpg+++ /dev/null@@ -1,6 +0,0 @@-<?xml version='1.0' encoding="UTF-8" standalone="no" ?>-<ServiceExceptionReport version="1.3.0" xmlns="http://www.opengis.net/ogc" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.opengis.net/ogc http://schemas.opengis.net/wms/1.3.0/exceptions_1_3_0.xsd">-<ServiceException code="LayerNotDefined">-msWMSLoadGetMapParams(): WMS server error. Invalid layer(s) given in the LAYERS parameter. A layer might be disabled for this request. Check wms/ows_enable_request settings.-</ServiceException>-</ServiceExceptionReport>diff --git a/web/images/layers/previews/antarctic/OrbitTracks_Sentinel-1C_Descending.jpg b/web/images/layers/previews/antarctic/OrbitTracks_Sentinel-1C_Descending.jpgdeleted file mode 100644index 5a3ff655b..000000000Binary files a/web/images/layers/previews/antarctic/OrbitTracks_Sentinel-1C_Descending.jpg and /dev/null differdiff --git a/web/images/layers/previews/arctic/OrbitTracks_Sentinel-1C_Ascending.jpg b/web/images/layers/previews/arctic/OrbitTracks_Sentinel-1C_Ascending.jpgdeleted file mode 100644index f5a3c60a7..000000000Binary files a/web/images/layers/previews/arctic/OrbitTracks_Sentinel-1C_Ascending.jpg and /dev/null differdiff --git a/web/images/layers/previews/arctic/OrbitTracks_Sentinel-1C_Descending.jpg b/web/images/layers/previews/arctic/OrbitTracks_Sentinel-1C_Descending.jpgdeleted file mode 100644index 3017ec811..000000000Binary files a/web/images/layers/previews/arctic/OrbitTracks_Sentinel-1C_Descending.jpg and /dev/null differ