Skip to content

Interactive interface for browsing global, full-resolution satellite imagery

WV-3750 ignore cache flag (#6194)

Patrick Moulden29d1ffe87fb1
diff --git a/package.json b/package.json
index ec364bfbd..538b2bff8 100644
--- a/package.json
+++ b/package.json
@@ -29,11 +29,14 @@
"analyze": "cross-env ANALYZE_MODE=true NODE_ENV=production webpack",
"build": "run-s clean getcapabilities build:config build:dev",
"build:verbose": "run-s clean getcapabilities:verbose build:config:verbose build:dev",
+ "build:debug": "run-s clean getcapabilities:debug build:config:debug build:dev",
+ "build:nocache": "run-s clean getcapabilities:nocache build:config:nocache build:dev",
"build:ci": "run-s getcapabilities build:config build:prod",
"build:options": "bash tasks/buildOptions.sh",
- "build:options:verbose": "bash tasks/buildOptions.sh -v",
"build:config": "run-s build:options && node ./tasks/util/config.js",
- "build:config:verbose": "run-s build:options:verbose && node ./tasks/util/config.js",
+ "build:config:verbose": "run-s build:options -- -v && node ./tasks/util/config.js",
+ "build:config:debug": "run-s build:options -- -d && node ./tasks/util/config.js",
+ "build:config:nocache": "run-s build:options -- -f && node ./tasks/util/config.js",
"build:dev": "cross-env NODE_ENV=development webpack",
"build:prod": "cross-env NODE_ENV=production webpack",
"clean": "node ./tasks/util/clean.js",
@@ -51,6 +54,8 @@
"e2e": "npm run docker:restart && docker exec -it worldview /bin/bash /build/e2e/docker-ci.sh ; npm run docker:stop",
"getcapabilities": "bash -c 'FETCH_GC=1 npm run build:options'",
"getcapabilities:verbose": "bash -c 'FETCH_GC=1 npm run build:options -- -v'",
+ "getcapabilities:debug": "bash -c 'FETCH_GC=1 npm run build:options -- -d'",
+ "getcapabilities:nocache": "bash -c 'FETCH_GC=1 npm run build:options -- -f'",
"generatePreviews": "node ./tasks/build-options/fetchPreviewSnapshots.py",
"linkcheck": "node ./tasks/link-check",
"lint": "run-s lint:js lint:scss",
diff --git a/tasks/build-options/extractConfigFromWMTS.js b/tasks/build-options/extractConfigFromWMTS.js
index 3e0d0bc32..a89efe2f0 100644
--- a/tasks/build-options/extractConfigFromWMTS.js
+++ b/tasks/build-options/extractConfigFromWMTS.js
@@ -34,6 +34,12 @@ const options = yargs(hideBin(process.argv))
type: 'string',
description: 'mode'
})
+ .option('cacheMode', {
+ demandOption: false,
+ alias: 'cm',
+ type: 'string',
+ description: 'Cache mode for fetching data'
+ })
.epilog('Extracts configuration information from a WMTS GetCapabilities file, converts the XML to JSON')
const { argv } = options
@@ -46,6 +52,7 @@ const config = JSON.parse(fs.readFileSync(configFile, 'utf-8'))
const inputDir = argv.inputDir
const outputDir = argv.outputDir
const mode = argv.mode
+const cacheMode = argv.cacheMode
if (!Object.prototype.hasOwnProperty.call(config, 'wv-options-wmts')) {
throw new Error(`${prog}: Error: "wv-options-wmts" not in config file`)
@@ -175,7 +182,7 @@ async function processEntry (entry) {
async function processLayer (gcLayer, wvLayers, entry) {
const ident = gcLayer['ows:Identifier']._text
- if (mode === 'verbose') console.warn(`${prog}: Processing layer ${ident}...`)
+ if (mode === 'verbose') console.trace(`Processing layer ${ident}...`)
if (skip.includes(ident)) {
console.log(`${ident}: skipping`)
throw new SkipException(ident)
@@ -194,7 +201,7 @@ async function processLayer (gcLayer, wvLayers, entry) {
const dimension = gcLayer.Dimension
if (dimension['ows:Identifier']._text === 'Time') {
try {
- wvLayer = await processTemporalLayer(wvLayer, dimension.Value, entry.source)
+ wvLayer = await processTemporalLayer(wvLayer, dimension.Value, entry.source, cacheMode)
} catch (e) {
console.error(e)
console.error(`${prog}: ERROR: [${ident}] Error processing time values.`)
diff --git a/tasks/build-options/fetchPreviewSnapshots.js b/tasks/build-options/fetchPreviewSnapshots.js
index 89e78b9b7..7c9dcd346 100644
--- a/tasks/build-options/fetchPreviewSnapshots.js
+++ b/tasks/build-options/fetchPreviewSnapshots.js
@@ -27,6 +27,12 @@ const options = yargs(hideBin(process.argv))
type: 'string',
description: 'features file'
})
+ .option('cacheMode', {
+ demandOption: false,
+ alias: 'c',
+ type: 'string',
+ description: 'Cache mode for fetching data'
+ })
.epilog('Fetch preview images from WV Snapshots for any layers which they are missing.')
const { argv } = options
@@ -37,6 +43,7 @@ if (!argv.wvJsonFile && !argv.overridesFile && !argv.featuresFile) {
const wvJsonFile = argv.wvJsonFile
const overridesFile = argv.overridesFile
const featuresFile = argv.featuresFile
+const cacheMode = argv.cacheMode
let overrideDatesDict = {}
const badSnapshots = []
@@ -250,12 +257,24 @@ async function getSnapshots (layer) {
params.LAYERS = gibsLayerId
}
+ let headers = {}
+
+ if (cacheMode === 'no-store') {
+ const noCacheHeaders = {
+ 'Cache-Control': 'no-cache no-store',
+ Pragma: 'no-cache',
+ Expires: '0'
+ }
+ headers = Object.assign(headers, noCacheHeaders)
+ }
+
try {
const imageReq = await axios({
method: 'get',
url: snapshotsUrl,
params,
- responseType: 'stream'
+ responseType: 'stream',
+ headers
})
let statusText
if (imageReq.status === 200) {
diff --git a/tasks/build-options/getCapabilities.js b/tasks/build-options/getCapabilities.js
index 232af7b92..8edd9f916 100644
--- a/tasks/build-options/getCapabilities.js
+++ b/tasks/build-options/getCapabilities.js
@@ -31,6 +31,12 @@ const options = yargs(hideBin(process.argv))
type: 'string',
description: 'mode'
})
+ .option('cacheMode', {
+ demandOption: false,
+ alias: 'f',
+ type: 'string',
+ description: 'Cache mode for fetching data'
+ })
.epilog('Pulls GetCapabilities XML and linked metadata from configured locations')
const { argv } = options
@@ -39,6 +45,7 @@ if (!argv.config && !argv.getcapabilities) {
}
const configFile = argv.config
const outputDir = argv.getcapabilities
+const cacheMode = argv.cacheMode
const colormaps = {}
const vectorstyles = {}
const vectordata = {}
@@ -49,6 +56,17 @@ const vectordataDir = path.join(outputDir, 'vectordata')
const configData = fs.readFileSync(configFile)
const config = JSON.parse(configData)
+let headers = {}
+
+if (cacheMode === 'no-store') {
+ const noCacheHeaders = {
+ 'Cache-Control': 'no-cache no-store',
+ Pragma: 'no-cache',
+ Expires: '0'
+ }
+ headers = Object.assign(headers, noCacheHeaders)
+}
+
async function main () {
if (!fs.existsSync(outputDir)) {
fs.mkdirSync(outputDir)
@@ -81,17 +99,17 @@ async function getCapabilities () {
// convert to superagent and use promises
async function fetchConfigs (inputFile, outputFile) {
const writer = await fs.createWriteStream(outputFile)
- if (argv.mode === 'verbose') console.warn(`${prog}: Fetching ${inputFile}...`)
- return axios({
+ if (argv.mode === 'verbose') console.trace(`Fetching ${inputFile}...`)
+ const response = await axios({
method: 'get',
url: inputFile,
responseType: 'stream',
- timeout: 100000
- }).then(async (response) => {
- if (argv.mode === 'verbose') console.warn(`${prog}: Writing ${outputFile}...`)
- await response.data.pipe(writer)
- return finished(writer)
+ timeout: 100000,
+ headers
})
+ if (argv.mode === 'verbose') console.trace(`Writing ${outputFile}...`)
+ await response.data.pipe(writer)
+ return finished(writer)
}
async function handleException (error, link, dir, ext, count) {
@@ -113,7 +131,7 @@ async function processVectorData (layer) {
Object.values(layer['ows:Metadata']).forEach((item) => {
const schemaVersion = item._attributes['xlink:role']
if (schemaVersion === 'http://earthdata.nasa.gov/gibs/metadata-type/layer/1.0') {
- if (argv.mode === 'verbose') console.warn(` Processing Metadata: ${item._attributes['xlink:href']}`)
+ if (argv.mode === 'verbose') console.trace(` Processing Metadata: ${item._attributes['xlink:href']}`)
const vectorDataLink = item._attributes['xlink:href']
const vectorDataFile = path.basename(vectorDataLink)
const vectorDataId = path.parse(vectorDataFile).name
@@ -125,13 +143,13 @@ async function processVectorData (layer) {
async function processLayer (layer) {
const ident = layer['ows:Identifier']._text
- if (argv.mode === 'verbose') console.warn(`Processing layer ${ident}...`)
+ if (argv.mode === 'verbose') console.trace(`Processing layer ${ident}...`)
if (layer['ows:Metadata']) {
if (config.skipPalettes) {
console.warn(`${prog}: WARN: Skipping palette for ${ident} \n`)
} else {
Object.values(layer['ows:Metadata']).forEach((item) => {
- if (argv.mode === 'verbose') console.warn(` Processing pallette: ${item._attributes['xlink:href']}`)
+ if (argv.mode === 'verbose') console.trace(` Processing pallette: ${item._attributes['xlink:href']}`)
const schemaVersion = item._attributes['xlink:role']
if (schemaVersion === 'http://earthdata.nasa.gov/gibs/metadata-type/colormap/1.3') {
const colormapLink = item._attributes['xlink:href']
diff --git a/tasks/build-options/getVisMetadata.js b/tasks/build-options/getVisMetadata.js
index 79de39d06..c66dba301 100644
--- a/tasks/build-options/getVisMetadata.js
+++ b/tasks/build-options/getVisMetadata.js
@@ -27,6 +27,12 @@ const options = yargs(hideBin(process.argv))
type: 'string',
description: 'layer-metadata/all.json file'
})
+ .option('cacheMode', {
+ demandOption: false,
+ alias: 'c',
+ type: 'string',
+ description: 'Cache mode for fetching data'
+ })
.epilog('Creates a layer-metadata file containing all layers')
const { argv } = options
@@ -34,6 +40,8 @@ if (!argv.features && !argv.layerOrder && !argv.layerMetadata) {
throw new Error('Invalid number of arguments')
}
+const cacheMode = argv.cacheMode
+
const featuresFile = argv.features
let featuresData = fs.readFileSync(featuresFile)
let features = JSON.parse(featuresData)
@@ -171,6 +179,17 @@ function getDAAC (metadata) {
return metadata
}
+let headers = {}
+
+if (cacheMode === 'no-store') {
+ const noCacheHeaders = {
+ 'Cache-Control': 'no-cache no-store',
+ Pragma: 'no-cache',
+ Expires: '0'
+ }
+ headers = Object.assign(headers, noCacheHeaders)
+}
+
async function getMetadata (layerId, baseUrl, count) {
if (count) console.warn(`retry #${count} for ${layerId}`)
try {
@@ -178,7 +197,8 @@ async function getMetadata (layerId, baseUrl, count) {
method: 'get',
url: `${baseUrl}${layerId}.json`,
responseType: 'json',
- timeout: 10000
+ timeout: 10000,
+ headers
})
const metadata = response.data
const daac = getDAAC(metadata)
diff --git a/tasks/build-options/processTemporalLayer.js b/tasks/build-options/processTemporalLayer.js
index de8860a9f..310428348 100644
--- a/tasks/build-options/processTemporalLayer.js
+++ b/tasks/build-options/processTemporalLayer.js
@@ -11,14 +11,15 @@ function toList (val) {
return val instanceof Array ? val : [val]
}
-async function processTemporalLayer (wvLayer, value, source = 'GIBS:geographic') {
+async function processTemporalLayer (wvLayer, value, source = 'GIBS:geographic', cacheMode) {
const dateFormat = 'YYYY-MM-DD'
const dateTimeFormat = 'YYYY-MM-DD HH:mm:ss'
+ const fetchOpts = cacheMode === 'no-store' ? { cache: cacheMode } : undefined
try {
let ranges = toList(value)
const describeDomainsUrl = `https://gibs.earthdata.nasa.gov/wmts/${projDict[source]}/best/1.0.0/${wvLayer.id}/default/250m/all/all.xml`
try {
- const describeDomainsResponse = await fetch(describeDomainsUrl)
+ const describeDomainsResponse = await fetch(describeDomainsUrl, fetchOpts)
if (describeDomainsResponse?.ok) {
const describeDomainsText = await describeDomainsResponse?.text?.() || ''
const parser = new xml2js.Parser()
diff --git a/tasks/build-options/validateConfigs.js b/tasks/build-options/validateConfigs.js
index 2d85f39a2..9b2bea872 100644
--- a/tasks/build-options/validateConfigs.js
+++ b/tasks/build-options/validateConfigs.js
@@ -67,7 +67,7 @@ async function main () {
}
async function validateFile (filePath) {
- if (argv.mode === 'verbose') console.warn(`${prog}: Validating ${filePath}`)
+ if (argv.mode === 'verbose') console.trace(`Validating ${filePath}`)
const layerFile = fs.readFileSync(filePath)
const layer = JSON.parse(layerFile)
const valid = validate(layer)
diff --git a/tasks/build-options/validateOptions.js b/tasks/build-options/validateOptions.js
index dde6f808b..99d84bbd3 100644
--- a/tasks/build-options/validateOptions.js
+++ b/tasks/build-options/validateOptions.js
@@ -20,6 +20,12 @@ const options = yargs(hideBin(process.argv))
type: 'string',
description: 'config directory'
})
+ .option('cacheMode', {
+ demandOption: false,
+ alias: 'cm',
+ type: 'string',
+ description: 'Cache mode for fetching data'
+ })
.epilog('Validates and corrects the configuration files.')
const { argv } = options
@@ -29,6 +35,7 @@ if (!argv.optionsFile && !argv.configDir) {
const optionsFile = argv.optionsFile
const configDir = argv.configDir
+const cacheMode = argv.cacheMode
let errorCount = 0
let warningCount = 0
@@ -110,7 +117,7 @@ async function main () {
}
if ('temporal' in layer) {
warn(`[${layerId}] GC Layer temporal values overwritten by Options`)
- layer = await processTemporalLayer(layer, layer.temporal)
+ layer = await processTemporalLayer(layer, layer.temporal, undefined, cacheMode)
}
if (layer.futureTime) {
if ('endDate' in layer) delete layer.endDate
diff --git a/tasks/buildOptions.sh b/tasks/buildOptions.sh
index 675116d49..4760e2f07 100755
--- a/tasks/buildOptions.sh
+++ b/tasks/buildOptions.sh
@@ -8,13 +8,25 @@ DEST_DIR="$BASE/build/options"
SCRIPTS_DIR="$BASE/tasks/build-options"
MODE="default"
+CACHE_MODE="default"
-while getopts ":v" option; do
+while getopts ":vfd" option; do
echo "Option -$option set"
case $option in
v)
- echo "Verbose Mode Activated"
- MODE="verbose";;
+ echo "Verbose Mode Activated"
+ MODE="verbose"
+ ;;
+ f)
+ echo "Force Mode Activated"
+ CACHE_MODE="no-store"
+ ;;
+ d)
+ echo "Debug Mode Activated"
+ export NODE_OPTIONS='--inspect'
+ export NODE_DEBUG=http,http2
+ echo "Open debugger to view results (change '--inspect' to '--inspect-brk' to break on start)"
+ ;;
esac
done
@@ -66,7 +78,8 @@ if [ "$FETCH_GC" ] ; then
`node $SCRIPTS_DIR/getCapabilities.js \
--config "$OPT_DIR/$OPT_SUBDIR/config.json" \
--getcapabilities "$OPT_DIR/$OPT_SUBDIR/gc" \
- --mode "$MODE"`
+ --mode "$MODE" \
+ --cacheMode "$CACHE_MODE"`
# Get metadata for files in layerOrder.json and combine this data into 1 file
if [ "$MODE" == "verbose" ] ; then
@@ -78,7 +91,8 @@ if [ "$FETCH_GC" ] ; then
--features "$BUILD_DIR/features.json" \
--layerOrder "$BUILD_DIR/config/wv.json/layerOrder.json" \
--layerMetadata "$OPT_DIR/$OPT_SUBDIR/layer-metadata/all.json" \
- --mode "$MODE"`
+ --mode "$MODE" \
+ --cacheMode "$CACHE_MODE"`
else
# Validate layers in wv.json with a JSON schema
if [ "$MODE" == "verbose" ] ; then
@@ -87,7 +101,8 @@ else
`node $SCRIPTS_DIR/validateConfigs.js \
--inputDirectory "$SRC_DIR/common/config/wv.json/layers" \
--schemaFile "$BASE/schemas/layer-config.json" \
- --mode "$MODE"`
+ --mode "$MODE" \
+ --cacheMode "$CACHE_MODE"`
if [ -e "$BUILD_DIR/features.json" ] ; then
if [ "$MODE" == "verbose" ] ; then
@@ -105,7 +120,8 @@ else
--config "$BUILD_DIR/config.json" \
--inputDir "$BUILD_DIR/gc" \
--outputDir "$BUILD_DIR/_wmts" \
- --mode "$MODE"`
+ --mode "$MODE" \
+ --cacheMode "$CACHE_MODE"`
fi
# Run processVectorStyles.js and move vectorstyles where we want them
@@ -118,7 +134,8 @@ else
--inputDir "$BUILD_DIR/gc/vectorstyles" \
--wvStylesDir "$SRC_DIR/common/vectorstyles" \
--outputDir "$BUILD_DIR/config/wv.json/vectorstyles" \
- --mode "$MODE"`
+ --mode "$MODE" \
+ --cacheMode "$CACHE_MODE"`
fi
# Run processVectorData.js and move vectordata where we want them
@@ -130,7 +147,8 @@ else
`node $SCRIPTS_DIR/processVectorData.js \
--inputDir "$BUILD_DIR/gc/vectordata" \
--outputDir "$BUILD_DIR/config/wv.json/vectordata" \
- --mode "$MODE"`
+ --mode "$MODE" \
+ --cacheMode "$CACHE_MODE"`
fi
# Run processColormap.js and move colormaps where we want them
@@ -147,7 +165,8 @@ else
--inputDir "$BUILD_DIR/colormaps" \
--outputDir "$BUILD_DIR/config/palettes" \
--layersDir "$BUILD_DIR/_wmts" \
- --mode "$MODE"`
+ --mode "$MODE" \
+ --cacheMode "$CACHE_MODE"`
fi
# Throw error if no categoryGroupOrder.json file present
@@ -156,7 +175,8 @@ else
`node $SCRIPTS_DIR/generateCategoryGroupOrder.js \
--inputDir "$SRC_DIR/common/config/wv.json/categories/" \
--outputDir "$SRC_DIR/common/config/wv.json/" \
- --mode "$MODE"`
+ --mode "$MODE" \
+ --cacheMode "$CACHE_MODE"`
fi
if [ -e "$OPT_DIR/$OPT_SUBDIR/layer-metadata/all.json" ] ; then
@@ -186,7 +206,8 @@ else
`node $SCRIPTS_DIR/mergeConfigWithWMTS.js \
--inputDir "$BUILD_DIR/_wmts" \
--outputFile "$DEST_DIR/config/wv.json" \
- --mode "$MODE"`
+ --mode "$MODE" \
+ --cacheMode "$CACHE_MODE"`
# Copy brand files from build to dest
if [ "$MODE" == "verbose" ] ; then
@@ -203,7 +224,8 @@ else
`node $SCRIPTS_DIR/validateOptions.js \
--optionsFile "$BUILD_DIR/config.json" \
--configDir "$DEST_DIR/config" \
- --mode "$MODE"`
+ --mode "$MODE" \
+ --cacheMode "$CACHE_MODE"`
# Fetch preview images from WV Snapshots for any layers which they are missing
if [ "$MODE" == "verbose" ] ; then
@@ -213,6 +235,7 @@ else
--wvJsonFile "$DEST_DIR/config/wv.json" \
--overridesFile "$OPT_DIR/common/previewLayerOverrides.json" \
--featuresFile "$BUILD_DIR/features.json" \
- --mode "$MODE"`
+ --mode "$MODE" \
+ --cacheMode "$CACHE_MODE"`
fi
exit 0
diff --git a/web/images/layers/previews/antarctic/OrbitTracks_Sentinel-1C_Ascending.jpg b/web/images/layers/previews/antarctic/OrbitTracks_Sentinel-1C_Ascending.jpg
deleted file mode 100644
index 7d9c05ee4..000000000
--- a/web/images/layers/previews/antarctic/OrbitTracks_Sentinel-1C_Ascending.jpg
+++ /dev/null
@@ -1,6 +0,0 @@
-<?xml version='1.0' encoding="UTF-8" standalone="no" ?>
-<ServiceExceptionReport version="1.3.0" xmlns="http://www.opengis.net/ogc" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.opengis.net/ogc http://schemas.opengis.net/wms/1.3.0/exceptions_1_3_0.xsd">
-<ServiceException code="LayerNotDefined">
-msWMSLoadGetMapParams(): WMS server error. Invalid layer(s) given in the LAYERS parameter. A layer might be disabled for this request. Check wms/ows_enable_request settings.
-</ServiceException>
-</ServiceExceptionReport>
diff --git a/web/images/layers/previews/antarctic/OrbitTracks_Sentinel-1C_Descending.jpg b/web/images/layers/previews/antarctic/OrbitTracks_Sentinel-1C_Descending.jpg
deleted file mode 100644
index 5a3ff655b..000000000
Binary files a/web/images/layers/previews/antarctic/OrbitTracks_Sentinel-1C_Descending.jpg and /dev/null differ
diff --git a/web/images/layers/previews/arctic/OrbitTracks_Sentinel-1C_Ascending.jpg b/web/images/layers/previews/arctic/OrbitTracks_Sentinel-1C_Ascending.jpg
deleted file mode 100644
index f5a3c60a7..000000000
Binary files a/web/images/layers/previews/arctic/OrbitTracks_Sentinel-1C_Ascending.jpg and /dev/null differ
diff --git a/web/images/layers/previews/arctic/OrbitTracks_Sentinel-1C_Descending.jpg b/web/images/layers/previews/arctic/OrbitTracks_Sentinel-1C_Descending.jpg
deleted file mode 100644
index 3017ec811..000000000
Binary files a/web/images/layers/previews/arctic/OrbitTracks_Sentinel-1C_Descending.jpg and /dev/null differ