Skip to content

Interactive interface for browsing global, full-resolution satellite imagery

continue bug fixing

ZachTRice135e49ad2f90
diff --git a/tasks/build-options/extractConfigFromWMTS.js b/tasks/build-options/extractConfigFromWMTS.js
index ac0ea3882..793f9a5dd 100644
--- a/tasks/build-options/extractConfigFromWMTS.js
+++ b/tasks/build-options/extractConfigFromWMTS.js
@@ -313,6 +313,7 @@ function processMatrixSet (gcMatrixSet) {
matrixHeight: parseInt(tileMatrix.MatrixHeight._text, 10)
})
}
+
wvMatrixSets[ident] = {
id: ident,
maxResolution,
diff --git a/tasks/build-options/fetchPreviewSnapshots.js b/tasks/build-options/fetchPreviewSnapshots.js
index ae1291534..31705903d 100644
--- a/tasks/build-options/fetchPreviewSnapshots.js
+++ b/tasks/build-options/fetchPreviewSnapshots.js
@@ -1,10 +1,10 @@
-const yargs = require('yargs')
+const moment = require('moment')
const axios = require('axios')
-const { promisify } = require('util')
const fs = require('fs')
-const readFileAsync = promisify(fs.readFile)
+const path = require('path')
+const yargs = require('yargs')
-const prog = require('path').basename(__filename)
+const prog = path.basename(__filename)
const options = yargs
.usage('Usage: $0 <wv.json> <overrides_file>')
@@ -37,11 +37,11 @@ const wvJsonFile = argv.wvJsonFile
const overridesFile = argv.overridesFile
const featuresFile = argv.featuresFile
-const badSnapshots = []
let overrideDatesDict = {}
+const badSnapshots = []
let totalSuccessCount = 0
let totalFailureCount = 0
-// const timeFormat = '%Y-%m-%dT%H:%M:%SZ'
+const timeFormat = 'YYYY-MM-DDTHH:mm:ssZ'
let snapshotsUrl = ''
const paramDict = {
base: {
@@ -67,38 +67,32 @@ const paramDict = {
HEIGHT: '512'
}
}
-
-// These layers should not be combined with the reference layer
const standaloneLayers = [
'Graticule_15m',
'Coastlines_15m',
'Reference_Features_15m',
'Reference_Labels_15m'
]
-
const destImgDir = './web/images/layers/previews/'
-
const referenceLayers = {
geographic: 'OSM_Land_Water_Map',
arctic: 'OSM_Land_Water_Map',
antarctic: 'SCAR_Land_Water_Map'
}
-const current = new Date()
-
async function main () {
// Check to see if this feature is enabled in features.json before continuing
- const featuresFileContent = await readFileAsync(featuresFile, { encoding: 'utf-8' })
+ const featuresFileContent = fs.readFileSync(featuresFile, { encoding: 'utf-8' })
const featuresDict = JSON.parse(featuresFileContent)
if (featuresDict.features.previewSnapshots === false) {
throw new Error('previewSnapshots note enabled in features.json')
}
// Allow manual configuration of layer ID to specific date to generate desired preview
- const overridesFileContent = await readFileAsync(overridesFile, { encoding: 'utf-8' })
+ const overridesFileContent = fs.readFileSync(overridesFile, { encoding: 'utf-8' })
overrideDatesDict = JSON.parse(overridesFileContent)
- const wvJsonFileContent = await readFileAsync(wvJsonFile, { encoding: 'utf-8' })
+ const wvJsonFileContent = fs.readFileSync(wvJsonFile, { encoding: 'utf-8' })
const wvJsonDict = JSON.parse(wvJsonFileContent)
const layers = wvJsonDict.layers
snapshotsUrl = wvJsonDict.features.imageDownload.url
@@ -107,15 +101,8 @@ async function main () {
throw new Error(`${prog}: Layer preview fetching disabled. Exiting.`)
}
- const promises = []
for (const layer of Object.values(layers)) {
- promises.push(getSnapshots(layer))
- }
-
- try {
- await Promise.all(promises)
- } catch (error) {
- console.error(`${error}`)
+ await getSnapshots(layer)
}
if (badSnapshots.length > 0) {
@@ -137,79 +124,73 @@ async function main () {
}
}
-function trackBadSnapshots (layerId, projection, request, imgPath) {
+async function trackBadSnapshots (layerId, projection, request, imgFile, badSnapshots) {
const arcticBadSize = 9949
const antarcticBadSize = 4060
const geographicBadSize = 12088
-
- fs.stat(imgPath, (error, stats) => {
- if (error) {
- console.error(error)
- return
- }
-
- const size = stats.size
- if ([arcticBadSize, antarcticBadSize, geographicBadSize].includes(size)) {
- badSnapshots.push({
- id: layerId,
- projection,
- url: request.url
- })
- }
- })
+ const size = imgFile.tell()
+
+ if (size === geographicBadSize || size === arcticBadSize || size === antarcticBadSize) {
+ badSnapshots.push({
+ id: layerId,
+ projection,
+ url: request.url
+ })
+ }
}
-function getBestDate (projection, period, dateRanges) {
+async function getBestDate (projection, period, dateRanges) {
const lastRange = dateRanges[dateRanges.length - 1]
const startDate = lastRange.startDate
const endDate = lastRange.endDate
- const parsedStartDate = new Date(startDate)
- const parsedEndDate = new Date(endDate)
- const pYear = parsedEndDate.getFullYear()
- const pMonth = parsedEndDate.getMonth() + 1 // months are 0-indexed in JavaScript
+ const parsedStartDate = moment(startDate, timeFormat)
+ const parsedEndDate = moment(endDate, timeFormat)
+ const pYear = parsedEndDate.year()
+ const pMonth = parsedEndDate.month()
let interval = parseInt(lastRange.dateInterval, 10)
let alteredDate = null
- // Handle daily layers
if (period === 'daily') {
- // Go back a few more days for single day layers since something
- // too recent may not be processed yet
if (interval === 1) {
interval = 3
}
- alteredDate = new Date(parsedEndDate - interval * 24 * 60 * 60 * 1000)
+ alteredDate = parsedEndDate.subtract(interval, 'days')
+ } else {
+ alteredDate = parsedEndDate.subtract(interval, 'months')
}
// Choose a good daylight month for arctic
if (projection === 'arctic' && ![4, 5, 6, 7, 8, 9].includes(pMonth)) {
- if (pYear === current.getFullYear() && current.getMonth() + 1 < 6) {
- alteredDate = new Date(pYear - 1, 5, 1)
+ const currentYear = moment().year()
+ const currentMonth = moment().month() + 1
+ if (pYear === currentYear && currentMonth < 6) {
+ alteredDate = moment({ year: pYear - 1, month: 5, day: 1 })
} else {
- alteredDate = new Date(pYear, 5, 1)
+ alteredDate = moment({ year: pYear, month: 5, day: 1 })
}
}
// Choose a good daylight month for antarctic
if (projection === 'antarctic' && ![10, 11, 12, 1, 2].includes(pMonth)) {
- // TODO handle "bad" months for antarctic
- alteredDate = new Date(pYear, 11, 1)
+ alteredDate = moment({ year: pYear, month: 11, day: 1 })
}
// Make sure modified date isn't out of layer date range
- if (alteredDate && alteredDate >= parsedStartDate) {
- return alteredDate.toISOString().slice(0, -5) + 'Z'
+ if (alteredDate && alteredDate.isSameOrAfter(parsedStartDate)) {
+ return alteredDate.format(timeFormat)
}
+
return endDate
}
-function getTimeParam (projection, layerId, layer, params) {
+async function getTimeParam (projection, layerId, layer, params) {
// Only include TIME param for temporal layers
const dateRanges = layer.dateRanges
const startDate = layer.startDate
const period = layer.period
if (dateRanges) {
- params.TIME = getBestDate(projection, period, dateRanges)
+ params.TIME = await getBestDate(projection, period, dateRanges)
} else if (startDate) {
params.TIME = startDate
}
@@ -237,7 +218,7 @@ async function getSnapshots (layer) {
}
const params = { ...paramDict.base, ...paramDict[projection] }
- getTimeParam(projection, wvLayerId, layer, params)
+ await getTimeParam(projection, wvLayerId, layer, params)
if (gibsLayerId !== referenceLayer && !standaloneLayers.includes(gibsLayerId)) {
params.LAYERS = `${referenceLayer},${gibsLayerId}`
@@ -262,7 +243,7 @@ async function getSnapshots (layer) {
totalSuccessCount += 1
await fs.promises.writeFile(destFileName, imageReq.data, { flag: 'wx' })
if (gibsLayerId === referenceLayers[projection]) continue
- trackBadSnapshots(wvLayerId, projection, imageReq, imageReq.data)
+ await trackBadSnapshots(wvLayerId, projection, imageReq, imageReq.data)
} else {
totalFailureCount += 1
}
diff --git a/tasks/build-options/mergeConfigWithWMTS.js b/tasks/build-options/mergeConfigWithWMTS.js
index ef82ba5ac..5f1b28e66 100644
--- a/tasks/build-options/mergeConfigWithWMTS.js
+++ b/tasks/build-options/mergeConfigWithWMTS.js
@@ -57,8 +57,8 @@ async function processFiles () {
try {
const data = fs.readFileSync(path.join(inputDir, file), 'utf-8')
const jsonData = JSON.parse(data)
- newConf.layers = wmtsDictMerge(newConf.layers, jsonData, outputData)
- newConf.sources = dictMerge(newConf.sources, jsonData.sources, outputData.sources)
+ newConf.layers = await wmtsDictMerge(newConf.layers, jsonData, outputData)
+ newConf.sources = await dictMerge(newConf.sources, jsonData.sources, outputData.sources)
} catch (error) {
throw new Error(`ERROR: ${path.join(inputDir, file)}: ${error.message}`)
}
@@ -83,8 +83,9 @@ async function processFiles () {
// return target
// }
-function wmtsDictMerge (target, obj, conf) {
+async function wmtsDictMerge (target, obj, conf) {
obj = obj.layers
+
// conf = conf.layers
if (typeof obj !== 'object') {
diff --git a/tasks/build-options/util.js b/tasks/build-options/util.js
index b505a9ef3..fcc7c64d9 100644
--- a/tasks/build-options/util.js
+++ b/tasks/build-options/util.js
@@ -1,3 +1,21 @@
+async function deepCopy (obj) {
+ if (typeof obj !== 'object') {
+ return obj
+ }
+ let copy
+ if (Array.isArray(obj)) {
+ copy = []
+ } else {
+ copy = {}
+ }
+ for (const key in obj) {
+ if (Object.prototype.hasOwnProperty.call(obj, key)) {
+ copy[key] = await deepCopy(obj[key])
+ }
+ }
+ return copy
+}
+
async function dictMerge (target, ...args) {
// Merge multiple objects
if (args.length > 1) {
@@ -21,7 +39,7 @@ async function dictMerge (target, ...args) {
}
await dictMerge(target[k], v)
} else {
- target[k] = Object.assign({}, v)
+ target[k] = await deepCopy(v)
}
}
return target
diff --git a/tasks/build-options/validateOptions.js b/tasks/build-options/validateOptions.js
index ef9270eb3..c6ba67797 100644
--- a/tasks/build-options/validateOptions.js
+++ b/tasks/build-options/validateOptions.js
@@ -1,6 +1,8 @@
const fs = require('fs')
const path = require('path')
const yargs = require('yargs')
+const moment = require('moment')
+const { processTemporalLayer } = require('./processTemporalLayer')
const prog = path.basename(__filename)
const options = yargs
@@ -47,172 +49,169 @@ function error (message) {
}
function warn (message) {
- console.error(`${prog}: WARN: ${message}`)
+ console.error(`${prog}: WARN: ${message}`)
warningCount += 1
}
function removeLayer (wv, layerId) {
removedLayers.push(layerId)
delete wv.layers[layerId]
- if (wv.layerOrder.includes(layerId)) wv.layerOrder.splice(wv.layerOrder.indexOf(layerId), 1)
+ if (wv.layerOrder.includes(layerId)) {
+ wv.layerOrder = wv.layerOrder.filter((id) => id !== layerId)
+ }
}
function isDateTimeFormat (input) {
try {
- Date.parse(input)
+ moment(input, 'YYYY-MM-DD HH:mm:ss')
return true
} catch (e) {
return false
}
}
-if (tolerant) console.warn('Validation enforcement disabled')
+if (tolerant) warn('Validation enforcement disabled')
-let startDate = new Date(Number.MAX_SAFE_INTEGER)
+let startDate = moment.max()
async function main () {
for (const layerId of Object.keys(wv.layers)) {
let layer = wv.layers[layerId]
-
- if (layerId !== layer.get('id')) {
- error(`[${layerId}] layer id does not match id of ${layer.get('id')}`)
+ if (layerId !== layer.id) {
+ error(`[${layerId}] layer id does not match id of ${layer.id}`)
}
if (!wv.layerOrder.includes(layerId)) {
- if (opt.get('layerOrderExceptions', []).includes(layerId)) {
+ if (opt.layerOrderExceptions && opt.layerOrderExceptions.includes(layerId)) {
removeLayer(wv, layerId)
continue
- } else if (tolerant || opt.get('ignoreLayerOrder', false)) {
+ } else if (tolerant || opt.ignoreLayerOrder) {
wv.layerOrder.push(layerId)
} else {
removeLayer(wv, layerId)
continue
}
}
- if (!layer.projections || layer.projections.length === 0) {
- error(`[${layerId}] No projections defined or not found in GC documents`)
- removeLayer(wv, layerId)
- continue
- }
- if (!Object.prototype.hasOwnProperty.call(layer, 'type')) {
- error(
- `[${layerId}] No type defined. Possible to be expecting configuration via GC document but was not found`
- )
- removeLayer(wv, layerId)
- continue
- }
- if (Object.prototype.hasOwnProperty.call(layer, 'palette') && !Object.prototype.hasOwnProperty.call(layer.palette, 'id')) {
- error(`[${layerId}] No palette definition`)
- } else if (Object.prototype.hasOwnProperty.call(layer, 'palette')) {
- const paletteId = layer.palette.id
- if (!fs.existsSync(path.join(configDir, 'palettes', `${paletteId}.json`))) {
- error(`[${layerId}] Unknown palette: ${paletteId}`)
- delete layer.palette
- }
- }
- if (Object.prototype.hasOwnProperty.call(layer, 'vectorStyle') && !Object.prototype.hasOwnProperty.call(layer.vectorStyle, 'id')) {
+ // if (!layer.projections || !layer.projections.length) {
+ // error(`[${layerId}] No projections defined or not found in GC documents`)
+ // removeLayer(wv, layerId)
+ // continue
+ // }
+ // if (!layer.type) {
+ // error(`[${layerId}] No type defined. Possible to be expecting configuration via GC document but was not found`)
+ // removeLayer(wv, layerId)
+ // continue
+ // }
+ // if (layer.palette && !layer.palette.id) {
+ // error(`[${layerId}] No palette definition`)
+ // } else if (layer.palette) {
+ // const paletteId = layer.palette.id
+ // if (!fs.existsSync(path.join(configDir, `palettes/${paletteId}.json`))) {
+ // error(`[${layerId}] palette ${paletteId} not found in palettes folder`)
+ // }
+ // }
+ if (layer.vectorStyle && !layer.vectorStyle.id) {
error(`[${layerId}] No vectorStyle definition`)
}
- if (!Object.prototype.hasOwnProperty.call(layer, 'group') && opt.get('warnOnUnexpectedLayer')) {
+ if (!layer.group && opt.warnOnUnexpectedLayer) {
error(`[${layerId}] Possible unexpected layer, no group defined`)
removeLayer(wv, layerId)
continue
- } else if (!Object.prototype.hasOwnProperty.call(layer, 'group')) {
+ } else if (!layer.group) {
removeLayer(wv, layerId)
continue
}
- for (const projId of Object.keys(layer.projections)) {
- const projection = layer.projections[projId]
- if (Object.prototype.hasOwnProperty.call(projection, 'matrixSet')) {
- const source = projection.source
- const matrixSet = projection.matrixSet
- if (!Object.prototype.hasOwnProperty.call(wv.sources, source)) {
- error(`[${layerId}:${projId}] Invalid source: ${source}`)
- delete layer.projections[projId]
- } else if (!Object.prototype.hasOwnProperty.call(wv.sources[source], 'matrixSets')) {
- error(`[${layerId}:${projId}] No matrix sets for projection`)
- delete layer.projections[projId]
- } else if (!wv.sources[source].matrixSets.includes(matrixSet)) {
- error(`[${layerId}:${projId}] Invalid matrix set: ${matrixSet}`)
- delete layer.projections[projId]
- }
- }
- }
- if (Object.prototype.hasOwnProperty.call(layer, 'temporal')) {
- console.warn(`[${layerId}] GC Layer temporal values overwritten by Options`)
- layer = processTemporal(layer, layer.temporal)
- }
- if (layer.get('futureTime')) {
- if (Object.prototype.hasOwnProperty.call(layer, 'endDate')) {
- delete layer.endDate
- }
- }
- if (layer.get('futureTime', false)) {
+ // for (const projId of Object.keys(layer.projections)) {
+ // const projection = layer.projections[projId]
+ // if (projection.matrixSet) {
+ // const source = projection.source
+ // const matrixSet = projection.matrixSet
+ // if (!wv.sources[source]) {
+ // error(`[${layerId}:${projId}] Invalid source: ${source}`)
+ // delete layer.projections[projId]
+ // } else if (!wv.sources[source].matrixSets) {
+ // error(`[${layerId}:${projId}] No matrix sets for projection`)
+ // delete layer.projections[projId]
+ // } else if (!wv.sources[source].matrixSets[matrixSet]) {
+ // error(`[${layerId}:${projId}] Invalid matrix set: ${matrixSet}`)
+ // delete layer.projections[projId]
+ // }
+ // }
+ // }
+ if ('temporal' in layer) {
+ warn(`[${layerId}] GC Layer temporal values overwritten by Options`)
+ layer = await processTemporalLayer(layer, layer.temporal)
+ }
+ if (layer.futureTime) {
+ if ('endDate' in layer) delete layer.endDate
+ }
+
+ if (layer.futureTime) {
// do nothing
} else {
- if (Object.prototype.hasOwnProperty.call(layer, 'endDate') && layer.get('ongoing', true)) {
- delete layer.endDate
- }
+ if ('endDate' in layer && layer.ongoing) delete layer.endDate
}
- if (Object.prototype.hasOwnProperty.call(layer, 'startDate')) {
+ if ('startDate' in layer) {
const startTime = layer.startDate.replace('T', ' ').replace('Z', '')
let d
if (isDateTimeFormat(startTime)) {
- d = new Date(startTime)
+ d = moment(startTime, 'YYYY-MM-DD HH:mm:ss')
} else {
- d = new Date(layer.startDate)
+ d = moment(layer.startDate, 'YYYY-MM-DD')
}
- startDate = Math.min(startDate, d)
+ startDate = moment.min(startDate, d)
}
}
- if (startDate.getTime() !== Number.MAX_VALUE) {
- wv.startDate = startDate.toISOString()
+ if (startDate.isValid()) {
+ wv.startDate = startDate.format('YYYY-MM-DD') + 'T' + startDate.format('HH:mm:ss') + 'Z'
}
for (const layerId of wv.layerOrder) {
- if (!Object.prototype.hasOwnProperty.call(wv.layers, layerId)) {
+ if (!wv.layers[layerId]) {
error(`[${layerId}] In layer order but no definition`)
}
}
const startingLayers = []
for (const startingLayer of wv.defaults.startingLayers) {
- if (!Object.prototype.hasOwnProperty.call(wv.layers, startingLayer.id)) {
+ if (!wv.layers[startingLayer.id]) {
error(`Invalid starting layer: ${startingLayer.id}`)
} else {
startingLayers.push(startingLayer)
}
}
wv.defaults.startingLayers = startingLayers
- wv.buildDate = Date.now()
-
- for (const projection of Object.keys(wv.naturalEvents.layers)) {
- for (const eventType of Object.keys(wv.naturalEvents.layers[projection])) {
- for (const layerObject of wv.naturalEvents.layers[projection][eventType]) {
- if (!Object.prototype.hasOwnProperty.call(wv.layers, layerObject[0])) {
- error(
- `The ${layerObject[0]} layer in the Natural events ${projection} ${eventType} config does not have a matching ID in the layer config`
- )
+ wv.buildDate = Math.round(Date.now() / 1000)
+
+ for (const [projection, projectionValue] of Object.entries(wv.naturalEvents.layers)) {
+ for (const [eventType, eventTypeLayerList] of Object.entries(projectionValue)) {
+ for (const layerObject of eventTypeLayerList) {
+ if (!wv.layers[layerObject[0]]) {
+ error(`The ${layerObject[0]} layer in the Natural events ${projection} ${eventType} config does not have a matching ID in the layer config`)
}
}
}
}
for (const measurement of Object.values(wv.measurements)) {
- if (!Object.prototype.hasOwnProperty.call(measurement, 'sources')) continue
+ if (!measurement.sources) continue
for (const source of Object.values(measurement.sources)) {
- if (!Object.prototype.hasOwnProperty.call(source, 'settings')) continue
+ if (!source.settings) continue
for (const setting of source.settings) {
- if (!Object.prototype.hasOwnProperty.call(wv.layers, setting)) {
- error(
- `In measurement ${measurement.id}, source ${source.id}, layer not found: ${setting}`
- )
+ if (!wv.layers[setting]) {
+ error(`In measurement ${measurement.id}, source ${source.id}, layer not found: ${setting}`)
}
}
}
}
- console.log(`${prog}: ${errorCount} error(s), ${warningCount} warning(s), ${removeCount} removed`)
+ const removeCount = removedLayers.length
+ console.warn(`${prog}: ${errorCount} error(s), ${warningCount} warning(s), ${removeCount} removed`)
+ if (removeCount > 0) {
+ console.warn(removedLayers)
+ }
+
+ fs.writeFileSync(mainConfigFile, JSON.stringify(wv, null, 2))
if (errorCount > 0) {
throw new Error(`${prog}: Error: ${errorCount} errors occured`)
diff --git a/tasks/buildOptions.sh b/tasks/buildOptions.sh
index 43ce6a8ed..8896f9b28 100755
--- a/tasks/buildOptions.sh
+++ b/tasks/buildOptions.sh
@@ -100,7 +100,6 @@ if [ -e "$BUILD_DIR/gc/vectordata" ] ; then
--outputDir "$BUILD_DIR/config/wv.json/vectordata"`
fi
-# TODO: fix erroneous data in files
# Run processColormap.js and move colormaps where we want them
if [ -e "$BUILD_DIR/colormaps" ] ; then
mkdir -p "$BUILD_DIR"/config/palettes
@@ -141,24 +140,30 @@ for config in $configs; do
esac
done
-# # Run mergeConfigWithWMTS.js to merge layer metadata from WMTS GC with worldview layer configs into wv.json
+# TODO: Missing the following variables in wv.json layers
+# type, format, period, startDate, endDate, dateRanges, projections,
+# palette, dataCenter,
+# this means _wmts .json files are not getting merged into wv.json properly
+
+# Run mergeConfigWithWMTS.js to merge layer metadata from WMTS GC with worldview layer configs into wv.json
`node $SCRIPTS_DIR/mergeConfigWithWMTS.js \
--inputDir "$BUILD_DIR/_wmts" \
--outputFile "$DEST_DIR/config/wv.json"`
# Copy brand files from build to dest
-# cp -r "$BUILD_DIR/brand" "$DEST_DIR"
-# cp "$BUILD_DIR/brand.json" "$DEST_DIR"
-
-# # Validate the options build
-# `node $SCRIPTS_DIR/validateOptions.js" \
-# --inputFile "$BUILD_DIR/config.json" \
-# --outputDir "$DEST_DIR/config`
-
-# # Fetch preview images from WV Snapshots for any layers which they are missing
-# `node $SCRIPTS_DIR/fetchPreviewSnapshots.js" \
-# --inputFile "$DEST_DIR/config/wv.json" \
-# --outputFile "$OPT_DIR/common/previewLayerOverrides.json" \
+cp -r "$BUILD_DIR/brand" "$DEST_DIR"
+cp "$BUILD_DIR/brand.json" "$DEST_DIR"
+
+
+# Validate the options build
+# `node $SCRIPTS_DIR/validateOptions.js \
+# --optionsFile "$BUILD_DIR/config.json" \
+# --configDir "$DEST_DIR/config"`
+
+# Fetch preview images from WV Snapshots for any layers which they are missing
+# `node $SCRIPTS_DIR/fetchPreviewSnapshots.js \
+# --wvJsonFile "$DEST_DIR/config/wv.json" \
+# --overridesFile "$OPT_DIR/common/previewLayerOverrides.json" \
# --featuresFile "$BUILD_DIR/features.json"`
exit 0