Interactive interface for browsing global, full-resolution satellite imagery
continue bug fixing
ZachTRice135e49ad2f90
diff --git a/tasks/build-options/extractConfigFromWMTS.js b/tasks/build-options/extractConfigFromWMTS.jsindex ac0ea3882..793f9a5dd 100644--- a/tasks/build-options/extractConfigFromWMTS.js+++ b/tasks/build-options/extractConfigFromWMTS.js@@ -313,6 +313,7 @@ function processMatrixSet (gcMatrixSet) {matrixHeight: parseInt(tileMatrix.MatrixHeight._text, 10)})}+wvMatrixSets[ident] = {id: ident,maxResolution,diff --git a/tasks/build-options/fetchPreviewSnapshots.js b/tasks/build-options/fetchPreviewSnapshots.jsindex ae1291534..31705903d 100644--- a/tasks/build-options/fetchPreviewSnapshots.js+++ b/tasks/build-options/fetchPreviewSnapshots.js@@ -1,10 +1,10 @@-const yargs = require('yargs')+const moment = require('moment')const axios = require('axios')-const { promisify } = require('util')const fs = require('fs')-const readFileAsync = promisify(fs.readFile)+const path = require('path')+const yargs = require('yargs')-const prog = require('path').basename(__filename)+const prog = path.basename(__filename)const options = yargs.usage('Usage: $0 <wv.json> <overrides_file>')@@ -37,11 +37,11 @@ const wvJsonFile = argv.wvJsonFileconst overridesFile = argv.overridesFileconst featuresFile = argv.featuresFile-const badSnapshots = []let overrideDatesDict = {}+const badSnapshots = []let totalSuccessCount = 0let totalFailureCount = 0-// const timeFormat = '%Y-%m-%dT%H:%M:%SZ'+const timeFormat = 'YYYY-MM-DDTHH:mm:ssZ'let snapshotsUrl = ''const paramDict = {base: {@@ -67,38 +67,32 @@ const paramDict = {HEIGHT: '512'}}--// These layers should not be combined with the reference layerconst standaloneLayers = ['Graticule_15m','Coastlines_15m','Reference_Features_15m','Reference_Labels_15m']-const destImgDir = './web/images/layers/previews/'-const referenceLayers = {geographic: 'OSM_Land_Water_Map',arctic: 'OSM_Land_Water_Map',antarctic: 'SCAR_Land_Water_Map'}-const current = new Date()-async function main () {// Check to see if this feature is enabled in features.json before continuing- const featuresFileContent = await readFileAsync(featuresFile, { encoding: 'utf-8' })+ const featuresFileContent = fs.readFileSync(featuresFile, { encoding: 'utf-8' })const featuresDict = JSON.parse(featuresFileContent)if (featuresDict.features.previewSnapshots === false) {throw new Error('previewSnapshots note enabled in features.json')}// Allow manual configuration of layer ID to specific date to generate desired preview- const overridesFileContent = await readFileAsync(overridesFile, { encoding: 'utf-8' })+ const overridesFileContent = fs.readFileSync(overridesFile, { encoding: 'utf-8' })overrideDatesDict = JSON.parse(overridesFileContent)- const wvJsonFileContent = await readFileAsync(wvJsonFile, { encoding: 'utf-8' })+ const wvJsonFileContent = fs.readFileSync(wvJsonFile, { encoding: 'utf-8' })const wvJsonDict = JSON.parse(wvJsonFileContent)const layers = wvJsonDict.layerssnapshotsUrl = wvJsonDict.features.imageDownload.url@@ -107,15 +101,8 @@ async function main () {throw new Error(`${prog}: Layer preview fetching disabled. Exiting.`)}- const promises = []for (const layer of Object.values(layers)) {- promises.push(getSnapshots(layer))- }-- try {- await Promise.all(promises)- } catch (error) {- console.error(`${error}`)+ await getSnapshots(layer)}if (badSnapshots.length > 0) {@@ -137,79 +124,73 @@ async function main () {}}-function trackBadSnapshots (layerId, projection, request, imgPath) {+async function trackBadSnapshots (layerId, projection, request, imgFile, badSnapshots) {const arcticBadSize = 9949const antarcticBadSize = 4060const geographicBadSize = 12088-- fs.stat(imgPath, (error, stats) => {- if (error) {- console.error(error)- return- }-- const size = stats.size- if ([arcticBadSize, antarcticBadSize, geographicBadSize].includes(size)) {- badSnapshots.push({- id: layerId,- projection,- url: request.url- })- }- })+ const size = imgFile.tell()++ if (size === geographicBadSize || size === arcticBadSize || size === antarcticBadSize) {+ badSnapshots.push({+ id: layerId,+ projection,+ url: request.url+ })+ }}-function getBestDate (projection, period, dateRanges) {+async function getBestDate (projection, period, dateRanges) {const lastRange = dateRanges[dateRanges.length - 1]const startDate = lastRange.startDateconst endDate = lastRange.endDate- const parsedStartDate = new Date(startDate)- const parsedEndDate = new Date(endDate)- const pYear = parsedEndDate.getFullYear()- const pMonth = parsedEndDate.getMonth() + 1 // months are 0-indexed in JavaScript+ const parsedStartDate = moment(startDate, timeFormat)+ const parsedEndDate = moment(endDate, timeFormat)+ const pYear = parsedEndDate.year()+ const pMonth = parsedEndDate.month()let interval = parseInt(lastRange.dateInterval, 10)let alteredDate = null- // Handle daily layersif (period === 'daily') {- // Go back a few more days for single day layers since something- // too recent may not be processed yetif (interval === 1) {interval = 3}- alteredDate = new Date(parsedEndDate - interval * 24 * 60 * 60 * 1000)+ alteredDate = parsedEndDate.subtract(interval, 'days')+ } else {+ alteredDate = parsedEndDate.subtract(interval, 'months')}// Choose a good daylight month for arcticif (projection === 'arctic' && ![4, 5, 6, 7, 8, 9].includes(pMonth)) {- if (pYear === current.getFullYear() && current.getMonth() + 1 < 6) {- alteredDate = new Date(pYear - 1, 5, 1)+ const currentYear = moment().year()+ const currentMonth = moment().month() + 1+ if (pYear === currentYear && currentMonth < 6) {+ alteredDate = moment({ year: pYear - 1, month: 5, day: 1 })} else {- alteredDate = new Date(pYear, 5, 1)+ alteredDate = moment({ year: pYear, month: 5, day: 1 })}}// Choose a good daylight month for antarcticif (projection === 'antarctic' && ![10, 11, 12, 1, 2].includes(pMonth)) {- // TODO handle "bad" months for antarctic- alteredDate = new Date(pYear, 11, 1)+ alteredDate = moment({ year: pYear, month: 11, day: 1 })}// Make sure modified date isn't out of layer date range- if (alteredDate && alteredDate >= parsedStartDate) {- return alteredDate.toISOString().slice(0, -5) + 'Z'+ if (alteredDate && alteredDate.isSameOrAfter(parsedStartDate)) {+ return alteredDate.format(timeFormat)}+return endDate}-function getTimeParam (projection, layerId, layer, params) {+async function getTimeParam (projection, layerId, layer, params) {// Only include TIME param for temporal layersconst dateRanges = layer.dateRangesconst startDate = layer.startDateconst period = layer.periodif (dateRanges) {- params.TIME = getBestDate(projection, period, dateRanges)+ params.TIME = await getBestDate(projection, period, dateRanges)} else if (startDate) {params.TIME = startDate}@@ -237,7 +218,7 @@ async function getSnapshots (layer) {}const params = { ...paramDict.base, ...paramDict[projection] }- getTimeParam(projection, wvLayerId, layer, params)+ await getTimeParam(projection, wvLayerId, layer, params)if (gibsLayerId !== referenceLayer && !standaloneLayers.includes(gibsLayerId)) {params.LAYERS = `${referenceLayer},${gibsLayerId}`@@ -262,7 +243,7 @@ async function getSnapshots (layer) {totalSuccessCount += 1await fs.promises.writeFile(destFileName, imageReq.data, { flag: 'wx' })if (gibsLayerId === referenceLayers[projection]) continue- trackBadSnapshots(wvLayerId, projection, imageReq, imageReq.data)+ await trackBadSnapshots(wvLayerId, projection, imageReq, imageReq.data)} else {totalFailureCount += 1}diff --git a/tasks/build-options/mergeConfigWithWMTS.js b/tasks/build-options/mergeConfigWithWMTS.jsindex ef82ba5ac..5f1b28e66 100644--- a/tasks/build-options/mergeConfigWithWMTS.js+++ b/tasks/build-options/mergeConfigWithWMTS.js@@ -57,8 +57,8 @@ async function processFiles () {try {const data = fs.readFileSync(path.join(inputDir, file), 'utf-8')const jsonData = JSON.parse(data)- newConf.layers = wmtsDictMerge(newConf.layers, jsonData, outputData)- newConf.sources = dictMerge(newConf.sources, jsonData.sources, outputData.sources)+ newConf.layers = await wmtsDictMerge(newConf.layers, jsonData, outputData)+ newConf.sources = await dictMerge(newConf.sources, jsonData.sources, outputData.sources)} catch (error) {throw new Error(`ERROR: ${path.join(inputDir, file)}: ${error.message}`)}@@ -83,8 +83,9 @@ async function processFiles () {// return target// }-function wmtsDictMerge (target, obj, conf) {+async function wmtsDictMerge (target, obj, conf) {obj = obj.layers+// conf = conf.layersif (typeof obj !== 'object') {diff --git a/tasks/build-options/util.js b/tasks/build-options/util.jsindex b505a9ef3..fcc7c64d9 100644--- a/tasks/build-options/util.js+++ b/tasks/build-options/util.js@@ -1,3 +1,21 @@+async function deepCopy (obj) {+ if (typeof obj !== 'object') {+ return obj+ }+ let copy+ if (Array.isArray(obj)) {+ copy = []+ } else {+ copy = {}+ }+ for (const key in obj) {+ if (Object.prototype.hasOwnProperty.call(obj, key)) {+ copy[key] = await deepCopy(obj[key])+ }+ }+ return copy+}+async function dictMerge (target, ...args) {// Merge multiple objectsif (args.length > 1) {@@ -21,7 +39,7 @@ async function dictMerge (target, ...args) {}await dictMerge(target[k], v)} else {- target[k] = Object.assign({}, v)+ target[k] = await deepCopy(v)}}return targetdiff --git a/tasks/build-options/validateOptions.js b/tasks/build-options/validateOptions.jsindex ef9270eb3..c6ba67797 100644--- a/tasks/build-options/validateOptions.js+++ b/tasks/build-options/validateOptions.js@@ -1,6 +1,8 @@const fs = require('fs')const path = require('path')const yargs = require('yargs')+const moment = require('moment')+const { processTemporalLayer } = require('./processTemporalLayer')const prog = path.basename(__filename)const options = yargs@@ -47,172 +49,169 @@ function error (message) {}function warn (message) {- console.error(`${prog}: WARN: ${message}`)+ console.error(`${prog}: WARN: ${message}`)warningCount += 1}function removeLayer (wv, layerId) {removedLayers.push(layerId)delete wv.layers[layerId]- if (wv.layerOrder.includes(layerId)) wv.layerOrder.splice(wv.layerOrder.indexOf(layerId), 1)+ if (wv.layerOrder.includes(layerId)) {+ wv.layerOrder = wv.layerOrder.filter((id) => id !== layerId)+ }}function isDateTimeFormat (input) {try {- Date.parse(input)+ moment(input, 'YYYY-MM-DD HH:mm:ss')return true} catch (e) {return false}}-if (tolerant) console.warn('Validation enforcement disabled')+if (tolerant) warn('Validation enforcement disabled')-let startDate = new Date(Number.MAX_SAFE_INTEGER)+let startDate = moment.max()async function main () {for (const layerId of Object.keys(wv.layers)) {let layer = wv.layers[layerId]-- if (layerId !== layer.get('id')) {- error(`[${layerId}] layer id does not match id of ${layer.get('id')}`)+ if (layerId !== layer.id) {+ error(`[${layerId}] layer id does not match id of ${layer.id}`)}if (!wv.layerOrder.includes(layerId)) {- if (opt.get('layerOrderExceptions', []).includes(layerId)) {+ if (opt.layerOrderExceptions && opt.layerOrderExceptions.includes(layerId)) {removeLayer(wv, layerId)continue- } else if (tolerant || opt.get('ignoreLayerOrder', false)) {+ } else if (tolerant || opt.ignoreLayerOrder) {wv.layerOrder.push(layerId)} else {removeLayer(wv, layerId)continue}}- if (!layer.projections || layer.projections.length === 0) {- error(`[${layerId}] No projections defined or not found in GC documents`)- removeLayer(wv, layerId)- continue- }- if (!Object.prototype.hasOwnProperty.call(layer, 'type')) {- error(- `[${layerId}] No type defined. Possible to be expecting configuration via GC document but was not found`- )- removeLayer(wv, layerId)- continue- }- if (Object.prototype.hasOwnProperty.call(layer, 'palette') && !Object.prototype.hasOwnProperty.call(layer.palette, 'id')) {- error(`[${layerId}] No palette definition`)- } else if (Object.prototype.hasOwnProperty.call(layer, 'palette')) {- const paletteId = layer.palette.id- if (!fs.existsSync(path.join(configDir, 'palettes', `${paletteId}.json`))) {- error(`[${layerId}] Unknown palette: ${paletteId}`)- delete layer.palette- }- }- if (Object.prototype.hasOwnProperty.call(layer, 'vectorStyle') && !Object.prototype.hasOwnProperty.call(layer.vectorStyle, 'id')) {+ // if (!layer.projections || !layer.projections.length) {+ // error(`[${layerId}] No projections defined or not found in GC documents`)+ // removeLayer(wv, layerId)+ // continue+ // }+ // if (!layer.type) {+ // error(`[${layerId}] No type defined. Possible to be expecting configuration via GC document but was not found`)+ // removeLayer(wv, layerId)+ // continue+ // }+ // if (layer.palette && !layer.palette.id) {+ // error(`[${layerId}] No palette definition`)+ // } else if (layer.palette) {+ // const paletteId = layer.palette.id+ // if (!fs.existsSync(path.join(configDir, `palettes/${paletteId}.json`))) {+ // error(`[${layerId}] palette ${paletteId} not found in palettes folder`)+ // }+ // }+ if (layer.vectorStyle && !layer.vectorStyle.id) {error(`[${layerId}] No vectorStyle definition`)}- if (!Object.prototype.hasOwnProperty.call(layer, 'group') && opt.get('warnOnUnexpectedLayer')) {+ if (!layer.group && opt.warnOnUnexpectedLayer) {error(`[${layerId}] Possible unexpected layer, no group defined`)removeLayer(wv, layerId)continue- } else if (!Object.prototype.hasOwnProperty.call(layer, 'group')) {+ } else if (!layer.group) {removeLayer(wv, layerId)continue}- for (const projId of Object.keys(layer.projections)) {- const projection = layer.projections[projId]- if (Object.prototype.hasOwnProperty.call(projection, 'matrixSet')) {- const source = projection.source- const matrixSet = projection.matrixSet- if (!Object.prototype.hasOwnProperty.call(wv.sources, source)) {- error(`[${layerId}:${projId}] Invalid source: ${source}`)- delete layer.projections[projId]- } else if (!Object.prototype.hasOwnProperty.call(wv.sources[source], 'matrixSets')) {- error(`[${layerId}:${projId}] No matrix sets for projection`)- delete layer.projections[projId]- } else if (!wv.sources[source].matrixSets.includes(matrixSet)) {- error(`[${layerId}:${projId}] Invalid matrix set: ${matrixSet}`)- delete layer.projections[projId]- }- }- }- if (Object.prototype.hasOwnProperty.call(layer, 'temporal')) {- console.warn(`[${layerId}] GC Layer temporal values overwritten by Options`)- layer = processTemporal(layer, layer.temporal)- }- if (layer.get('futureTime')) {- if (Object.prototype.hasOwnProperty.call(layer, 'endDate')) {- delete layer.endDate- }- }- if (layer.get('futureTime', false)) {+ // for (const projId of Object.keys(layer.projections)) {+ // const projection = layer.projections[projId]+ // if (projection.matrixSet) {+ // const source = projection.source+ // const matrixSet = projection.matrixSet+ // if (!wv.sources[source]) {+ // error(`[${layerId}:${projId}] Invalid source: ${source}`)+ // delete layer.projections[projId]+ // } else if (!wv.sources[source].matrixSets) {+ // error(`[${layerId}:${projId}] No matrix sets for projection`)+ // delete layer.projections[projId]+ // } else if (!wv.sources[source].matrixSets[matrixSet]) {+ // error(`[${layerId}:${projId}] Invalid matrix set: ${matrixSet}`)+ // delete layer.projections[projId]+ // }+ // }+ // }+ if ('temporal' in layer) {+ warn(`[${layerId}] GC Layer temporal values overwritten by Options`)+ layer = await processTemporalLayer(layer, layer.temporal)+ }+ if (layer.futureTime) {+ if ('endDate' in layer) delete layer.endDate+ }++ if (layer.futureTime) {// do nothing} else {- if (Object.prototype.hasOwnProperty.call(layer, 'endDate') && layer.get('ongoing', true)) {- delete layer.endDate- }+ if ('endDate' in layer && layer.ongoing) delete layer.endDate}- if (Object.prototype.hasOwnProperty.call(layer, 'startDate')) {+ if ('startDate' in layer) {const startTime = layer.startDate.replace('T', ' ').replace('Z', '')let dif (isDateTimeFormat(startTime)) {- d = new Date(startTime)+ d = moment(startTime, 'YYYY-MM-DD HH:mm:ss')} else {- d = new Date(layer.startDate)+ d = moment(layer.startDate, 'YYYY-MM-DD')}- startDate = Math.min(startDate, d)+ startDate = moment.min(startDate, d)}}- if (startDate.getTime() !== Number.MAX_VALUE) {- wv.startDate = startDate.toISOString()+ if (startDate.isValid()) {+ wv.startDate = startDate.format('YYYY-MM-DD') + 'T' + startDate.format('HH:mm:ss') + 'Z'}for (const layerId of wv.layerOrder) {- if (!Object.prototype.hasOwnProperty.call(wv.layers, layerId)) {+ if (!wv.layers[layerId]) {error(`[${layerId}] In layer order but no definition`)}}const startingLayers = []for (const startingLayer of wv.defaults.startingLayers) {- if (!Object.prototype.hasOwnProperty.call(wv.layers, startingLayer.id)) {+ if (!wv.layers[startingLayer.id]) {error(`Invalid starting layer: ${startingLayer.id}`)} else {startingLayers.push(startingLayer)}}wv.defaults.startingLayers = startingLayers- wv.buildDate = Date.now()-- for (const projection of Object.keys(wv.naturalEvents.layers)) {- for (const eventType of Object.keys(wv.naturalEvents.layers[projection])) {- for (const layerObject of wv.naturalEvents.layers[projection][eventType]) {- if (!Object.prototype.hasOwnProperty.call(wv.layers, layerObject[0])) {- error(- `The ${layerObject[0]} layer in the Natural events ${projection} ${eventType} config does not have a matching ID in the layer config`- )+ wv.buildDate = Math.round(Date.now() / 1000)++ for (const [projection, projectionValue] of Object.entries(wv.naturalEvents.layers)) {+ for (const [eventType, eventTypeLayerList] of Object.entries(projectionValue)) {+ for (const layerObject of eventTypeLayerList) {+ if (!wv.layers[layerObject[0]]) {+ error(`The ${layerObject[0]} layer in the Natural events ${projection} ${eventType} config does not have a matching ID in the layer config`)}}}}for (const measurement of Object.values(wv.measurements)) {- if (!Object.prototype.hasOwnProperty.call(measurement, 'sources')) continue+ if (!measurement.sources) continuefor (const source of Object.values(measurement.sources)) {- if (!Object.prototype.hasOwnProperty.call(source, 'settings')) continue+ if (!source.settings) continuefor (const setting of source.settings) {- if (!Object.prototype.hasOwnProperty.call(wv.layers, setting)) {- error(- `In measurement ${measurement.id}, source ${source.id}, layer not found: ${setting}`- )+ if (!wv.layers[setting]) {+ error(`In measurement ${measurement.id}, source ${source.id}, layer not found: ${setting}`)}}}}- console.log(`${prog}: ${errorCount} error(s), ${warningCount} warning(s), ${removeCount} removed`)+ const removeCount = removedLayers.length+ console.warn(`${prog}: ${errorCount} error(s), ${warningCount} warning(s), ${removeCount} removed`)+ if (removeCount > 0) {+ console.warn(removedLayers)+ }++ fs.writeFileSync(mainConfigFile, JSON.stringify(wv, null, 2))if (errorCount > 0) {throw new Error(`${prog}: Error: ${errorCount} errors occured`)diff --git a/tasks/buildOptions.sh b/tasks/buildOptions.shindex 43ce6a8ed..8896f9b28 100755--- a/tasks/buildOptions.sh+++ b/tasks/buildOptions.sh@@ -100,7 +100,6 @@ if [ -e "$BUILD_DIR/gc/vectordata" ] ; then--outputDir "$BUILD_DIR/config/wv.json/vectordata"`fi-# TODO: fix erroneous data in files# Run processColormap.js and move colormaps where we want themif [ -e "$BUILD_DIR/colormaps" ] ; thenmkdir -p "$BUILD_DIR"/config/palettes@@ -141,24 +140,30 @@ for config in $configs; doesacdone-# # Run mergeConfigWithWMTS.js to merge layer metadata from WMTS GC with worldview layer configs into wv.json+# TODO: Missing the following variables in wv.json layers+# type, format, period, startDate, endDate, dateRanges, projections,+# palette, dataCenter,+# this means _wmts .json files are not getting merged into wv.json properly++# Run mergeConfigWithWMTS.js to merge layer metadata from WMTS GC with worldview layer configs into wv.json`node $SCRIPTS_DIR/mergeConfigWithWMTS.js \--inputDir "$BUILD_DIR/_wmts" \--outputFile "$DEST_DIR/config/wv.json"`# Copy brand files from build to dest-# cp -r "$BUILD_DIR/brand" "$DEST_DIR"-# cp "$BUILD_DIR/brand.json" "$DEST_DIR"--# # Validate the options build-# `node $SCRIPTS_DIR/validateOptions.js" \-# --inputFile "$BUILD_DIR/config.json" \-# --outputDir "$DEST_DIR/config`--# # Fetch preview images from WV Snapshots for any layers which they are missing-# `node $SCRIPTS_DIR/fetchPreviewSnapshots.js" \-# --inputFile "$DEST_DIR/config/wv.json" \-# --outputFile "$OPT_DIR/common/previewLayerOverrides.json" \+cp -r "$BUILD_DIR/brand" "$DEST_DIR"+cp "$BUILD_DIR/brand.json" "$DEST_DIR"+++# Validate the options build+# `node $SCRIPTS_DIR/validateOptions.js \+# --optionsFile "$BUILD_DIR/config.json" \+# --configDir "$DEST_DIR/config"`++# Fetch preview images from WV Snapshots for any layers which they are missing+# `node $SCRIPTS_DIR/fetchPreviewSnapshots.js \+# --wvJsonFile "$DEST_DIR/config/wv.json" \+# --overridesFile "$OPT_DIR/common/previewLayerOverrides.json" \# --featuresFile "$BUILD_DIR/features.json"`exit 0