Interactive interface for browsing global, full-resolution satellite imagery
Fix most issues with extractConfigFromWMTS and processTemporalLayer
ZachTRicede6fc92d198c
diff --git a/package-lock.json b/package-lock.jsonindex 678641023..86c6c6ac2 100644--- a/package-lock.json+++ b/package-lock.json@@ -124,7 +124,6 @@"react-refresh": "^0.14.0","react-test-renderer": "^16.12.0","redux-mock-store": "^1.5.3",- "regex-parser": "^2.2.11","run-script-os": "^1.1.5","sass": "^1.49.7","sass-loader": "^10.1.0",@@ -18824,12 +18823,6 @@"node": ">=0.10.0"}},- "node_modules/regex-parser": {- "version": "2.2.11",- "resolved": "https://registry.npmjs.org/regex-parser/-/regex-parser-2.2.11.tgz",- "integrity": "sha512-jbD/FT0+9MBU2XAZluI7w2OBs1RBi6p9M83nkoZayQXXU9e8Robt69FcZc7wU4eJD/YFTjn1JdCk3rbMJajz8Q==",- "dev": true- },"node_modules/regexp.prototype.flags": {"version": "1.4.3","resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.4.3.tgz",@@ -37559,12 +37552,6 @@"safe-regex": "^1.1.0"}},- "regex-parser": {- "version": "2.2.11",- "resolved": "https://registry.npmjs.org/regex-parser/-/regex-parser-2.2.11.tgz",- "integrity": "sha512-jbD/FT0+9MBU2XAZluI7w2OBs1RBi6p9M83nkoZayQXXU9e8Robt69FcZc7wU4eJD/YFTjn1JdCk3rbMJajz8Q==",- "dev": true- },"regexp.prototype.flags": {"version": "1.4.3","resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.4.3.tgz",diff --git a/package.json b/package.jsonindex 3117bbeeb..676dfb06f 100644--- a/package.json+++ b/package.json@@ -135,7 +135,6 @@"react-refresh": "^0.14.0","react-test-renderer": "^16.12.0","redux-mock-store": "^1.5.3",- "regex-parser": "^2.2.11","run-script-os": "^1.1.5","sass": "^1.49.7","sass-loader": "^10.1.0",diff --git a/tasks/build/extractConfigFromWMTS.js b/tasks/build/extractConfigFromWMTS.jsindex 177efdca4..0bcc3ddb6 100644--- a/tasks/build/extractConfigFromWMTS.js+++ b/tasks/build/extractConfigFromWMTS.js@@ -3,7 +3,7 @@ const path = require('path')const yargs = require('yargs')const console = require('console')const convert = require('xml-js')-const processTemporalLayer = require('./processTemporalLayer')+const { processTemporalLayer } = require('./processTemporalLayer')const prog = path.basename(__filename)@@ -56,6 +56,11 @@ let totalLayerCount = 0let totalWarningCount = 0let totalErrorCount = 0+const wv = {+ layers: {},+ sources: {}+}+class SkipException extends Error {constructor (message) {super(message)@@ -74,6 +79,13 @@ async function main () {const { errorCount, warningCount, layerCount } = await processEntry(entry)console.warn(`${prog}: ${errorCount} errors, ${warningCount} warnings, ${layerCount} layers for ${entry.source}`)+ const outputFile = path.join(outputDir, entry.to)+ fs.writeFile(outputFile, JSON.stringify(wv, null, 2), 'utf-8', err => {+ if (err) {+ console.error(err)+ }+ })+totalErrorCount += errorCounttotalWarningCount += warningCounttotalLayerCount += layerCount@@ -86,8 +98,88 @@ async function main () {}}-function processLayer (gcLayer, wvLayers, entry) {- const ident = gcLayer['ows:Identifier']+async function processEntry (entry) {+ let layerCount = 0+ let warningCount = 0+ let errorCount = 0++ wv.sources[entry.source] = {+ matrixSets: wvMatrixSets+ }++ const inputFile = path.join(inputDir, entry.from)+ const gcId = path.basename(inputFile)+ let gc+ try {+ const xml = await fs.promises.readFile(inputFile, 'utf8')+ gc = JSON.parse(convert.xml2json(xml, { compact: true, spaces: 2 }))+ } catch (e) {+ if (tolerant) {+ process.stderr.write(`${prog}: WARN: [${inputFile}] Unable to get GC: ${e}\n`)+ warningCount += 1+ } else {+ process.stderr.write(`${prog}: ERROR: [${inputFile}] Unable to get GC: ${e}\n`)+ errorCount += 1+ }+ return { errorCount, warningCount, layerCount }+ }++ const gcContents = gc.Capabilities.Contents+ const wvLayers = wv.layers++ if (!gcContents || !gcContents.Layer) {+ errorCount += 1+ process.stderr.write(`${prog}: ERROR: [${gcId}] No layers\n`)+ return { errorCount, warningCount, layerCount }+ }++ // if (typeof gc.Capabilities.Contents.Layer === 'object') {+ // const gcLayer = gc.Capabilities.Contents.Layer+ // const ident = gcLayer['ows:Identifier']+ // try {+ // layerCount += 1+ // processLayer(gcLayer, wvLayers, entry)+ // } catch (error) {+ // if (error instanceof SkipException) {+ // warningCount += 1+ // process.stderr.write(`${prog}: WARNING: [${ident}] Skipping\n`)+ // } else {+ // errorCount += 1+ // console.error(error.stack)+ // process.stderr.write(`${prog}: ERROR: [${gcId}:${ident}] ${e}\n`)+ // }+ // }+ // } else {+ for (const gcLayer of gcContents.Layer) {+ try {+ layerCount += 1+ processLayer(gcLayer, wvLayers, entry)+ } catch (error) {+ if (error instanceof SkipException) {+ warningCount += 1+ process.stderr.write(`${prog}: WARNING: [${id}] Skipping\n`)+ } else {+ errorCount += 1+ console.error(error.stack)+ process.stderr.write(`${prog}: ERROR: [${gcId}:${ident}] ${e}\n`)+ }+ }+ }+ // }++ if (gcContents.TileMatrixSet === 'Object') {+ processMatrixSet(gcContents.TileMatrixSet)+ } else {+ gcContents.TileMatrixSet.forEach(gcMatrixSet => {+ processMatrixSet(gcMatrixSet)+ })+ }++ return { errorCount, warningCount, layerCount }+}++async function processLayer (gcLayer, wvLayers, entry) {+ const ident = gcLayer['ows:Identifier']._textif (skip.includes(ident)) {console.log(`${ident}: skipping`)throw new SkipException(ident)@@ -97,16 +189,16 @@ function processLayer (gcLayer, wvLayers, entry) {let wvLayer = wvLayers[ident]wvLayer.id = identwvLayer.type = 'wmts'- wvLayer.format = gcLayer.Format+ wvLayer.format = gcLayer.Format._textconst temporalForProjection = {}// Extract start and end datesif ('Dimension' in gcLayer) {const dimension = gcLayer.Dimension- if (dimension['ows:Identifier'] === 'Time') {+ if (dimension['ows:Identifier']._text === 'Time') {try {- wvLayer = processTemporalLayer(wvLayer, dimension.Value)+ wvLayer = await processTemporalLayer(wvLayer, dimension.Value)} catch (e) {console.error(e)console.error(`${prog}: ERROR: [${ident}] Error processing time values.`)@@ -116,7 +208,7 @@ function processLayer (gcLayer, wvLayers, entry) {// Extract matrix setconst matrixSetLink = gcLayer.TileMatrixSetLink- const matrixSet = matrixSetLink.TileMatrixSet+ const matrixSet = matrixSetLink.TileMatrixSet._textconst projectionInfo = {source: entry.source,@@ -141,7 +233,7 @@ function processLayer (gcLayer, wvLayers, entry) {const mappedSetLimits = []for (const setLimit of matrixSetLimits) {mappedSetLimits.push({- tileMatrix: setLimit.TileMatrix,+ tileMatrix: setLimit.TileMatrix._text,minTileRow: parseInt(setLimit.MinTileRow, 10),maxTileRow: parseInt(setLimit.MaxTileRow, 10),minTileCol: parseInt(setLimit.MinTileCol, 10),@@ -201,92 +293,9 @@ function processLayer (gcLayer, wvLayers, entry) {}}-async function processEntry (entry) {- let layerCount = 0- let warningCount = 0- let errorCount = 0- const wv = {- layers: {},- sources: {}- }-- wv.sources[entry.source] = {- matrixSets: wvMatrixSets- }-- const inputFile = path.join(inputDir, entry.from)- const gcId = path.basename(inputFile)- let gc- try {- const xml = await fs.promises.readFile(inputFile, 'utf8')- gc = JSON.parse(convert.xml2json(xml, { compact: true, spaces: 2 }))- } catch (e) {- if (tolerant) {- process.stderr.write(`${prog}: WARN: [${inputFile}] Unable to get GC: ${e}\n`)- warningCount += 1- } else {- process.stderr.write(`${prog}: ERROR: [${inputFile}] Unable to get GC: ${e}\n`)- errorCount += 1- }- return { errorCount, warningCount, layerCount }- }-- const gcContents = gc.Capabilities.Contents- const wvLayers = wv.layers-- if (!gcContents || !gcContents.Layer) {- errorCount += 1- process.stderr.write(`${prog}: ERROR: [${gcId}] No layers\n`)- return { errorCount, warningCount, layerCount }- }-- // if (typeof gc.Capabilities.Contents.Layer === 'object') {- // const gcLayer = gc.Capabilities.Contents.Layer- // const ident = gcLayer['ows:Identifier']- // try {- // layerCount += 1- // processLayer(gcLayer, wvLayers, entry)- // } catch (error) {- // if (error instanceof SkipException) {- // warningCount += 1- // process.stderr.write(`${prog}: WARNING: [${ident}] Skipping\n`)- // } else {- // errorCount += 1- // console.error(error.stack)- // process.stderr.write(`${prog}: ERROR: [${gcId}:${ident}] ${e}\n`)- // }- // }- // } else {- for (const gcLayer of gcContents.Layer) {- try {- layerCount += 1- processLayer(gcLayer, wvLayers, entry)- } catch (error) {- if (error instanceof SkipException) {- warningCount += 1- process.stderr.write(`${prog}: WARNING: [${id}] Skipping\n`)- } else {- errorCount += 1- console.error(error.stack)- process.stderr.write(`${prog}: ERROR: [${gcId}:${ident}] ${e}\n`)- }- }- }- // }-- if (gcContents.TileMatrixSet === 'Object') {- processMatrixSet(gcContents.TileMatrixSet)- } else {- gcContents.TileMatrixSet.forEach(gcMatrixSet => {- processMatrixSet(gcMatrixSet)- })- }- return { errorCount, warningCount, layerCount }-}-function processMatrixSet (gcMatrixSet) {const tileMatrixArr = gcMatrixSet.TileMatrix- const ident = gcMatrixSet['ows:Identifier']+ const ident = gcMatrixSet['ows:Identifier']._textconst zoomLevels = tileMatrixArr.lengthconst resolutions = []const formattedTileMatrixArr = []diff --git a/tasks/build/mergeConfigWithWMTS.js b/tasks/build/mergeConfigWithWMTS.jsindex 4bb5f8fd5..935e53c66 100644--- a/tasks/build/mergeConfigWithWMTS.js+++ b/tasks/build/mergeConfigWithWMTS.js@@ -36,27 +36,37 @@ const outputData = JSON.parse(fs.readFileSync(outputFile, 'utf-8'))let fileCount = 0async function main () {- fs.readdirSync(inputDir).forEach(file => {- try {- if (!file.endsWith('.json')) return- fileCount += 1- const data = JSON.parse(fs.readFileSync(path.join(inputDir, file), 'utf-8'))- newConf.layers = wmtsDictMerge(newConf.layers, data, outputData)- newConf.sources = dictMerge(newConf.sources, data.sources, outputData.sources)- } catch (e) {- throw new Error(`ERROR: ${path.join(inputDir, file)}: ${e.message}`)- }- })-- newConf = dictMerge(newConf, outputData)+ await processFiles()+ newConf = await dictMerge(newConf, outputData)const jsonOptions = {}jsonOptions.indent = 2jsonOptions.separators = [',', ': ']- fs.writeFileSync(outputFile, JSON.stringify(newConf, jsonOptions))+ await fs.writeFileSync(outputFile, JSON.stringify(newConf, jsonOptions))++ console.warn(`${prog}: ${fileCount} file(s) merged into ${path.basename(outputFile)}`)+}- console.log(`${prog}: ${fileCount} file(s) merged into ${path.basename(outputFile)}`)+async function processFiles () {+ try {+ console.warn(inputDir)+ const files = await fs.readdirSync(inputDir)+ for (const file of files) {+ if (!file.endsWith('.json')) continue+ fileCount += 1+ try {+ const data = await fs.readFile(path.join(inputDir, file), 'utf-8')+ const jsonData = JSON.parse(data)+ newConf.layers = wmtsDictMerge(newConf.layers, jsonData, outputData)+ newConf.sources = dictMerge(newConf.sources, jsonData.sources, outputData.sources)+ } catch (error) {+ throw new Error(`ERROR: ${path.join(inputDir, file)}: ${error.message}`)+ }+ }+ } catch (error) {+ console.error(error)+ }}function wmtsDictMerge (target, obj, conf) {diff --git a/tasks/build/processTemporalLayer.js b/tasks/build/processTemporalLayer.jsindex 6afb26307..b441d93dc 100644--- a/tasks/build/processTemporalLayer.js+++ b/tasks/build/processTemporalLayer.js@@ -1,22 +1,29 @@const moment = require('moment')-const re = require('regex-parser')function toList (val) {return val instanceof Array ? val : [val]}-function processTemporal (wvLayer, value) {+class ValueError extends Error {+ constructor (message) {+ super(message)+ this.name = 'ValueError'+ }+}++async function processTemporalLayer (wvLayer, value) {const dateFormat = 'YYYY-MM-DD'const timeFormat = 'HH:mm:ss'const dateTimeFormat = 'YYYY-MM-DD HH:mm:ss'try {const ranges = toList(value)- if (ranges[0].includes('T')) {+ if (ranges && ranges[0] && ranges[0]._text && ranges[0]._text.includes('T')) {+ console.warn(ranges[0]._text)wvLayer.period = 'subdaily'} else {- if (ranges[0].endsWith('Y')) {+ if (ranges && ranges[0] && ranges[0]._text && ranges[0]._text.endsWith('Y')) {wvLayer.period = 'yearly'- } else if (ranges[0].endsWith('M')) {+ } else if (ranges && ranges[0] && ranges[0]._text && ranges[0]._text.endsWith('M')) {wvLayer.period = 'monthly'} else {wvLayer.period = 'daily'@@ -28,7 +35,7 @@ function processTemporal (wvLayer, value) {const dateRangeEnd = []const rangeInterval = []for (const range of ranges) {- const [start, end, interval] = range.split('/')+ const [start, end, interval] = range._text.split('/')if (wvLayer.period === 'daily' ||wvLayer.period === 'monthly' ||@@ -53,9 +60,11 @@ function processTemporal (wvLayer, value) {)}if (interval !== 'P1D') {- endDate = moment(endDate).add(moment.duration(interval)).toDate()+ endDate = moment(endDate).add(moment.duration(interval))}- rangeInterval.push(re.parse(interval).match)+ const regex = new RegExp(/\d+/g)+ const match = regex.exec(interval)+ rangeInterval.push(match)} else {// Subdaily Layersconst startTime = start.replace('T', ' ').replace('Z', '')@@ -72,7 +81,7 @@ function processTemporal (wvLayer, value) {dateRangeEnd.push(endTimeParse.format(`${dateFormat}T${timeFormat}Z`))}- rangeInterval.push(re.search(/\d+/, interval).group())+ rangeInterval.push(interval.match(/\d+/)[0])}wvLayer.startDate = moment(startDate).format(`${dateFormat}T${timeFormat}Z`)@@ -97,5 +106,5 @@ function processTemporal (wvLayer, value) {}module.exports = {- processTemporal+ processTemporalLayer}diff --git a/tasks/buildOptions.sh b/tasks/buildOptions.shindex 3564cb37a..f7eea6d02 100755--- a/tasks/buildOptions.sh+++ b/tasks/buildOptions.sh@@ -47,25 +47,25 @@ fimkdir -p "$DEST_DIR/config"mkdir -p "$BUILD_DIR/colormaps"-# # If $FETCH_GC is set, make various API requests-# if [ "$FETCH_GC" ] ; then-# # TODO: Fix axios fetch timeouts-# # Fetch GC files and create colormaps, vectordata and vectorstyle files-# rm -rf "$OPT_DIR/$OPT_SUBDIR/gc/*"-# rm -rf "$OPT_DIR/$OPT_SUBDIR/colormaps/gc/*"-# `node $SCRIPTS_DIR/getCapabilities.js \-# --config "$OPT_DIR/$OPT_SUBDIR/config.json" \-# --getcapabilities "$OPT_DIR/$OPT_SUBDIR/gc"`+# If $FETCH_GC is set, make various API requests+if [ "$FETCH_GC" ] ; then+ # TODO: Fix axios fetch timeouts+ # Fetch GC files and create colormaps, vectordata and vectorstyle files+ rm -rf "$OPT_DIR/$OPT_SUBDIR/gc/*"+ rm -rf "$OPT_DIR/$OPT_SUBDIR/colormaps/gc/*"+ `node $SCRIPTS_DIR/getCapabilities.js \+ --config "$OPT_DIR/$OPT_SUBDIR/config.json" \+ --getcapabilities "$OPT_DIR/$OPT_SUBDIR/gc"`# TODO: Optimize performance of this script# Get metadata for files in layerOrder.json and combine this data into 1 file- # rm -rf "$OPT_DIR/$OPT_SUBDIR/layer-metadata"- # mkdir -p "$OPT_DIR/$OPT_SUBDIR/layer-metadata"- # `node $SCRIPTS_DIR/getVisMetadata.js \- # --features "$BUILD_DIR/features.json" \- # --layerOrder "$BUILD_DIR/config/wv.json/layerOrder.json" \- # --layerMetadata "$OPT_DIR/$OPT_SUBDIR/layer-metadata/all.json"`-# fi+ rm -rf "$OPT_DIR/$OPT_SUBDIR/layer-metadata"+ mkdir -p "$OPT_DIR/$OPT_SUBDIR/layer-metadata"+ `node $SCRIPTS_DIR/getVisMetadata.js \+ --features "$BUILD_DIR/features.json" \+ --layerOrder "$BUILD_DIR/config/wv.json/layerOrder.json" \+ --layerMetadata "$OPT_DIR/$OPT_SUBDIR/layer-metadata/all.json"`+fi# Validate layers in wv.json with a JSON schema`node $SCRIPTS_DIR/validateConfigs.js \@@ -76,10 +76,7 @@ if [ -e "$BUILD_DIR/features.json" ] ; thencp "$BUILD_DIR/features.json" "$BUILD_DIR/config/wv.json/_features.json"fi-# TODO: Fix sometimes the following error is thrown:-# TypeError: Converting circular structure to JSON-# extractConfigFromWMTS.js:85:11-# this is coming from an issue in getCapabilities.js+# TODO: Fix _wmts files containing all projections# Run extractConfigFromWMTS.js script with config.jsonif [ -e "$BUILD_DIR/config.json" ] ; then`node $SCRIPTS_DIR/extractConfigFromWMTS.js \@@ -146,9 +143,9 @@ for config in $configs; dodone# # Run mergeConfigWithWMTS.js to merge layer metadata from WMTS GC with worldview layer configs into wv.json-# `node $SCRIPTS_DIR/mergeConfigWithWMTS.js \-# --inputDir "$BUILD_DIR/_wmts" \-# --outputDir "$DEST_DIR/config/wv.json"`+`node $SCRIPTS_DIR/mergeConfigWithWMTS.js \+ --inputDir "$BUILD_DIR/_wmts" \+ --outputFile "$DEST_DIR/config/wv.json"`# # Copy brand files from build to dest# cp -r "$BUILD_DIR/brand" "$DEST_DIR"