Skip to content

Interactive interface for browsing global, full-resolution satellite imagery

remove python3 code and code cleanup

ZachTRice5d4ac0f5e84a
diff --git a/legacy/e2e/browserstack.conf.js b/legacy/e2e/browserstack.conf.js
deleted file mode 100644
index cbde533f5..000000000
--- a/legacy/e2e/browserstack.conf.js
+++ /dev/null
@@ -1,85 +0,0 @@
-const glob = require('glob')
-const environments = require('./environments.js')
-
-const files = glob.sync('./e2e/features/**/*-test.js')
-const now = new Date()
-const timeStamp = `${now.getMonth() + 1}/${now.getDate()} @ ${now.getHours()}:${now.getMinutes()}`
-
-const nightwatchConfig = {
- output_folder: './e2e/reports',
- globals_path: './globals.js',
- custom_assertions_path: ['./e2e/custom-assertions'],
- src_folders: files,
- selenium: {
- start_process: false,
- host: 'hub-cloud.browserstack.com',
- port: 80
- },
- common_capabilities: {
- 'browserstack.user': process.env.BROWSERSTACK_USER,
- 'browserstack.key': process.env.BROWSERSTACK_ACCESS_KEY,
- 'browserstack.local': true,
- 'browserstack.debug': true,
- 'browserstack.localIdentifier': `wvtester19234${process.env.BROWSERSTACK_USER}`.replace(/[^a-zA-Z0-9]/g, ''),
- build: `wv-nightwatch-${timeStamp}`,
- applicationCacheEnabled: false,
- webStorageEnabled: false,
- marionette: true,
- project: 'Worldview'
- },
- test_settings: {
- default: {},
- browserstack: {
- desiredCapabilities: {
- browser: 'chrome'
- }
- },
- chrome: {
- desiredCapabilities: {
- browser: 'chrome',
- marionette: true,
- 'browserstack.selenium_version': '3.5.2'
- }
- },
- safari: {
- desiredCapabilities: {
- browser: 'safari',
- 'browserstack.selenium_version': '3.5.2'
- }
- },
- firefox: {
- desiredCapabilities: {
- browser: 'firefox',
- marionette: true,
- browserName: 'firefox',
- javascriptEnabled: true,
- 'browserstack.selenium_version': '3.10.0',
- 'browserstack.geckodriver': '0.24.0'
- }
- }
- }
-}
-
-environments.forEach((e) => {
- const env = [
- e.browser,
- e.browser_version,
- e.os,
- e.os_version
- ].join('_').replace(/\./g, '-').replace(/ /g, '_')
- nightwatchConfig.test_settings[env] = { desiredCapabilities: e }
-})
-
-// Merge common_capabilities with each test_settings key
-Object.keys(nightwatchConfig.test_settings).forEach((settingKey) => {
- const config = nightwatchConfig.test_settings[settingKey]
- config.selenium_host = nightwatchConfig.selenium.host
- config.selenium_port = nightwatchConfig.selenium.port
- config.desiredCapabilities = config.desiredCapabilities || {}
- const capabilityKeys = Object.keys(nightwatchConfig.common_capabilities)
- capabilityKeys.forEach((capabilityKey) => {
- config.desiredCapabilities[capabilityKey] = config.desiredCapabilities[capabilityKey] ||
- nightwatchConfig.common_capabilities[capabilityKey]
- })
-})
-module.exports = nightwatchConfig
diff --git a/legacy/e2e/browserstack.js b/legacy/e2e/browserstack.js
deleted file mode 100644
index 9d8d2ac7e..000000000
--- a/legacy/e2e/browserstack.js
+++ /dev/null
@@ -1,58 +0,0 @@
-/* eslint new-cap: "off" */
-
-const Nightwatch = require('nightwatch')
-const browserstack = require('browserstack-local')
-const environments = require('./environments.js')
-
-const environmentNames = environments.map(
- (e) => [
- e.browser,
- e.browser_version,
- e.os,
- e.os_version
- ].join('_').replace(/\./g, '-').replace(/ /g, '_')
-)
-
-try {
- process.mainModule.filename = './node_modules/nightwatch/bin/nightwatch'
-
- // Code to start browserstack local before start of test
- console.log('Connecting localhost to Browserstack...')
- const bsLocal = new browserstack.Local()
- Nightwatch.bs_local = bsLocal
- bs_local.start({
- key: process.env.BROWSERSTACK_ACCESS_KEY,
- localIdentifier: `wvtester19234${process.env.BROWSERSTACK_USER}`.replace(/[^a-zA-Z0-9]/g, ''),
- force: 'true' // if you want to kill existing ports
- }, (error) => {
- if (error) throw new Error(error)
- console.log('Connected. Running tests...')
- console.log('Go to https://www.browserstack.com/automate to view tests in progress.')
-
- Nightwatch.cli((argv) => {
- const envString = environmentNames.join(',')
- argv.e = envString
- argv.env = envString
- Nightwatch.CliRunner(argv)
- .setup(null, () => {
- bs_local.stop(() => {
- process.exitCode = 0
- })
- })
- .runTests(() => {
- bs_local.stop(() => {
- if (bs_local.pid && process) {
- process.exitCode = 0
- }
- })
- }).catch((err) => {
- console.error(err)
- process.exitCode = 1
- })
- })
- })
-} catch (ex) {
- console.log('There was an error while starting the test runner:\n\n')
- process.stderr.write(`${ex.stack}\n`)
- process.exitCode = 1
-}
diff --git a/legacy/e2e/environments.js b/legacy/e2e/environments.js
deleted file mode 100644
index 9b6ae0cad..000000000
--- a/legacy/e2e/environments.js
+++ /dev/null
@@ -1,35 +0,0 @@
-const bsCapabilities = require('browserstack-capabilities')
-
-const createCapabilities = bsCapabilities(
- process.env.BROWSERSTACK_USER,
- process.env.BROWSERSTACK_ACCESS_KEY
-).create
-
-const capabilities = createCapabilities(
- [
- {
- browser: 'firefox',
- browser_version: ['69.0'],
- os: ['OS X'],
- os_version: ['High Sierra']
- },
- {
- browser: 'chrome',
- browser_version: ['76.0'],
- os: ['OS X'],
- os_version: ['Mojave']
- },
- {
- browser: 'ie',
- browser_version: ['11.0'],
- os: ['Windows'],
- os_version: ['10']
- }
- ]
-)
-
-capabilities.forEach((capability) => {
- capability.resolution = '1280x1024'
-})
-
-module.exports = capabilities
diff --git a/legacy/tasks/python3/util.py b/legacy/tasks/python3/util.py
deleted file mode 100755
index 195456e34..000000000
--- a/legacy/tasks/python3/util.py
+++ /dev/null
@@ -1,25 +0,0 @@
-from copy import deepcopy
-
-# dict_merge from
-# http://blog.impressiver.com/post/31434674390/deep-merge-multiple-python-dicts
-def dict_merge(target, *args):
- # Merge multiple dicts
- if len(args) > 1:
- for obj in args:
- dict_merge(target, obj)
- return target
-
- # Recursively merge dicts and set non-dict values
- obj = args[0]
- if not isinstance(obj, dict):
- return obj
- for k, v in obj.items():
-
- if k in target and isinstance(target[k], dict):
- if 'type' in v and 'type' in target[k]:
- if v['type'] != target[k]['type']:
- return target
- dict_merge(target[k], v)
- else:
- target[k] = deepcopy(v)
- return target
\ No newline at end of file
diff --git a/tasks/README.md b/tasks/README.md
deleted file mode 100644
index bb94a368c..000000000
--- a/tasks/README.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# tasks
-
-A collection of tasks used in ci/cd and to build worldview locally.
-
-## bamboo
-
-## link-check
-
-## build
-
-## util
diff --git a/tasks/build-options/validateOptions.js b/tasks/build-options/validateOptions.js
index c6ba67797..9fb6e3f3d 100644
--- a/tasks/build-options/validateOptions.js
+++ b/tasks/build-options/validateOptions.js
@@ -91,24 +91,6 @@ async function main () {
continue
}
}
- // if (!layer.projections || !layer.projections.length) {
- // error(`[${layerId}] No projections defined or not found in GC documents`)
- // removeLayer(wv, layerId)
- // continue
- // }
- // if (!layer.type) {
- // error(`[${layerId}] No type defined. Possible to be expecting configuration via GC document but was not found`)
- // removeLayer(wv, layerId)
- // continue
- // }
- // if (layer.palette && !layer.palette.id) {
- // error(`[${layerId}] No palette definition`)
- // } else if (layer.palette) {
- // const paletteId = layer.palette.id
- // if (!fs.existsSync(path.join(configDir, `palettes/${paletteId}.json`))) {
- // error(`[${layerId}] palette ${paletteId} not found in palettes folder`)
- // }
- // }
if (layer.vectorStyle && !layer.vectorStyle.id) {
error(`[${layerId}] No vectorStyle definition`)
}
@@ -120,23 +102,6 @@ async function main () {
removeLayer(wv, layerId)
continue
}
- // for (const projId of Object.keys(layer.projections)) {
- // const projection = layer.projections[projId]
- // if (projection.matrixSet) {
- // const source = projection.source
- // const matrixSet = projection.matrixSet
- // if (!wv.sources[source]) {
- // error(`[${layerId}:${projId}] Invalid source: ${source}`)
- // delete layer.projections[projId]
- // } else if (!wv.sources[source].matrixSets) {
- // error(`[${layerId}:${projId}] No matrix sets for projection`)
- // delete layer.projections[projId]
- // } else if (!wv.sources[source].matrixSets[matrixSet]) {
- // error(`[${layerId}:${projId}] Invalid matrix set: ${matrixSet}`)
- // delete layer.projections[projId]
- // }
- // }
- // }
if ('temporal' in layer) {
warn(`[${layerId}] GC Layer temporal values overwritten by Options`)
layer = await processTemporalLayer(layer, layer.temporal)
diff --git a/tasks/python3/extractConfigFromWMTS.py b/tasks/python3/extractConfigFromWMTS.py
deleted file mode 100755
index 574915ddb..000000000
--- a/tasks/python3/extractConfigFromWMTS.py
+++ /dev/null
@@ -1,290 +0,0 @@
-#!/usr/bin/env python
-
-from datetime import datetime, date, timedelta
-import json
-from optparse import OptionParser
-import os
-import sys
-import xmltodict
-import isodate
-from processTemporalLayer import process_temporal
-from collections import OrderedDict
-import traceback
-
-prog = os.path.basename(__file__)
-base_dir = os.path.join(os.path.dirname(__file__), "..")
-version = "2.0.0"
-help_description = """\
-Extracts configuration information from a WMTS GetCapabilities file, converts the XML to JSON
-"""
-
-parser = OptionParser(usage="Usage: %s <config_file> <input_dir> <output_dir>" % prog,
- version="%s version %s" % (prog, version),
- epilog=help_description)
-(options, args) = parser.parse_args()
-
-if len(args) != 3:
- parser.error("Invalid number of arguments")
-
-config_file = args[0]
-input_dir = args[1]
-output_dir = args[2]
-
-with open(config_file, "r", encoding="utf-8") as fp:
- config = json.load(fp)
-
-tolerant = config.get("tolerant", False)
-if not "wv-options-wmts" in config:
- sys.exit(0)
-
-if not os.path.exists(output_dir):
- os.makedirs(output_dir)
-
-entries = config["wv-options-wmts"]
-skip = []
-# Looks like this is for a list of layers needed to be skipped from the GC
-if "skip" in config:
- skip = config["skip"]
-
-total_layer_count = 0
-total_warning_count = 0
-total_error_count = 0
-
-json_options = {}
-json_options["indent"] = 2
-json_options["separators"] = (',', ': ')
-
-class SkipException(Exception):
- pass
-
-def process_layer(gc_layer, wv_layers, entry):
- ident = gc_layer["ows:Identifier"]
- if ident in skip:
- print("%s: skipping" % ident)
- raise SkipException(ident)
-
- wv_layers[ident] = {}
- wv_layer = wv_layers[ident]
- wv_layer["id"] = ident
- wv_layer["type"] = "wmts"
- wv_layer["format"] = gc_layer["Format"]
-
- temporalForProjection = {}
-
- # Extract start and end dates
- if "Dimension" in gc_layer:
- dimension = gc_layer["Dimension"]
- if dimension["ows:Identifier"] == "Time":
- try:
- wv_layer = process_temporal(wv_layer, dimension["Value"])
-
- # # TODO we need a better way to identify granule layers
- # if ('Granule' in ident):
- # temporalForProjection = {}
- # temporalForProjection[entry["projection"]] = {
- # "startDate": wv_layer["startDate"],
- # "endDate": wv_layer["endDate"],
- # "dateRanges": wv_layer["dateRanges"],
- # }
-
- except Exception as e:
- print(traceback.format_exc())
- sys.stderr.write("%s: ERROR: [%s] %s\n" % (prog, ident, "Error processing time values."))
-
- # Extract matrix set
- matrixSetLink = gc_layer["TileMatrixSetLink"]
- matrixSet = matrixSetLink["TileMatrixSet"]
-
- projectionInfo = {
- "source": entry["source"],
- "matrixSet": matrixSet,
- }
-
- wv_layer["projections"] = {}
- if temporalForProjection.get(entry["projection"], None) is not None:
- wv_layer["projections"][entry["projection"]] = {
- **projectionInfo,
- **temporalForProjection.get(entry["projection"], {})
- }
- wv_layer.pop("dateRanges")
- wv_layer.pop("startDate")
- wv_layer.pop("endDate")
- else:
- wv_layer["projections"][entry["projection"]] = { **projectionInfo }
-
- if "TileMatrixSetLimits" in matrixSetLink and matrixSetLink["TileMatrixSetLimits"] is not None:
- matrixSetLimits = matrixSetLink["TileMatrixSetLimits"]["TileMatrixLimits"]
- mappedSetLimits = []
- for setLimit in matrixSetLimits:
- mappedSetLimits.append({
- "tileMatrix": setLimit["TileMatrix"],
- "minTileRow": int(setLimit["MinTileRow"]),
- "maxTileRow": int(setLimit["MaxTileRow"]),
- "minTileCol": int(setLimit["MinTileCol"]),
- "maxTileCol": int(setLimit["MaxTileCol"]),
- })
- wv_layer["projections"][entry["projection"]]["matrixSetLimits"] = mappedSetLimits;
-
- # Vector data links
- if "ows:Metadata" in gc_layer and gc_layer["ows:Metadata"] is not None:
- for item in gc_layer["ows:Metadata"]:
- if "@xlink:role" not in item:
- raise KeyError("No xlink:role")
- schema_version = item["@xlink:role"]
-
- if schema_version == "http://earthdata.nasa.gov/gibs/metadata-type/layer/1.0":
- vector_data_link = item["@xlink:href"]
- vector_data_file = os.path.basename(vector_data_link)
- vector_data_id = os.path.splitext(vector_data_file)[0]
- wv_layer["vectorData"] = {
- "id": vector_data_id
- }
-
- # Colormap links
- if "ows:Metadata" in gc_layer and gc_layer["ows:Metadata"] is not None:
- if "skipPalettes" in config and ident in config["skipPalettes"]:
- sys.stderr.write("%s: WARNING: Skipping palette for %s\n" % (
- prog, ident))
- global total_warning_count
- total_warning_count += 1
- else:
- for item in gc_layer["ows:Metadata"]:
- if "@xlink:role" not in item:
- raise KeyError("No xlink:role")
- schema_version = item["@xlink:role"]
-
- if schema_version == "http://earthdata.nasa.gov/gibs/metadata-type/colormap/1.3":
- colormap_link = item["@xlink:href"]
- colormap_file = os.path.basename(colormap_link)
- colormap_id = os.path.splitext(colormap_file)[0]
- wv_layer["palette"] = {
- "id": colormap_id
- }
- elif schema_version == "http://earthdata.nasa.gov/gibs/metadata-type/mapbox-gl-style/1.0":
- vectorstyle_link = item["@xlink:href"]
- vectorstyle_file = os.path.basename(vectorstyle_link)
- vectorstyle_id = os.path.splitext(vectorstyle_file)[0]
- wv_layer["vectorStyle"] = {
- "id": vectorstyle_id
- }
-
-def process_entry(entry):
- layer_count = 0
- warning_count = 0
- error_count = 0
- wv = {
- "layers": {},
- "sources": {}
- }
- wv_matrix_sets = {}
- wv["sources"][entry["source"]] = {
- "matrixSets": wv_matrix_sets
- }
-
- input_file = os.path.join(input_dir, entry["from"])
- gc_id = os.path.basename(input_file)
- try:
- with open(input_file) as fp:
- xml = fp.read()
- gc = xmltodict.parse(xml)
- except Exception as e:
- if tolerant:
- sys.stderr.write("%s: WARN: [%s] Unable to get GC: %s\n" %
- (prog, input_file, str(e)))
- warning_count += 1
- else:
- sys.stderr.write("%s: ERROR: [%s] Unable to get GC: %s\n" %
- (prog, input_file, str(e)))
- error_count += 1
- return [error_count, warning_count, layer_count]
-
- gc_contents = gc["Capabilities"]["Contents"]
- wv_layers = wv["layers"]
-
- if gc_contents is None or "Layer" not in gc_contents:
- error_count += 1
- sys.stderr.write("%s: ERROR: [%s] %s\n" % (prog, gc_id, "No layers"))
- return [error_count, warning_count, layer_count]
-
- if(type(gc["Capabilities"]["Contents"]["Layer"]) is OrderedDict):
- gc_layer = gc["Capabilities"]["Contents"]["Layer"]
- ident = gc_layer["ows:Identifier"]
- try:
- layer_count += 1
- process_layer(gc_layer, wv_layers, entry)
- except SkipException as se:
- warning_count += 1
- sys.stderr.write("%s: WARNING: [%s] Skipping\n" % (prog, ident))
- except Exception as e:
- error_count += 1
- print(traceback.format_exc())
- sys.stderr.write("%s: ERROR: [%s:%s] %s\n" % (prog, gc_id, ident, str(e)))
- else:
- for gc_layer in gc_contents["Layer"]:
- ident = gc_layer["ows:Identifier"]
- try:
- layer_count += 1
- process_layer(gc_layer, wv_layers, entry)
- except SkipException as se:
- warning_count += 1
- sys.stderr.write("%s: WARNING: [%s] Skipping\n" % (prog, id))
- except Exception as e:
- error_count += 1
- print(traceback.format_exc())
- sys.stderr.write("%s: ERROR: [%s:%s] %s\n" % (prog, gc_id, ident, str(e)))
-
- def process_matrix_set(gc_matrix_set):
- tileMatrixArr = gc_matrix_set["TileMatrix"]
- ident = gc_matrix_set["ows:Identifier"]
- zoom_levels = len(tileMatrixArr)
- resolutions = []
- formattedTileMatrixArr = []
- max_resolution = entry["maxResolution"]
- for zoom in range(0, zoom_levels):
- resolutions = resolutions + [max_resolution / (2 ** zoom)]
-
- # We are assuming that width/heights are the same for every matrix and excluding
- # the "TileWidth", "TileHeight" properties here
- for tileMatrix in tileMatrixArr:
- formattedTileMatrixArr.append({
- "matrixWidth": int(tileMatrix["MatrixWidth"]),
- "matrixHeight": int(tileMatrix["MatrixHeight"]),
- })
-
- wv_matrix_sets[ident] = {
- "id": ident,
- "maxResolution": max_resolution,
- "resolutions": resolutions,
- "tileSize": [
- int(tileMatrixArr[0]["TileWidth"]),
- int(tileMatrixArr[0]["TileHeight"])
- ],
- "tileMatrices": formattedTileMatrixArr
- }
-
- if(type(gc_contents["TileMatrixSet"]) is OrderedDict):
- process_matrix_set(gc_contents["TileMatrixSet"])
- else:
- for gc_matrix_set in gc_contents["TileMatrixSet"]:
- process_matrix_set(gc_matrix_set)
-
- output_file = os.path.join(output_dir, entry["to"])
- with open(output_file, "w", encoding="utf-8") as fp:
- json.dump(wv, fp, **json_options)
- print("%s: %d error(s), %d warning(s), %d layers for %s" % (prog,
- error_count, warning_count, layer_count, entry["source"]))
- return [error_count, warning_count, layer_count]
-
-
-# Main
-for entry in entries:
- error_count, warning_count, layer_count = process_entry(entry)
- total_error_count += error_count
- total_warning_count += warning_count
- total_layer_count += layer_count
-
-print("%s: %d error(s), %d warning(s), %d layers" % (prog, total_error_count,
- total_warning_count, total_layer_count))
-
-if total_error_count > 0:
- sys.exit(1)
diff --git a/tasks/python3/fetchPreviewSnapshots.py b/tasks/python3/fetchPreviewSnapshots.py
deleted file mode 100755
index c36c07aaa..000000000
--- a/tasks/python3/fetchPreviewSnapshots.py
+++ /dev/null
@@ -1,232 +0,0 @@
-#!/usr/bin/env python
-
-from concurrent.futures import ThreadPoolExecutor
-from optparse import OptionParser
-import requests
-from datetime import datetime
-from datetime import timedelta
-import json
-import os
-import sys
-
-prog = os.path.basename(__file__)
-parser = OptionParser(usage="Usage: %s <wv.json> <overrides_file>" % prog)
-(options, args) = parser.parse_args()
-wv_json_file = args[0]
-overrides_file = args[1]
-features_file = args[2]
-
-override_dates_dict = {}
-bad_snapshots = []
-total_success_count = 0
-total_failure_count = 0
-time_format = "%Y-%m-%dT%H:%M:%SZ"
-snapshots_url = ''
-param_dict = {
- 'base': {
- 'REQUEST': 'GetSnapshot',
- 'FORMAT': 'image/jpeg'
- },
- 'geographic': {
- 'BBOX': '-90,-180,90,180',
- 'CRS': 'EPSG:4326',
- 'WIDTH': '768',
- 'HEIGHT': '384'
- },
- 'arctic': {
- 'BBOX': '-4195000,-4195000,4195000,4195000',
- 'CRS': 'EPSG:3413',
- 'WIDTH': '512',
- 'HEIGHT': '512'
- },
- 'antarctic': {
- 'BBOX': '-4195000,-4195000,4195000,4195000',
- 'CRS': 'EPSG:3031',
- 'WIDTH': '512',
- 'HEIGHT': '512'
- }
-}
-# These layers should not be combined with the reference layer
-standalone_layers = [
- 'Graticule_15m',
- 'Coastlines_15m',
- 'Reference_Features_15m',
- 'Reference_Labels_15m'
-]
-dest_img_dir = './web/images/layers/previews/'
-reference_layers = {
- 'geographic': 'OSM_Land_Water_Map',
- 'arctic': 'OSM_Land_Water_Map',
- 'antarctic': 'SCAR_Land_Water_Map'
-}
-current = datetime.now()
-
-
-def track_bad_snapshots(layer_id, projection, request, img_file):
- global bad_snapshots
-
- # File sizes with SCAR/OSM Land_Water_Map layer only
- arctic_bad_size = 9949
- antarctic_bad_size = 4060
- geographic_bad_size = 12088
- size = img_file.tell()
-
- if size in [ geographic_bad_size, arctic_bad_size, antarctic_bad_size ]:
- bad_snapshots.append({
- 'id': layer_id,
- 'projection': projection,
- 'url': request.url
- })
-
-
-def get_best_date(projection, period, date_ranges):
- global current
- last_range = date_ranges[len(date_ranges) - 1]
- start_date = last_range.get('startDate')
- end_date = last_range.get('endDate')
- parsed_start_date = datetime.strptime(start_date, time_format)
- parsed_end_date = datetime.strptime(end_date, time_format)
- p_year = parsed_end_date.year
- p_month = parsed_end_date.month
- interval = int(last_range.get('dateInterval'))
- altered_date = None
-
- # Handle daily layers
- if (period == "daily"):
- # Go back a few more days for single day layers since something
- # too recent may not be processed yet
- if (interval == 1):
- interval = 3
- altered_date = parsed_end_date - timedelta(days=interval)
-
- # Choose a good daylight month for arctic
- if projection == "arctic" and p_month not in [4, 5, 6, 7, 8, 9]:
- if p_year == current.year and current.month < 6:
- altered_date = parsed_end_date.replace(day=1, month=6, year=current.year-1)
- else:
- altered_date = parsed_end_date.replace(day=1, month=6)
-
- # Choose a good daylight month for antarctic
- if projection == "antarctic" and p_month not in [10, 11, 12, 1, 2]:
- # TODO handle "bad" months for antarctic
- altered_date = parsed_end_date.replace(month=12)
-
- # Make sure modified date isn't out of layer date range
- if altered_date and altered_date >= parsed_start_date:
- date = datetime.strftime(altered_date, time_format)
- else:
- date = end_date
-
- return date
-
-
-def get_time_param(projection, layer_id, layer, params):
- # Only include TIME param for temporal layers
- date_ranges = layer.get('dateRanges')
- start_date = layer.get('startDate')
- period = layer.get('period')
-
- if (date_ranges):
- params['TIME'] = get_best_date(projection, period, date_ranges)
- elif (start_date):
- params['TIME'] = start_date
-
- # Use any configured override dates
- if (override_dates_dict.get(layer_id)):
- params['TIME'] = override_dates_dict[layer_id]
-
-
-def get_snapshots(layer):
- global total_failure_count
- global total_success_count
- for projection, proj_dict in layer['projections'].items():
- reference_layer = reference_layers[projection]
-
- # Sometimes a layer id is provided per projection (e.g. Land Mask layers)
- # We need to use this layer id to request the layer from WVS/GIBS
- # But, we need to use the WV id as the file name (since that's how we will look up the image in WV)
- if (proj_dict.get('layer')):
- gibs_layer_id = proj_dict.get('layer')
- wv_layer_id = layer['id']
- else:
- gibs_layer_id = wv_layer_id = layer['id']
-
- params = { **param_dict['base'], **param_dict[projection] }
- get_time_param(projection, wv_layer_id, layer, params)
-
- if (gibs_layer_id is not reference_layer and gibs_layer_id not in standalone_layers):
- params['LAYERS'] = (reference_layer + ',' + gibs_layer_id)
- params['OPACITIES'] = '0.50,1'
- else:
- params['LAYERS'] = gibs_layer_id
-
- dest_file_name = dest_img_dir + projection + '/' + wv_layer_id + '.jpg'
-
- # Only get images that we don't have already
- if (os.path.exists(dest_file_name)):
- continue
-
- try:
- image_req = requests.get(snapshots_url, params=params)
- if image_req.status_code == 200:
- status_text = 'SUCCESS'
- total_success_count += 1
- with open(dest_file_name, 'xb') as image_file:
- image_file.write(image_req.content)
- if (gibs_layer_id == reference_layers[projection]):
- continue
- track_bad_snapshots(wv_layer_id, projection, image_req, image_file)
- else:
- total_failure_count += 1
- status_text = 'ERROR'
- print("\n%s: Result: %s - %s" % (prog, status_text, image_req.status_code))
- print("%s: Layer: %s" % (prog, wv_layer_id))
- print("%s: URL: %s" % (prog, image_req.url))
-
- except Exception as e:
- print("%s ERROR: %s" % (prog, e))
-
-
-if __name__ == "__main__":
- # Check to see if this feature is enabled in features.json before continuing
- with open(features_file, 'rt', encoding="utf-8") as features_json:
- features_dict = json.load(features_json)
- if features_dict['features'].get('previewSnapshots', False) is False:
- sys.exit();
-
- # Allow manual configuration of layer ID to specific date to generate desired preview
- with open(overrides_file, 'rt', encoding="utf-8") as overrides_json:
- override_dates_dict = json.load(overrides_json)
-
- with open(wv_json_file, 'rt', encoding="utf-8") as wv_json:
- wv_json_dict = json.load(wv_json)
- layers = wv_json_dict['layers']
- snapshots_url = wv_json_dict['features']['imageDownload']['url']
- fetch_snapshots = features_dict['features'].get('previewSnapshots', False)
- if not fetch_snapshots:
- print("%s: Layer preview fetching disabled. Exiting." % prog)
- sys.exit()
-
- futures = []
- with ThreadPoolExecutor() as executor:
- for layer in layers.values():
- futures.append(executor.submit(get_snapshots, layer))
- for f in futures:
- try:
- # Need to call result() on each future to catch any raised exceptions
- f.result()
- except Exception as e:
- print("%s:" % (e))
-
- if len(bad_snapshots) > 0:
- print("\n%s: WARNING: %s snapshots returned no content. See below for details: " % (prog, len(bad_snapshots)))
- for bad_layer in bad_snapshots:
- print("\n\t Layer: %s" % bad_layer['id'])
- print("\t URL: %s" % (bad_layer['url']))
-
- if total_success_count > 0:
- print('\n%s: Successfully retrieved %s snapshots!' % (prog, total_success_count))
- if total_failure_count > 0:
- print('\n%s: WARNING: Failed to retrieve %s snapshots!' % (prog, total_failure_count))
- if total_failure_count == 0 and total_success_count == 0:
- print('\n%s: No snapshots were retrieved. All layers found in wv.json have existing preview images!' % (prog))
\ No newline at end of file
diff --git a/tasks/python3/generateCategoryGroupOrder.py b/tasks/python3/generateCategoryGroupOrder.py
deleted file mode 100755
index be3bd6427..000000000
--- a/tasks/python3/generateCategoryGroupOrder.py
+++ /dev/null
@@ -1,31 +0,0 @@
-#!/usr/bin/env python
-
-from optparse import OptionParser
-import os
-import json
-import sys
-
-prog = os.path.basename(__file__)
-parser = OptionParser(usage="Usage: %s <input_dir> <output_dir>" % prog)
-(options, args) = parser.parse_args()
-
-category_directory = args[0]
-output_dir = args[1]
-
-category_dict = {}
-
-for root, dirs, files in os.walk(category_directory):
- for file in files:
- file_path = os.path.join(root, file)
- with open(file_path, "r", encoding="utf-8") as category_config:
- config = json.load(category_config)
- try:
- category = list(config['categories'])[0]
- category_dict[category] = True
- except Exception as e:
- print("%s ERROR: Could not read category config. Check formatting of: %s" % (prog, file))
- sys.exit(1)
-
-with open(output_dir + '/categoryGroupOrder.json', "w", encoding="utf-8") as fp:
- json.dump({ 'categoryGroupOrder': list(category_dict) }, fp, indent=2, sort_keys=True)
- print("%s Successfully generated categoryGroupOrder.json with these categories: %s" % (prog, list(category_dict)))
\ No newline at end of file
diff --git a/tasks/python3/getCapabilities.py b/tasks/python3/getCapabilities.py
deleted file mode 100755
index 9bb101e24..000000000
--- a/tasks/python3/getCapabilities.py
+++ /dev/null
@@ -1,147 +0,0 @@
-#!/usr/bin/env python
-
-from optparse import OptionParser
-from collections import OrderedDict
-import os
-import sys
-import json
-import xmltodict
-import httpx
-import asyncio
-
-prog = os.path.basename(__file__)
-base_dir = os.path.join(os.path.dirname(__file__), "..")
-version = "1.0.0"
-help_description = """\
-Pulls GetCapabilities XML and linked metadata from configured locations
-"""
-
-parser = OptionParser(usage="Usage: %s <config> <output_dir>" % prog,
- version="%s version %s" % (prog, version),
- epilog=help_description)
-
-(options, args) = parser.parse_args()
-if len(args) != 2:
- parser.error("Invalid number of arguments")
-
-config_file = args[0]
-output_dir = args[1]
-colormaps = {}
-vectorstyles = {}
-vectordata = {}
-colormaps_dir = os.path.join(output_dir, "colormaps")
-vectorstyles_dir = os.path.join(output_dir, "vectorstyles")
-vectordata_dir = os.path.join(output_dir, "vectordata")
-error_count = 0
-warning_count = 0
-
-if not os.path.exists(output_dir):
- os.makedirs(output_dir)
-
-with open(config_file, "r", encoding="utf-8") as fp:
- config = json.load(fp)
-
-def handle_exception(e, link):
- sys.stderr.write("\n %s: WARN: Unable to fetch %s %s" % (prog, link, str(e)))
- global warning_count
- warning_count += 1
-
-def process_vector_data(layer):
- if "ows:Metadata" in layer and layer["ows:Metadata"] is not None:
- for item in layer["ows:Metadata"]:
- schema_version = item["@xlink:role"]
- if schema_version == "http://earthdata.nasa.gov/gibs/metadata-type/layer/1.0":
- vector_data_link = item["@xlink:href"]
- vector_data_file = os.path.basename(vector_data_link)
- vector_data_id = os.path.splitext(vector_data_file)[0]
- vectordata[vector_data_id] = vector_data_link
-
-def process_layer(layer):
- ident = layer["ows:Identifier"]
- if "ows:Metadata" in layer:
- if ident in config.get("skipPalettes", []):
- sys.stderr.write("%s: WARN: Skipping palette for %s\n" % prog, ident)
- global warning_count
- warning_count += 1
- elif layer["ows:Metadata"] is not None:
- for item in layer["ows:Metadata"]:
- schema_version = item["@xlink:role"]
- if schema_version == "http://earthdata.nasa.gov/gibs/metadata-type/colormap/1.3":
- colormap_link = item["@xlink:href"]
- colormap_file = os.path.basename(colormap_link)
- colormap_id = os.path.splitext(colormap_file)[0]
- colormaps[colormap_id] = colormap_link
-
- elif schema_version == "http://earthdata.nasa.gov/gibs/metadata-type/mapbox-gl-style/1.0":
- vector_style_link = item["@xlink:href"]
- vector_style_file = os.path.basename(vector_style_link)
- vector_style_id = os.path.splitext(vector_style_file)[0]
- vectorstyles[vector_style_id] = vector_style_link
-
-async def process_remote(client, entry):
- try:
- url = entry["from"]
- print("%s: %s" % (prog, url))
- response = await client.get(url)
- contents = response.text
- output_file = os.path.join(output_dir, entry["to"])
-
- # Write GetCapabilities responses to XML files
- with open(output_file, "w", encoding="utf-8") as fp:
- fp.write(contents)
- gc = xmltodict.parse(contents)
-
- # Find all colormaps and vectorstyles in GetCapabilities responses and store them in memory
- if gc["Capabilities"]["Contents"] is None:
- print(('error: %s: no layers' % url))
- return
-
- layers = gc["Capabilities"]["Contents"]["Layer"]
-
-
- if(type(layers) is OrderedDict):
- process_layer(layers)
- process_vector_data(layers)
- else:
- [process_layer(layer) for layer in layers]
- [process_vector_data(layer) for layer in layers]
-
- except Exception as e:
- print(('ERROR: %s: %s' % (url, str(e))))
-
-async def process_metadata(client, link, dir, ext):
- try:
- response = await client.get(link)
- contents = response.text
- if link.endswith(ext):
- output_file = os.path.join(dir, os.path.basename(link))
- with open(output_file, "w", encoding="utf-8") as fp:
- fp.write(contents)
- except Exception as e:
- handle_exception(e, link)
-
-async def gather_process(type, typeStr, client, dir, ext):
- print("%s: Fetching %d %s" % (prog, len(type), typeStr))
- sys.stdout.flush()
- if not os.path.exists(dir):
- os.makedirs(dir)
- await asyncio.gather(*[process_metadata(client, link, dir, ext) for link in list(type.values())])
-
-async def main():
- if "wv-options-fetch" in config:
- limits = httpx.Limits(max_keepalive_connections=10, max_connections=10)
- async with httpx.AsyncClient(limits=limits, timeout=20.0) as client:
- await asyncio.gather(*[process_remote(client, entry) for entry in config["wv-options-fetch"]])
- if colormaps:
- await gather_process(colormaps, 'colormaps', client, colormaps_dir, '.xml')
- if vectorstyles:
- await gather_process(vectorstyles, 'vectorstyles', client, vectorstyles_dir, '.json')
- if vectordata:
- await gather_process(vectordata, 'vectordata', client, vectordata_dir, '.json')
-
-asyncio.run(main())
-
-print("%s: %d error(s)" % (prog, error_count))
-
-if error_count > 0:
- sys.exit(1)
diff --git a/tasks/python3/getVisMetadata.py b/tasks/python3/getVisMetadata.py
deleted file mode 100755
index dab91ac68..000000000
--- a/tasks/python3/getVisMetadata.py
+++ /dev/null
@@ -1,103 +0,0 @@
-#!/usr/bin/env python
-
-from optparse import OptionParser
-import os
-import json
-import httpx
-import asyncio
-
-prog = os.path.basename(__file__)
-parser = OptionParser(usage="Usage: %s <config> <output_dir>" % prog)
-(options, args) = parser.parse_args()
-prog = os.path.basename(__file__)
-
-features_file = args[0]
-layer_order_file = args[1]
-output_file = args[2]
-
-# These are alias or otherwise layers that don't exist in GIBS
-skip_layers = [
- 'Land_Water_Map',
- 'Land_Mask',
- 'World_Database_on_Protected_Areas',
-]
-
-# NOTE: Only using these properties at this time
-use_keys = [
- 'conceptIds',
- 'dataCenter',
- 'daynight',
- 'orbitTracks',
- 'orbitDirection',
- 'ongoing',
- 'layerPeriod',
- 'title',
- 'subtitle'
-]
-layer_metadata = {}
-daacMap = {}
-failed_requests = []
-
-def get_daac(metadata):
- if metadata.get('conceptIds', None) is None:
- return metadata
- for collection in metadata.get("conceptIds"):
- origDataCenter = collection.get("dataCenter", None)
- dataCenter = daacMap.get(origDataCenter, None)
- collection.pop('dataCenter', None)
- if dataCenter is None:
- continue
- if metadata.get("dataCenter", None) is None:
- metadata["dataCenter"] = [dataCenter]
- elif dataCenter not in metadata["dataCenter"]:
- metadata["dataCenter"].append(dataCenter)
- return metadata
-
-async def get_metadata(client, layer_id, base_url):
- try:
- response = await client.get(base_url + layer_id + '.json')
- metadata = response.json()
- layer_metadata[layer_id] = get_daac(metadata)
-
- # Remove any props we don't expect to use
- metadata_keys = dict(layer_metadata[layer_id]).keys()
- for key in metadata_keys:
- if key not in use_keys:
- layer_metadata[layer_id].pop(key, None)
-
- except Exception as e:
- global failed_requests
- if layer_id not in failed_requests:
- failed_requests.append(layer_id)
- print('%s WARNING: Failed to retrieve metadata config for [%s], will retry...' % (prog, layer_id))
- else:
- print('%s WARNING: Failed to retrieve metadata config for [%s] on second attempt' % (prog, layer_id))
-
-async def main(url):
- global skip_layers
-
- with open(layer_order_file, 'rt', encoding="utf-8") as layer_order:
- layer_ids = json.load(layer_order).get('layerOrder')
- filtered_ids = list(filter(lambda l: l not in skip_layers, layer_ids))
- print('%s: Pulling vis metadata for %s layers... ' % (prog, len(filtered_ids)))
-
- limits = httpx.Limits(max_keepalive_connections=10, max_connections=10)
- async with httpx.AsyncClient(limits=limits, timeout=10.0) as client:
- await asyncio.gather(*[get_metadata(client, layer_id, url) for layer_id in filtered_ids])
- await asyncio.gather(*[get_metadata(client, layer_id, url) for layer_id in failed_requests])
-
- with open(output_file, "w", encoding="utf-8") as fp:
- # Format of this object will determine how this data is combined into wv.json
- json.dump({ 'layers': layer_metadata}, fp, indent=2, sort_keys=True)
-
-#MAIN
-if __name__ == "__main__":
- with open(features_file, 'rt', encoding="utf-8") as features:
- metadata_config = json.load(features).get('features').get('vismetadata')
- if metadata_config is not None:
- url = metadata_config.get('url')
- daacMap = metadata_config.get('daacMap', {})
- asyncio.run(main(url))
- else:
- print('%s: Visualization metadata not configured. Exiting.' % (prog))
-
diff --git a/tasks/python3/mergeConfig.py b/tasks/python3/mergeConfig.py
deleted file mode 100755
index 74b10c764..000000000
--- a/tasks/python3/mergeConfig.py
+++ /dev/null
@@ -1,53 +0,0 @@
-#!/usr/bin/env python
-
-from util import dict_merge
-from optparse import OptionParser
-import os
-import sys
-import json
-
-prog = os.path.basename(__file__)
-base_dir = os.path.join(os.path.dirname(__file__), "..")
-version = "2.0.0"
-help_description = """\
-Concatenates all configuration items a directory into one configuration file.
-"""
-
-# MAIN
-parser = OptionParser(usage="Usage: %s <input_dir> <output_file>" % prog,
- version="%s version %s" % (prog, version),
- epilog=help_description)
-
-(options, args) = parser.parse_args()
-if len(args) != 2:
- parser.error("Invalid number of arguments")
-
-input_dir = args[0]
-output_file = args[1]
-
-conf = {}
-file_count = 0
-for root, dirs, files in os.walk(input_dir):
- dirs.sort()
- for file in sorted(files):
- try:
- if not file.endswith(".json"):
- continue
- file_count += 1
- with open(os.path.join(root, file), "r", encoding="utf-8") as fp:
- data = json.load(fp)
- dict_merge(conf, data)
- except Exception as e:
- sys.stderr.write("ERROR: %s: %s\n" %
- (os.path.join(root, file), str(e)))
- sys.exit(1)
-
-json_options = {}
-json_options["indent"] = 2
-json_options["separators"] = (',', ': ')
-
-with open(output_file, "w", encoding="utf-8") as fp:
- json.dump(conf, fp, **json_options)
-
-print("%s: %s file(s) merged into %s" % (prog, file_count,
- os.path.basename(output_file)))
diff --git a/tasks/python3/mergeConfigWithWMTS.py b/tasks/python3/mergeConfigWithWMTS.py
deleted file mode 100755
index da4510e29..000000000
--- a/tasks/python3/mergeConfigWithWMTS.py
+++ /dev/null
@@ -1,84 +0,0 @@
-#!/usr/bin/env python
-
-from util import dict_merge
-from copy import deepcopy
-from optparse import OptionParser
-import os
-import sys
-import json
-
-prog = os.path.basename(__file__)
-base_dir = os.path.join(os.path.dirname(__file__), "..")
-version = "2.0.0"
-help_description = """\
-Concatenates all configuration items a directory into one configuration file.
-"""
-
-def wmts_dict_merge(target, *args):
- # Merge multiple dicts
- # Recursively merge dicts and set non-dict values
- obj = args[0]['layers']
- conf = args[1]['layers']
-
- if not isinstance(obj, dict):
- return obj
- for k, v in obj.items():
- if k in conf and isinstance(conf[k], dict):
- foundSourceMisMatch = False
- if 'projections' in v and 'projections' in conf[k]:
- conf_projections = conf[k]['projections']
- for projectionKey, projection in v['projections'].items():
- source = projection['source']
- if projectionKey in conf_projections:
- if 'source' in conf_projections[projectionKey]:
- if source != conf_projections[projectionKey]['source']:
- foundSourceMisMatch = True
- continue
- if foundSourceMisMatch:
- continue
- if k in target and isinstance(target[k], dict):
- dict_merge(target[k], v, conf[k])
- else:
- target[k] = dict_merge(v, conf[k])
- return target
-
-# MAIN
-parser = OptionParser(usage="Usage: %s <input_dir> <output_file>" % prog,
- version="%s version %s" % (prog, version),
- epilog=help_description)
-
-(options, args) = parser.parse_args()
-if len(args) != 2:
- parser.error("Invalid number of arguments")
-input_dir = args[0]
-output_file = args[1]
-new_conf = {}
-new_conf["layers"] = {}
-new_conf["sources"] = {}
-
-with open(output_file, "r", encoding="utf-8") as fp:
- output_data = json.load(fp)
-file_count = 0
-for file in os.listdir(input_dir):
- try:
- if not file.endswith(".json"):
- continue
- file_count += 1
- with open(os.path.join(input_dir, file), "r", encoding="utf-8") as fp:
- data = json.load(fp)
- new_conf['layers'] = wmts_dict_merge(new_conf['layers'], data, output_data)
- new_conf["sources"] = dict_merge(new_conf["sources"], data['sources'], output_data['sources'])
- except Exception as e:
- sys.stderr.write("ERROR: %s: %s\n" %
- (os.path.join(input_dir, file), str(e)))
- sys.exit(1)
-new_conf = dict_merge(new_conf, output_data)
-json_options = {}
-json_options["indent"] = 2
-json_options["separators"] = (',', ': ')
-
-with open(output_file, "w", encoding="utf-8") as fp:
- json.dump(new_conf, fp, **json_options)
-
-print("%s: %s file(s) merged into %s" % (prog, file_count,
- os.path.basename(output_file)))
diff --git a/tasks/python3/processColormap.py b/tasks/python3/processColormap.py
deleted file mode 100755
index 5d15709e9..000000000
--- a/tasks/python3/processColormap.py
+++ /dev/null
@@ -1,200 +0,0 @@
-#!/usr/bin/env python
-
-from concurrent.futures import ProcessPoolExecutor
-from optparse import OptionParser
-import os
-import re
-import sys
-import json
-import xmltodict
-
-prog = os.path.basename(__file__)
-base_dir = os.path.join(os.path.dirname(__file__), "..")
-version = "3.0.0"
-help_description = """\
-Converts colormaps to JSON files
-"""
-
-parser = OptionParser(usage="Usage: %s <config_file> <input_dir> <output_dir>" % prog,
- version="%s version %s" % (prog, version),
- epilog=help_description)
-
-(options, args) = parser.parse_args()
-if len(args) != 3:
- parser.error("Invalid number of arguments")
-
-config_file = args[0]
-input_dir = args[1]
-output_dir = args[2]
-with open(config_file, "r", encoding="utf-8") as fp:
- config = json.load(fp)
-skips = config.get("skipPalettes", [])
-
-def to_list(v):
- return v if isinstance(v, list) else [v]
-
-def match_legend(entry, legends):
- try:
- matched = "false"
- for legend in legends:
- if "@id" not in legend:
- raise KeyError("No legend IDs")
- if legend["@id"] == entry["@ref"]:
- matched = legend
- return matched
- except ValueError as e:
- raise ValueError("Invalid reference: %s" % entry["@ref"])
-
-def process_entries(colormap):
- entries = to_list(colormap["Entries"]["ColorMapEntry"])
-
- transparent_map = "true"
-
- for entry in entries:
- if entry["@transparent"] == "false":
- transparent_map = "false"
- if (transparent_map == "true"):
- return "transparent"
-
- if "Legend" not in colormap:
- raise KeyError("No Legend")
- else:
- map_type = colormap["Legend"]["@type"]
- legends = to_list(colormap["Legend"]["LegendEntry"])
- colors = []
- values = []
- ticks = []
- tooltips = []
- legend_colors = []
- refs_list = []
- ref_skip_list = []
- color_format = "{0:02x}{1:02x}{2:02x}{3:02x}"
- for index, entry in enumerate(entries):
- legend = match_legend(entry, legends)
- if (legend == "false"):
- ref_skip_list += [entry['@ref']]
- continue
- r,g,b = entry["@rgb"].split(",")
- a = 0 if entry.get("@transparent", "false") == "true" else 255
- if a == 0:
- ref_skip_list += [entry['@ref']]
- continue
- if "@ref" not in entry:
- raise KeyError("No ref in legend")
- refs_list += [entry['@ref']]
- colors += [color_format.format(int(r), int(g), int(b), a)]
- if (map_type == "continuous") or (map_type == "discrete"):
- items = re.sub(r"[\(\)\[\]]", "", entry["@value"]).split(",")
- try:
- new_items = []
- for item in items:
- v = float(item)
- if v == float("inf"):
- v = sys.float_info.max
- if v == float("-inf"):
- v = sys.float_info.min
- new_items += [v]
- values += [new_items]
- except ValueError as e:
- raise ValueError("Invalid value: %s" % entry["@value"])
- skip_index = 0
- id_list = []
- for index, entry in enumerate(legends):
- if entry['@id'] in ref_skip_list:
- skip_index = skip_index + 1
- continue
- r,g,b = entry["@rgb"].split(",")
- legend_colors += [color_format.format(int(r), int(g), int(b), 255)]
- if "@tooltip" not in entry:
- raise KeyError("No tooltips in legend")
- tooltips += [entry["@tooltip"]]
- if "@id" not in entry:
- raise KeyError("No id in legend")
- id_list += [entry['@id']]
- if "@showTick" in entry:
- ticks += [index - skip_index]
-
- result = {
- "type": map_type,
- "entries": {
- "type": map_type,
- "colors": colors,
- "refs": refs_list
- },
- "legend": {
- "colors": legend_colors,
- "type": map_type,
- "tooltips": tooltips,
- "ticks": ticks,
- "refs": id_list
- }
- }
- if (map_type == "continuous") or (map_type == "discrete"):
- result["entries"]["values"] = values
-
- return result
-
-def read_file(file):
- id = os.path.splitext(os.path.basename(file))[0]
- if id in skips:
- sys.stderr.write("%s: WARN: [%s] %s\n" % (prog, file), "Skipping")
- return
- with open(file, "r", encoding="utf-8") as fp:
- xml = fp.read()
- return (id, xml)
-
-def process_file(id, xml):
- document = xmltodict.parse(xml)
- colormaps = to_list(document["ColorMaps"]["ColorMap"])
- maps = []
- for colormap in colormaps:
- result = process_entries(colormap)
- if result == "transparent":
- continue
- result["title"] = colormap["@title"]
- result["entries"]["title"] = colormap["@title"]
- result["legend"]["title"] = colormap["@title"]
- result["legend"]["id"] = id + "_" + str(len(maps)) + "_legend"
- if "@minLabel" in colormap["Legend"]:
- result["legend"]["minLabel"] = colormap["Legend"]["@minLabel"]
- if "@maxLabel" in colormap["Legend"]:
- result["legend"]["maxLabel"] = colormap["Legend"]["@maxLabel"]
- if "@units" in colormap:
- result["legend"]["units"] = colormap["@units"]
- maps += [result]
-
- data = {
- "id": id,
- "maps": maps
- }
- json_options = {}
- json_options["indent"] = 2
- json_options["separators"] = (',', ': ')
-
- output_file = os.path.join(output_dir, id + ".json")
- with open(output_file, "w", encoding="utf-8") as fp:
- json.dump(data, fp, **json_options)
-
-# Main
-file_count = 0
-error_count = 0
-futures = []
-if __name__ == "__main__":
- with ProcessPoolExecutor() as readExecutor:
- for root, dirs, files in os.walk(input_dir):
- for file in files:
- input_file = os.path.join(root, file)
- futures.append(readExecutor.submit(read_file, input_file))
- with ProcessPoolExecutor() as writeExecutor:
- for future in futures:
- try:
- file_count += 1
- (id, xml) = future.result()
- writeExecutor.submit(process_file, id, xml)
- except Exception as e:
- sys.stderr.write("%s: ERROR: [%s] %s\n" % (prog, id, str(e)))
- error_count += 1
- print("%s: %d error(s), %d file(s)" % (prog, error_count, file_count))
-
-if error_count > 0:
- sys.exit(1)
diff --git a/tasks/python3/processTemporalLayer.py b/tasks/python3/processTemporalLayer.py
deleted file mode 100755
index 626ccea7c..000000000
--- a/tasks/python3/processTemporalLayer.py
+++ /dev/null
@@ -1,79 +0,0 @@
-from datetime import datetime
-import isodate
-import re
-import traceback
-
-def to_list(val):
- return [val] if not hasattr(val, 'reverse') else val
-
-# Add duration to end date using
-# ISO 8601 duration keys
-def determine_end_date(key, date):
- return date + isodate.parse_duration(key)
-
-# This method takes a layer and a temporal
-# value and tranlates it to start and end dates
-def process_temporal(wv_layer, value):
- dateFormat = "%Y-%m-%d"
- timeFormat = "%H:%M:%S"
- dateTimeFormat = "%Y-%m-%d %H:%M:%S"
- try:
- ranges = to_list(value)
- if "T" in ranges[0]:
- wv_layer["period"] = "subdaily"
-
- else:
- if ranges[0].endswith("Y"):
- wv_layer["period"] = "yearly"
- elif ranges[0].endswith("M"):
- wv_layer["period"] = "monthly"
- else:
- wv_layer["period"] = "daily"
- start_date = datetime.max
- end_date = datetime.min
- date_range_start, date_range_end, range_interval = [], [], []
-
- for range in ranges:
- start, end, interval = range.split('/')
- if wv_layer["period"] == "daily" \
- or wv_layer["period"] == "monthly" \
- or wv_layer["period"] == "yearly":
- start_date = min(start_date, datetime.strptime(start, dateFormat))
- end_date = max(end_date, datetime.strptime(end, dateFormat))
- if start_date:
- startDateParse = datetime.strptime(start, dateFormat)
- date_range_start.append(startDateParse.strftime(dateFormat) + "T" + startDateParse.strftime(timeFormat) + "Z")
- if end_date:
- endDateParse = datetime.strptime(end, dateFormat)
- date_range_end.append(endDateParse.strftime(dateFormat) + "T" + endDateParse.strftime(timeFormat) + "Z")
- if interval != "P1D":
- end_date = determine_end_date(interval, end_date)
- range_interval.append(re.search(r'\d+', interval).group())
-
- # Subdaily Layers
- else:
- startTime = start.replace('T', ' ').replace('Z', '')
- endTime = end.replace('T', ' ').replace('Z', '')
- start_date = min(start_date, datetime.strptime(startTime, dateTimeFormat))
- end_date = max(end_date, datetime.strptime(endTime, dateTimeFormat))
-
- if start_date:
- startTimeParse = datetime.strptime(startTime, dateTimeFormat)
- date_range_start.append(startTimeParse.strftime(dateFormat) + "T" + startTimeParse.strftime(timeFormat) + "Z")
- if end_date:
- endTimeParse = datetime.strptime(endTime, dateTimeFormat)
- date_range_end.append(endTimeParse.strftime(dateFormat) + "T" + endTimeParse.strftime(timeFormat) + "Z")
-
- range_interval.append(re.search(r'\d+', interval).group())
-
- wv_layer["startDate"] = start_date.strftime(dateFormat) + "T" + start_date.strftime(timeFormat) + "Z"
- if end_date != datetime.min:
- wv_layer["endDate"] = end_date.strftime(dateFormat) + "T" + end_date.strftime(timeFormat) + "Z"
- if date_range_start and date_range_end:
- wv_layer["dateRanges"] = [{"startDate": s, "endDate": e, "dateInterval": i} for s, e, i in zip(date_range_start, date_range_end, range_interval)]
- except ValueError:
- raise Exception("Invalid time: {0}".format(range))
- except Exception as e:
- print(traceback.format_exc())
- raise Exception("Error processing temporal layer: {0}".format(e))
- return wv_layer
diff --git a/tasks/python3/processVectorData.py b/tasks/python3/processVectorData.py
deleted file mode 100755
index 3c77d7bec..000000000
--- a/tasks/python3/processVectorData.py
+++ /dev/null
@@ -1,63 +0,0 @@
-#!/usr/bin/env python
-
-from optparse import OptionParser
-import os
-import sys
-import json
-import shutil
-
-prog = os.path.basename(__file__)
-base_dir = os.path.join(os.path.dirname(__file__), "..")
-version = "3.0.0"
-help_description = """\
-Converts vector data to JSON files
-"""
-
-parser = OptionParser(usage="Usage: %s <config_file> <vectordata_input_dir> <output_dir>" % prog,
- version="%s version %s" % (prog, version),
- epilog=help_description)
-
-(options, args) = parser.parse_args()
-if len(args) != 3:
- parser.error("Invalid number of arguments")
-
-config_file = args[0]
-vectordata_input_dir = args[1]
-output_dir = args[2]
-
-with open(config_file, "r", encoding="utf-8") as fp:
- config = json.load(fp)
-
-def copy_file(file):
- input_file = os.path.join(root, file)
- if input_file.endswith('.json'):
- response_data = {}
- vector_layer_filename = file;
- vector_layer_id = vector_layer_filename.split(".json", 1)[0]
- response_data["vectorData"] = {}
- response_data["vectorData"][vector_layer_id] = {}
- with open(input_file, "r", encoding="utf-8") as json_file:
- initial_data = json.load(json_file)
- for i in initial_data:
- response_data["vectorData"][vector_layer_id][i] = initial_data[i]
- with open(input_file, "w", encoding="utf-8") as json_file:
- json.dump(response_data, json_file,indent=2)
- shutil.copy(input_file, output_dir)
-
-# Main
-file_count = 0
-error_count = 0
-
-for root, dirs, files in os.walk(vectordata_input_dir):
- for file in files:
- try:
- file_count += 1
- copy_file(file)
- except Exception as e:
- sys.stderr.write("%s: ERROR: [%s] %s\n" % (prog, file, str(e)))
- error_count += 1
-
-print("%s: %d error(s), %d file(s)" % (prog, error_count, file_count))
-
-if error_count > 0:
- sys.exit(1)
diff --git a/tasks/python3/processVectorStyles.py b/tasks/python3/processVectorStyles.py
deleted file mode 100755
index 57c29a70d..000000000
--- a/tasks/python3/processVectorStyles.py
+++ /dev/null
@@ -1,63 +0,0 @@
-#!/usr/bin/env python
-
-from optparse import OptionParser
-import os
-import sys
-import json
-import shutil
-
-prog = os.path.basename(__file__)
-base_dir = os.path.join(os.path.dirname(__file__), "..")
-version = "3.0.0"
-help_description = """\
-Converts vector data to JSON files
-"""
-
-parser = OptionParser(usage="Usage: %s <config_file> <vectordata_input_dir> <output_dir>" % prog,
- version="%s version %s" % (prog, version),
- epilog=help_description)
-
-(options, args) = parser.parse_args()
-if len(args) != 3:
- parser.error("Invalid number of arguments")
-
-config_file = args[0]
-vectordata_input_dir = args[1]
-output_dir = args[2]
-
-with open(config_file, "r", encoding="utf-8") as fp:
- config = json.load(fp)
-
-def copy_file(file):
- input_file = os.path.join(root, file)
- if input_file.endswith('.json'):
- response_data = {}
- vector_layer_filename = file;
- vector_layer_id = vector_layer_filename.split(".json", 1)[0]
- response_data["vectorStyles"] = {}
- response_data["vectorStyles"][vector_layer_id] = {}
- with open(input_file, "r", encoding="utf-8") as json_file:
- initial_data = json.load(json_file)
- for i in initial_data:
- response_data["vectorStyles"][vector_layer_id][i] = initial_data[i]
- with open(input_file, "w", encoding="utf-8") as json_file:
- json.dump(response_data, json_file,indent=2)
- shutil.copy(input_file, output_dir)
-
-# Main
-file_count = 0
-error_count = 0
-
-for root, dirs, files in os.walk(vectordata_input_dir):
- for file in files:
- try:
- file_count += 1
- copy_file(file)
- except Exception as e:
- sys.stderr.write("%s: ERROR: [%s] %s\n" % (prog, file, str(e)))
- error_count += 1
-
-print("%s: %d error(s), %d file(s)" % (prog, error_count, file_count))
-
-if error_count > 0:
- sys.exit(1)
diff --git a/tasks/python3/validateConfigs.py b/tasks/python3/validateConfigs.py
deleted file mode 100755
index 3b3872fdf..000000000
--- a/tasks/python3/validateConfigs.py
+++ /dev/null
@@ -1,48 +0,0 @@
-#!/usr/bin/env python
-
-from jsonschema import Draft7Validator
-from optparse import OptionParser
-import os
-import json
-import glob
-import sys
-
-
-prog = os.path.basename(__file__)
-parser = OptionParser(usage="Usage: %s <input_directory> <schema_file>" % prog)
-(options, args) = parser.parse_args()
-input_directory = args[0]
-schema_file = args[1]
-
-layerConfigFiles = []
-invalidJsonFiles = []
-
-print('%s: Validating layer configs...' % (prog))
-
-layerConfigSchema = json.load(open(schema_file))
-
-files = glob.glob(input_directory + '/**/*.json', recursive=True)
-
-for file_path in files:
- configJson = json.load(open(file_path))
- # Validate against the JSON schema
- val = Draft7Validator(layerConfigSchema)
- errors = sorted(val.iter_errors(configJson), key=lambda e: e.path)
- split_path = file_path.split('/')
- name_index = len(split_path) - 1
- file_name = split_path[name_index]
-
- if len(errors):
- invalidJsonFiles.append(file_name)
- print('%s: ERROR: %s' % (prog, file_name))
-
- for err in errors:
- print('%s: - %s' % (prog, err.message))
-
-if len(invalidJsonFiles) > 0:
- print('%s: FAILED: %s layer configs failed validation.' % (prog, len(invalidJsonFiles)))
- sys.exit(1)
-else:
- print('%s: PASSED: All layer configs passed validation!' % (prog))
-
-
diff --git a/tasks/python3/validateOptions.py b/tasks/python3/validateOptions.py
deleted file mode 100755
index 51434a153..000000000
--- a/tasks/python3/validateOptions.py
+++ /dev/null
@@ -1,197 +0,0 @@
-#!/usr/bin/env python
-
-from datetime import datetime, date, timedelta
-import json
-from optparse import OptionParser
-import os
-import sys
-import time
-from processTemporalLayer import process_temporal
-
-prog = os.path.basename(__file__)
-base_dir = os.path.join(os.path.dirname(__file__), "..")
-version = "1.0.0"
-help_description = """\
-Validates and corrects the configuration files.
-"""
-
-parser = OptionParser(usage="Usage: %s <config_json> <config_dir>" % prog,
- version="%s version %s" % (prog, version),
- epilog=help_description)
-(options, args) = parser.parse_args()
-
-if len(args) != 2:
- parser.error("Invalid number of arguments")
-
-options_file = args[0]
-config_dir = args[1]
-
-error_count = 0
-warning_count = 0
-removed_layers = []
-
-main_config_file = os.path.join(config_dir, "wv.json")
-with open(main_config_file, "r", encoding="utf-8") as fp:
- wv = json.load(fp)
-with open(options_file, "r", encoding="utf-8") as fp:
- opt = json.load(fp)
-
-tolerant = opt.get("tolerant", False)
-
-def fail(message):
- global error_count
- sys.stderr.write("%s: ERROR: %s\n" % (prog, message))
- error_count += 1
-
-def error(message):
- warn(message) if tolerant else fail(message)
-
-def warn(message):
- global warning_count
- sys.stderr.write("%s: WARN: %s\n" % (prog, message))
- warning_count += 1
-
-def remove_layer(wv, layer_id):
- global removed_layers
- removed_layers.append(layer_id);
- del wv["layers"][layer_id]
- if layer_id in wv["layerOrder"]: wv["layerOrder"].remove(layer_id)
-
-def isDateTimeFormat(input):
- try:
- datetime.strptime(input, '%Y-%m-%d %H:%M:%S')
- return True
- except ValueError:
- return False
-
-if tolerant:
- warn("Validation enforcement disabled")
-
-start_date = datetime.max
-
-for layer_id in list(wv["layers"].keys()):
- layer = wv["layers"][layer_id]
-
- if layer_id != layer.get('id'):
- error("[%s] layer id does not match id of %s" % (layer_id, layer.get('id')))
- if layer_id not in wv["layerOrder"]:
- if layer_id in opt.get("layerOrderExceptions", []):
- remove_layer(wv, layer_id)
- continue
- elif tolerant or opt.get("ignoreLayerOrder", False):
- wv["layerOrder"] += [layer_id]
- else:
- remove_layer(wv, layer_id)
- continue
- if "projections" not in layer or len(layer["projections"]) == 0:
- error("[%s] No projections defined or not found in GC documents" %
- layer_id)
- remove_layer(wv, layer_id)
- continue
- if "type" not in layer:
- error("[" + layer_id + "] No type defined. Possible to be expecting " +
- "configuration via GC document but was not found")
- remove_layer(wv, layer_id)
- continue
- if "palette" in layer and "id" not in layer["palette"]:
- error("[%s] No palette definition" % (layer_id))
- elif "palette" in layer:
- palette_id = layer["palette"]["id"]
- if not os.path.exists(os.path.join(config_dir, "palettes",
- palette_id + ".json")):
- error("[%s] Unknown palette: %s" % (layer_id, palette_id))
- del layer["palette"]
- if "vectorStyle" in layer and "id" not in layer["vectorStyle"]:
- error("[%s] No vectorStyle definition" % (layer_id))
- if "group" not in layer and opt.get("warnOnUnexpectedLayer"):
- error("[%s] Possible unexpected layer, no group defined" % layer_id)
- remove_layer(wv, layer_id)
- continue
- elif "group" not in layer:
- remove_layer(wv, layer_id)
- continue
- for proj_id in list(layer["projections"].keys()):
- projection = layer["projections"][proj_id]
- if "matrixSet" in projection:
- source = projection["source"]
- matrix_set = projection["matrixSet"]
- if source not in wv["sources"]:
- error("[%s:%s] Invalid source: %s" %
- (layer_id, proj_id, source))
- del layer["projections"][proj_id]
- elif "matrixSets" not in wv["sources"][source]:
- error("[%s:%s] No matrix sets for projection" %
- (layer_id, proj_id))
- del layer["projections"][proj_id]
- elif matrix_set not in wv["sources"][source]["matrixSets"]:
- error("[%s:%s] Invalid matrix set: %s" %
- (layer_id, proj_id, matrix_set))
- del layer["projections"][proj_id]
- if "temporal" in layer:
- warn("[%s] GC Layer temporal values overwritten by Options" % layer_id)
- layer = process_temporal(layer, layer["temporal"])
- if layer.get("futureTime"):
- if "endDate" in layer:
- del layer["endDate"]
-
- if layer.get("futureTime", False):
- pass
- else:
- if "endDate" in layer and layer.get("ongoing", True):
- del layer["endDate"]
- if "startDate" in layer:
- startTime = layer["startDate"].replace('T', ' ').replace('Z', '')
- if isDateTimeFormat(startTime):
- d = datetime.strptime(startTime, "%Y-%m-%d %H:%M:%S")
- else:
- d = datetime.strptime(layer["startDate"], "%Y-%m-%d")
- start_date = min(start_date, d)
-
-if start_date != datetime.max:
- wv["startDate"] = start_date.strftime("%Y-%m-%d") + "T" + start_date.strftime("%H:%M:%S") + "Z"
-
-for layer_id in wv["layerOrder"]:
- if layer_id not in wv["layers"]:
- error("[%s] In layer order but no definition" % layer_id)
-
-startingLayers = []
-for startingLayer in wv["defaults"]["startingLayers"]:
- if startingLayer["id"] not in wv["layers"]:
- error("Invalid starting layer: %s" % (startingLayer["id"]))
- else:
- startingLayers += [startingLayer]
-wv["defaults"]["startingLayers"] = startingLayers
-wv["buildDate"] = int(round(time.time() * 1000))
-
-for projection, projectionValue in wv["naturalEvents"]["layers"].items():
- for eventType, eventTypeLayerList in projectionValue.items():
- for layerObject in eventTypeLayerList:
- if layerObject[0] not in wv["layers"]:
- error("The %s layer in the Natural events %s %s config does not have a matching ID in the layer config" %
- (layerObject[0], projection, eventType))
-
-for measurement in list(wv["measurements"].values()):
- if "sources" not in measurement:
- continue
- for source in list(measurement["sources"].values()):
- if "settings" not in source:
- continue
- for setting in source["settings"]:
- if setting not in wv["layers"]:
- error("In measurement %s, source %s, layer not found: %s" %
- (measurement["id"], source["id"], setting))
-
-remove_count = len(removed_layers)
-print("%s: %d error(s), %d warning(s), %d removed" % (prog, error_count, warning_count, remove_count))
-if remove_count > 0:
- print(removed_layers)
-
-json_options = {}
-json_options["indent"] = 2
-json_options["separators"] = (',', ': ')
-
-with open(main_config_file, "w", encoding="utf-8") as fp:
- json.dump(wv, fp)
-
-if error_count > 0:
- sys.exit(1)