Interactive interface for browsing global, full-resolution satellite imagery
remove python3 code and code cleanup
ZachTRice5d4ac0f5e84a
diff --git a/legacy/e2e/browserstack.conf.js b/legacy/e2e/browserstack.conf.jsdeleted file mode 100644index cbde533f5..000000000--- a/legacy/e2e/browserstack.conf.js+++ /dev/null@@ -1,85 +0,0 @@-const glob = require('glob')-const environments = require('./environments.js')--const files = glob.sync('./e2e/features/**/*-test.js')-const now = new Date()-const timeStamp = `${now.getMonth() + 1}/${now.getDate()} @ ${now.getHours()}:${now.getMinutes()}`--const nightwatchConfig = {- output_folder: './e2e/reports',- globals_path: './globals.js',- custom_assertions_path: ['./e2e/custom-assertions'],- src_folders: files,- selenium: {- start_process: false,- host: 'hub-cloud.browserstack.com',- port: 80- },- common_capabilities: {- 'browserstack.user': process.env.BROWSERSTACK_USER,- 'browserstack.key': process.env.BROWSERSTACK_ACCESS_KEY,- 'browserstack.local': true,- 'browserstack.debug': true,- 'browserstack.localIdentifier': `wvtester19234${process.env.BROWSERSTACK_USER}`.replace(/[^a-zA-Z0-9]/g, ''),- build: `wv-nightwatch-${timeStamp}`,- applicationCacheEnabled: false,- webStorageEnabled: false,- marionette: true,- project: 'Worldview'- },- test_settings: {- default: {},- browserstack: {- desiredCapabilities: {- browser: 'chrome'- }- },- chrome: {- desiredCapabilities: {- browser: 'chrome',- marionette: true,- 'browserstack.selenium_version': '3.5.2'- }- },- safari: {- desiredCapabilities: {- browser: 'safari',- 'browserstack.selenium_version': '3.5.2'- }- },- firefox: {- desiredCapabilities: {- browser: 'firefox',- marionette: true,- browserName: 'firefox',- javascriptEnabled: true,- 'browserstack.selenium_version': '3.10.0',- 'browserstack.geckodriver': '0.24.0'- }- }- }-}--environments.forEach((e) => {- const env = [- e.browser,- e.browser_version,- e.os,- e.os_version- ].join('_').replace(/\./g, '-').replace(/ /g, '_')- nightwatchConfig.test_settings[env] = { desiredCapabilities: e }-})--// Merge common_capabilities with each test_settings key-Object.keys(nightwatchConfig.test_settings).forEach((settingKey) => {- const config = nightwatchConfig.test_settings[settingKey]- config.selenium_host = nightwatchConfig.selenium.host- config.selenium_port = nightwatchConfig.selenium.port- config.desiredCapabilities = config.desiredCapabilities || {}- const capabilityKeys = Object.keys(nightwatchConfig.common_capabilities)- capabilityKeys.forEach((capabilityKey) => {- config.desiredCapabilities[capabilityKey] = config.desiredCapabilities[capabilityKey] ||- nightwatchConfig.common_capabilities[capabilityKey]- })-})-module.exports = nightwatchConfigdiff --git a/legacy/e2e/browserstack.js b/legacy/e2e/browserstack.jsdeleted file mode 100644index 9d8d2ac7e..000000000--- a/legacy/e2e/browserstack.js+++ /dev/null@@ -1,58 +0,0 @@-/* eslint new-cap: "off" */--const Nightwatch = require('nightwatch')-const browserstack = require('browserstack-local')-const environments = require('./environments.js')--const environmentNames = environments.map(- (e) => [- e.browser,- e.browser_version,- e.os,- e.os_version- ].join('_').replace(/\./g, '-').replace(/ /g, '_')-)--try {- process.mainModule.filename = './node_modules/nightwatch/bin/nightwatch'-- // Code to start browserstack local before start of test- console.log('Connecting localhost to Browserstack...')- const bsLocal = new browserstack.Local()- Nightwatch.bs_local = bsLocal- bs_local.start({- key: process.env.BROWSERSTACK_ACCESS_KEY,- localIdentifier: `wvtester19234${process.env.BROWSERSTACK_USER}`.replace(/[^a-zA-Z0-9]/g, ''),- force: 'true' // if you want to kill existing ports- }, (error) => {- if (error) throw new Error(error)- console.log('Connected. Running tests...')- console.log('Go to https://www.browserstack.com/automate to view tests in progress.')-- Nightwatch.cli((argv) => {- const envString = environmentNames.join(',')- argv.e = envString- argv.env = envString- Nightwatch.CliRunner(argv)- .setup(null, () => {- bs_local.stop(() => {- process.exitCode = 0- })- })- .runTests(() => {- bs_local.stop(() => {- if (bs_local.pid && process) {- process.exitCode = 0- }- })- }).catch((err) => {- console.error(err)- process.exitCode = 1- })- })- })-} catch (ex) {- console.log('There was an error while starting the test runner:\n\n')- process.stderr.write(`${ex.stack}\n`)- process.exitCode = 1-}diff --git a/legacy/e2e/environments.js b/legacy/e2e/environments.jsdeleted file mode 100644index 9b6ae0cad..000000000--- a/legacy/e2e/environments.js+++ /dev/null@@ -1,35 +0,0 @@-const bsCapabilities = require('browserstack-capabilities')--const createCapabilities = bsCapabilities(- process.env.BROWSERSTACK_USER,- process.env.BROWSERSTACK_ACCESS_KEY-).create--const capabilities = createCapabilities(- [- {- browser: 'firefox',- browser_version: ['69.0'],- os: ['OS X'],- os_version: ['High Sierra']- },- {- browser: 'chrome',- browser_version: ['76.0'],- os: ['OS X'],- os_version: ['Mojave']- },- {- browser: 'ie',- browser_version: ['11.0'],- os: ['Windows'],- os_version: ['10']- }- ]-)--capabilities.forEach((capability) => {- capability.resolution = '1280x1024'-})--module.exports = capabilitiesdiff --git a/legacy/tasks/python3/util.py b/legacy/tasks/python3/util.pydeleted file mode 100755index 195456e34..000000000--- a/legacy/tasks/python3/util.py+++ /dev/null@@ -1,25 +0,0 @@-from copy import deepcopy--# dict_merge from-# http://blog.impressiver.com/post/31434674390/deep-merge-multiple-python-dicts-def dict_merge(target, *args):- # Merge multiple dicts- if len(args) > 1:- for obj in args:- dict_merge(target, obj)- return target-- # Recursively merge dicts and set non-dict values- obj = args[0]- if not isinstance(obj, dict):- return obj- for k, v in obj.items():-- if k in target and isinstance(target[k], dict):- if 'type' in v and 'type' in target[k]:- if v['type'] != target[k]['type']:- return target- dict_merge(target[k], v)- else:- target[k] = deepcopy(v)- return target\ No newline at end of filediff --git a/tasks/README.md b/tasks/README.mddeleted file mode 100644index bb94a368c..000000000--- a/tasks/README.md+++ /dev/null@@ -1,11 +0,0 @@-# tasks--A collection of tasks used in ci/cd and to build worldview locally.--## bamboo--## link-check--## build--## utildiff --git a/tasks/build-options/validateOptions.js b/tasks/build-options/validateOptions.jsindex c6ba67797..9fb6e3f3d 100644--- a/tasks/build-options/validateOptions.js+++ b/tasks/build-options/validateOptions.js@@ -91,24 +91,6 @@ async function main () {continue}}- // if (!layer.projections || !layer.projections.length) {- // error(`[${layerId}] No projections defined or not found in GC documents`)- // removeLayer(wv, layerId)- // continue- // }- // if (!layer.type) {- // error(`[${layerId}] No type defined. Possible to be expecting configuration via GC document but was not found`)- // removeLayer(wv, layerId)- // continue- // }- // if (layer.palette && !layer.palette.id) {- // error(`[${layerId}] No palette definition`)- // } else if (layer.palette) {- // const paletteId = layer.palette.id- // if (!fs.existsSync(path.join(configDir, `palettes/${paletteId}.json`))) {- // error(`[${layerId}] palette ${paletteId} not found in palettes folder`)- // }- // }if (layer.vectorStyle && !layer.vectorStyle.id) {error(`[${layerId}] No vectorStyle definition`)}@@ -120,23 +102,6 @@ async function main () {removeLayer(wv, layerId)continue}- // for (const projId of Object.keys(layer.projections)) {- // const projection = layer.projections[projId]- // if (projection.matrixSet) {- // const source = projection.source- // const matrixSet = projection.matrixSet- // if (!wv.sources[source]) {- // error(`[${layerId}:${projId}] Invalid source: ${source}`)- // delete layer.projections[projId]- // } else if (!wv.sources[source].matrixSets) {- // error(`[${layerId}:${projId}] No matrix sets for projection`)- // delete layer.projections[projId]- // } else if (!wv.sources[source].matrixSets[matrixSet]) {- // error(`[${layerId}:${projId}] Invalid matrix set: ${matrixSet}`)- // delete layer.projections[projId]- // }- // }- // }if ('temporal' in layer) {warn(`[${layerId}] GC Layer temporal values overwritten by Options`)layer = await processTemporalLayer(layer, layer.temporal)diff --git a/tasks/python3/extractConfigFromWMTS.py b/tasks/python3/extractConfigFromWMTS.pydeleted file mode 100755index 574915ddb..000000000--- a/tasks/python3/extractConfigFromWMTS.py+++ /dev/null@@ -1,290 +0,0 @@-#!/usr/bin/env python--from datetime import datetime, date, timedelta-import json-from optparse import OptionParser-import os-import sys-import xmltodict-import isodate-from processTemporalLayer import process_temporal-from collections import OrderedDict-import traceback--prog = os.path.basename(__file__)-base_dir = os.path.join(os.path.dirname(__file__), "..")-version = "2.0.0"-help_description = """\-Extracts configuration information from a WMTS GetCapabilities file, converts the XML to JSON-"""--parser = OptionParser(usage="Usage: %s <config_file> <input_dir> <output_dir>" % prog,- version="%s version %s" % (prog, version),- epilog=help_description)-(options, args) = parser.parse_args()--if len(args) != 3:- parser.error("Invalid number of arguments")--config_file = args[0]-input_dir = args[1]-output_dir = args[2]--with open(config_file, "r", encoding="utf-8") as fp:- config = json.load(fp)--tolerant = config.get("tolerant", False)-if not "wv-options-wmts" in config:- sys.exit(0)--if not os.path.exists(output_dir):- os.makedirs(output_dir)--entries = config["wv-options-wmts"]-skip = []-# Looks like this is for a list of layers needed to be skipped from the GC-if "skip" in config:- skip = config["skip"]--total_layer_count = 0-total_warning_count = 0-total_error_count = 0--json_options = {}-json_options["indent"] = 2-json_options["separators"] = (',', ': ')--class SkipException(Exception):- pass--def process_layer(gc_layer, wv_layers, entry):- ident = gc_layer["ows:Identifier"]- if ident in skip:- print("%s: skipping" % ident)- raise SkipException(ident)-- wv_layers[ident] = {}- wv_layer = wv_layers[ident]- wv_layer["id"] = ident- wv_layer["type"] = "wmts"- wv_layer["format"] = gc_layer["Format"]-- temporalForProjection = {}-- # Extract start and end dates- if "Dimension" in gc_layer:- dimension = gc_layer["Dimension"]- if dimension["ows:Identifier"] == "Time":- try:- wv_layer = process_temporal(wv_layer, dimension["Value"])-- # # TODO we need a better way to identify granule layers- # if ('Granule' in ident):- # temporalForProjection = {}- # temporalForProjection[entry["projection"]] = {- # "startDate": wv_layer["startDate"],- # "endDate": wv_layer["endDate"],- # "dateRanges": wv_layer["dateRanges"],- # }-- except Exception as e:- print(traceback.format_exc())- sys.stderr.write("%s: ERROR: [%s] %s\n" % (prog, ident, "Error processing time values."))-- # Extract matrix set- matrixSetLink = gc_layer["TileMatrixSetLink"]- matrixSet = matrixSetLink["TileMatrixSet"]-- projectionInfo = {- "source": entry["source"],- "matrixSet": matrixSet,- }-- wv_layer["projections"] = {}- if temporalForProjection.get(entry["projection"], None) is not None:- wv_layer["projections"][entry["projection"]] = {- **projectionInfo,- **temporalForProjection.get(entry["projection"], {})- }- wv_layer.pop("dateRanges")- wv_layer.pop("startDate")- wv_layer.pop("endDate")- else:- wv_layer["projections"][entry["projection"]] = { **projectionInfo }-- if "TileMatrixSetLimits" in matrixSetLink and matrixSetLink["TileMatrixSetLimits"] is not None:- matrixSetLimits = matrixSetLink["TileMatrixSetLimits"]["TileMatrixLimits"]- mappedSetLimits = []- for setLimit in matrixSetLimits:- mappedSetLimits.append({- "tileMatrix": setLimit["TileMatrix"],- "minTileRow": int(setLimit["MinTileRow"]),- "maxTileRow": int(setLimit["MaxTileRow"]),- "minTileCol": int(setLimit["MinTileCol"]),- "maxTileCol": int(setLimit["MaxTileCol"]),- })- wv_layer["projections"][entry["projection"]]["matrixSetLimits"] = mappedSetLimits;-- # Vector data links- if "ows:Metadata" in gc_layer and gc_layer["ows:Metadata"] is not None:- for item in gc_layer["ows:Metadata"]:- if "@xlink:role" not in item:- raise KeyError("No xlink:role")- schema_version = item["@xlink:role"]-- if schema_version == "http://earthdata.nasa.gov/gibs/metadata-type/layer/1.0":- vector_data_link = item["@xlink:href"]- vector_data_file = os.path.basename(vector_data_link)- vector_data_id = os.path.splitext(vector_data_file)[0]- wv_layer["vectorData"] = {- "id": vector_data_id- }-- # Colormap links- if "ows:Metadata" in gc_layer and gc_layer["ows:Metadata"] is not None:- if "skipPalettes" in config and ident in config["skipPalettes"]:- sys.stderr.write("%s: WARNING: Skipping palette for %s\n" % (- prog, ident))- global total_warning_count- total_warning_count += 1- else:- for item in gc_layer["ows:Metadata"]:- if "@xlink:role" not in item:- raise KeyError("No xlink:role")- schema_version = item["@xlink:role"]-- if schema_version == "http://earthdata.nasa.gov/gibs/metadata-type/colormap/1.3":- colormap_link = item["@xlink:href"]- colormap_file = os.path.basename(colormap_link)- colormap_id = os.path.splitext(colormap_file)[0]- wv_layer["palette"] = {- "id": colormap_id- }- elif schema_version == "http://earthdata.nasa.gov/gibs/metadata-type/mapbox-gl-style/1.0":- vectorstyle_link = item["@xlink:href"]- vectorstyle_file = os.path.basename(vectorstyle_link)- vectorstyle_id = os.path.splitext(vectorstyle_file)[0]- wv_layer["vectorStyle"] = {- "id": vectorstyle_id- }--def process_entry(entry):- layer_count = 0- warning_count = 0- error_count = 0- wv = {- "layers": {},- "sources": {}- }- wv_matrix_sets = {}- wv["sources"][entry["source"]] = {- "matrixSets": wv_matrix_sets- }-- input_file = os.path.join(input_dir, entry["from"])- gc_id = os.path.basename(input_file)- try:- with open(input_file) as fp:- xml = fp.read()- gc = xmltodict.parse(xml)- except Exception as e:- if tolerant:- sys.stderr.write("%s: WARN: [%s] Unable to get GC: %s\n" %- (prog, input_file, str(e)))- warning_count += 1- else:- sys.stderr.write("%s: ERROR: [%s] Unable to get GC: %s\n" %- (prog, input_file, str(e)))- error_count += 1- return [error_count, warning_count, layer_count]-- gc_contents = gc["Capabilities"]["Contents"]- wv_layers = wv["layers"]-- if gc_contents is None or "Layer" not in gc_contents:- error_count += 1- sys.stderr.write("%s: ERROR: [%s] %s\n" % (prog, gc_id, "No layers"))- return [error_count, warning_count, layer_count]-- if(type(gc["Capabilities"]["Contents"]["Layer"]) is OrderedDict):- gc_layer = gc["Capabilities"]["Contents"]["Layer"]- ident = gc_layer["ows:Identifier"]- try:- layer_count += 1- process_layer(gc_layer, wv_layers, entry)- except SkipException as se:- warning_count += 1- sys.stderr.write("%s: WARNING: [%s] Skipping\n" % (prog, ident))- except Exception as e:- error_count += 1- print(traceback.format_exc())- sys.stderr.write("%s: ERROR: [%s:%s] %s\n" % (prog, gc_id, ident, str(e)))- else:- for gc_layer in gc_contents["Layer"]:- ident = gc_layer["ows:Identifier"]- try:- layer_count += 1- process_layer(gc_layer, wv_layers, entry)- except SkipException as se:- warning_count += 1- sys.stderr.write("%s: WARNING: [%s] Skipping\n" % (prog, id))- except Exception as e:- error_count += 1- print(traceback.format_exc())- sys.stderr.write("%s: ERROR: [%s:%s] %s\n" % (prog, gc_id, ident, str(e)))-- def process_matrix_set(gc_matrix_set):- tileMatrixArr = gc_matrix_set["TileMatrix"]- ident = gc_matrix_set["ows:Identifier"]- zoom_levels = len(tileMatrixArr)- resolutions = []- formattedTileMatrixArr = []- max_resolution = entry["maxResolution"]- for zoom in range(0, zoom_levels):- resolutions = resolutions + [max_resolution / (2 ** zoom)]-- # We are assuming that width/heights are the same for every matrix and excluding- # the "TileWidth", "TileHeight" properties here- for tileMatrix in tileMatrixArr:- formattedTileMatrixArr.append({- "matrixWidth": int(tileMatrix["MatrixWidth"]),- "matrixHeight": int(tileMatrix["MatrixHeight"]),- })-- wv_matrix_sets[ident] = {- "id": ident,- "maxResolution": max_resolution,- "resolutions": resolutions,- "tileSize": [- int(tileMatrixArr[0]["TileWidth"]),- int(tileMatrixArr[0]["TileHeight"])- ],- "tileMatrices": formattedTileMatrixArr- }-- if(type(gc_contents["TileMatrixSet"]) is OrderedDict):- process_matrix_set(gc_contents["TileMatrixSet"])- else:- for gc_matrix_set in gc_contents["TileMatrixSet"]:- process_matrix_set(gc_matrix_set)-- output_file = os.path.join(output_dir, entry["to"])- with open(output_file, "w", encoding="utf-8") as fp:- json.dump(wv, fp, **json_options)- print("%s: %d error(s), %d warning(s), %d layers for %s" % (prog,- error_count, warning_count, layer_count, entry["source"]))- return [error_count, warning_count, layer_count]---# Main-for entry in entries:- error_count, warning_count, layer_count = process_entry(entry)- total_error_count += error_count- total_warning_count += warning_count- total_layer_count += layer_count--print("%s: %d error(s), %d warning(s), %d layers" % (prog, total_error_count,- total_warning_count, total_layer_count))--if total_error_count > 0:- sys.exit(1)diff --git a/tasks/python3/fetchPreviewSnapshots.py b/tasks/python3/fetchPreviewSnapshots.pydeleted file mode 100755index c36c07aaa..000000000--- a/tasks/python3/fetchPreviewSnapshots.py+++ /dev/null@@ -1,232 +0,0 @@-#!/usr/bin/env python--from concurrent.futures import ThreadPoolExecutor-from optparse import OptionParser-import requests-from datetime import datetime-from datetime import timedelta-import json-import os-import sys--prog = os.path.basename(__file__)-parser = OptionParser(usage="Usage: %s <wv.json> <overrides_file>" % prog)-(options, args) = parser.parse_args()-wv_json_file = args[0]-overrides_file = args[1]-features_file = args[2]--override_dates_dict = {}-bad_snapshots = []-total_success_count = 0-total_failure_count = 0-time_format = "%Y-%m-%dT%H:%M:%SZ"-snapshots_url = ''-param_dict = {- 'base': {- 'REQUEST': 'GetSnapshot',- 'FORMAT': 'image/jpeg'- },- 'geographic': {- 'BBOX': '-90,-180,90,180',- 'CRS': 'EPSG:4326',- 'WIDTH': '768',- 'HEIGHT': '384'- },- 'arctic': {- 'BBOX': '-4195000,-4195000,4195000,4195000',- 'CRS': 'EPSG:3413',- 'WIDTH': '512',- 'HEIGHT': '512'- },- 'antarctic': {- 'BBOX': '-4195000,-4195000,4195000,4195000',- 'CRS': 'EPSG:3031',- 'WIDTH': '512',- 'HEIGHT': '512'- }-}-# These layers should not be combined with the reference layer-standalone_layers = [- 'Graticule_15m',- 'Coastlines_15m',- 'Reference_Features_15m',- 'Reference_Labels_15m'-]-dest_img_dir = './web/images/layers/previews/'-reference_layers = {- 'geographic': 'OSM_Land_Water_Map',- 'arctic': 'OSM_Land_Water_Map',- 'antarctic': 'SCAR_Land_Water_Map'-}-current = datetime.now()---def track_bad_snapshots(layer_id, projection, request, img_file):- global bad_snapshots-- # File sizes with SCAR/OSM Land_Water_Map layer only- arctic_bad_size = 9949- antarctic_bad_size = 4060- geographic_bad_size = 12088- size = img_file.tell()-- if size in [ geographic_bad_size, arctic_bad_size, antarctic_bad_size ]:- bad_snapshots.append({- 'id': layer_id,- 'projection': projection,- 'url': request.url- })---def get_best_date(projection, period, date_ranges):- global current- last_range = date_ranges[len(date_ranges) - 1]- start_date = last_range.get('startDate')- end_date = last_range.get('endDate')- parsed_start_date = datetime.strptime(start_date, time_format)- parsed_end_date = datetime.strptime(end_date, time_format)- p_year = parsed_end_date.year- p_month = parsed_end_date.month- interval = int(last_range.get('dateInterval'))- altered_date = None-- # Handle daily layers- if (period == "daily"):- # Go back a few more days for single day layers since something- # too recent may not be processed yet- if (interval == 1):- interval = 3- altered_date = parsed_end_date - timedelta(days=interval)-- # Choose a good daylight month for arctic- if projection == "arctic" and p_month not in [4, 5, 6, 7, 8, 9]:- if p_year == current.year and current.month < 6:- altered_date = parsed_end_date.replace(day=1, month=6, year=current.year-1)- else:- altered_date = parsed_end_date.replace(day=1, month=6)-- # Choose a good daylight month for antarctic- if projection == "antarctic" and p_month not in [10, 11, 12, 1, 2]:- # TODO handle "bad" months for antarctic- altered_date = parsed_end_date.replace(month=12)-- # Make sure modified date isn't out of layer date range- if altered_date and altered_date >= parsed_start_date:- date = datetime.strftime(altered_date, time_format)- else:- date = end_date-- return date---def get_time_param(projection, layer_id, layer, params):- # Only include TIME param for temporal layers- date_ranges = layer.get('dateRanges')- start_date = layer.get('startDate')- period = layer.get('period')-- if (date_ranges):- params['TIME'] = get_best_date(projection, period, date_ranges)- elif (start_date):- params['TIME'] = start_date-- # Use any configured override dates- if (override_dates_dict.get(layer_id)):- params['TIME'] = override_dates_dict[layer_id]---def get_snapshots(layer):- global total_failure_count- global total_success_count- for projection, proj_dict in layer['projections'].items():- reference_layer = reference_layers[projection]-- # Sometimes a layer id is provided per projection (e.g. Land Mask layers)- # We need to use this layer id to request the layer from WVS/GIBS- # But, we need to use the WV id as the file name (since that's how we will look up the image in WV)- if (proj_dict.get('layer')):- gibs_layer_id = proj_dict.get('layer')- wv_layer_id = layer['id']- else:- gibs_layer_id = wv_layer_id = layer['id']-- params = { **param_dict['base'], **param_dict[projection] }- get_time_param(projection, wv_layer_id, layer, params)-- if (gibs_layer_id is not reference_layer and gibs_layer_id not in standalone_layers):- params['LAYERS'] = (reference_layer + ',' + gibs_layer_id)- params['OPACITIES'] = '0.50,1'- else:- params['LAYERS'] = gibs_layer_id-- dest_file_name = dest_img_dir + projection + '/' + wv_layer_id + '.jpg'-- # Only get images that we don't have already- if (os.path.exists(dest_file_name)):- continue-- try:- image_req = requests.get(snapshots_url, params=params)- if image_req.status_code == 200:- status_text = 'SUCCESS'- total_success_count += 1- with open(dest_file_name, 'xb') as image_file:- image_file.write(image_req.content)- if (gibs_layer_id == reference_layers[projection]):- continue- track_bad_snapshots(wv_layer_id, projection, image_req, image_file)- else:- total_failure_count += 1- status_text = 'ERROR'- print("\n%s: Result: %s - %s" % (prog, status_text, image_req.status_code))- print("%s: Layer: %s" % (prog, wv_layer_id))- print("%s: URL: %s" % (prog, image_req.url))-- except Exception as e:- print("%s ERROR: %s" % (prog, e))---if __name__ == "__main__":- # Check to see if this feature is enabled in features.json before continuing- with open(features_file, 'rt', encoding="utf-8") as features_json:- features_dict = json.load(features_json)- if features_dict['features'].get('previewSnapshots', False) is False:- sys.exit();-- # Allow manual configuration of layer ID to specific date to generate desired preview- with open(overrides_file, 'rt', encoding="utf-8") as overrides_json:- override_dates_dict = json.load(overrides_json)-- with open(wv_json_file, 'rt', encoding="utf-8") as wv_json:- wv_json_dict = json.load(wv_json)- layers = wv_json_dict['layers']- snapshots_url = wv_json_dict['features']['imageDownload']['url']- fetch_snapshots = features_dict['features'].get('previewSnapshots', False)- if not fetch_snapshots:- print("%s: Layer preview fetching disabled. Exiting." % prog)- sys.exit()-- futures = []- with ThreadPoolExecutor() as executor:- for layer in layers.values():- futures.append(executor.submit(get_snapshots, layer))- for f in futures:- try:- # Need to call result() on each future to catch any raised exceptions- f.result()- except Exception as e:- print("%s:" % (e))-- if len(bad_snapshots) > 0:- print("\n%s: WARNING: %s snapshots returned no content. See below for details: " % (prog, len(bad_snapshots)))- for bad_layer in bad_snapshots:- print("\n\t Layer: %s" % bad_layer['id'])- print("\t URL: %s" % (bad_layer['url']))-- if total_success_count > 0:- print('\n%s: Successfully retrieved %s snapshots!' % (prog, total_success_count))- if total_failure_count > 0:- print('\n%s: WARNING: Failed to retrieve %s snapshots!' % (prog, total_failure_count))- if total_failure_count == 0 and total_success_count == 0:- print('\n%s: No snapshots were retrieved. All layers found in wv.json have existing preview images!' % (prog))\ No newline at end of filediff --git a/tasks/python3/generateCategoryGroupOrder.py b/tasks/python3/generateCategoryGroupOrder.pydeleted file mode 100755index be3bd6427..000000000--- a/tasks/python3/generateCategoryGroupOrder.py+++ /dev/null@@ -1,31 +0,0 @@-#!/usr/bin/env python--from optparse import OptionParser-import os-import json-import sys--prog = os.path.basename(__file__)-parser = OptionParser(usage="Usage: %s <input_dir> <output_dir>" % prog)-(options, args) = parser.parse_args()--category_directory = args[0]-output_dir = args[1]--category_dict = {}--for root, dirs, files in os.walk(category_directory):- for file in files:- file_path = os.path.join(root, file)- with open(file_path, "r", encoding="utf-8") as category_config:- config = json.load(category_config)- try:- category = list(config['categories'])[0]- category_dict[category] = True- except Exception as e:- print("%s ERROR: Could not read category config. Check formatting of: %s" % (prog, file))- sys.exit(1)--with open(output_dir + '/categoryGroupOrder.json', "w", encoding="utf-8") as fp:- json.dump({ 'categoryGroupOrder': list(category_dict) }, fp, indent=2, sort_keys=True)- print("%s Successfully generated categoryGroupOrder.json with these categories: %s" % (prog, list(category_dict)))\ No newline at end of filediff --git a/tasks/python3/getCapabilities.py b/tasks/python3/getCapabilities.pydeleted file mode 100755index 9bb101e24..000000000--- a/tasks/python3/getCapabilities.py+++ /dev/null@@ -1,147 +0,0 @@-#!/usr/bin/env python--from optparse import OptionParser-from collections import OrderedDict-import os-import sys-import json-import xmltodict-import httpx-import asyncio--prog = os.path.basename(__file__)-base_dir = os.path.join(os.path.dirname(__file__), "..")-version = "1.0.0"-help_description = """\-Pulls GetCapabilities XML and linked metadata from configured locations-"""--parser = OptionParser(usage="Usage: %s <config> <output_dir>" % prog,- version="%s version %s" % (prog, version),- epilog=help_description)--(options, args) = parser.parse_args()-if len(args) != 2:- parser.error("Invalid number of arguments")--config_file = args[0]-output_dir = args[1]-colormaps = {}-vectorstyles = {}-vectordata = {}-colormaps_dir = os.path.join(output_dir, "colormaps")-vectorstyles_dir = os.path.join(output_dir, "vectorstyles")-vectordata_dir = os.path.join(output_dir, "vectordata")-error_count = 0-warning_count = 0--if not os.path.exists(output_dir):- os.makedirs(output_dir)--with open(config_file, "r", encoding="utf-8") as fp:- config = json.load(fp)--def handle_exception(e, link):- sys.stderr.write("\n %s: WARN: Unable to fetch %s %s" % (prog, link, str(e)))- global warning_count- warning_count += 1--def process_vector_data(layer):- if "ows:Metadata" in layer and layer["ows:Metadata"] is not None:- for item in layer["ows:Metadata"]:- schema_version = item["@xlink:role"]- if schema_version == "http://earthdata.nasa.gov/gibs/metadata-type/layer/1.0":- vector_data_link = item["@xlink:href"]- vector_data_file = os.path.basename(vector_data_link)- vector_data_id = os.path.splitext(vector_data_file)[0]- vectordata[vector_data_id] = vector_data_link--def process_layer(layer):- ident = layer["ows:Identifier"]- if "ows:Metadata" in layer:- if ident in config.get("skipPalettes", []):- sys.stderr.write("%s: WARN: Skipping palette for %s\n" % prog, ident)- global warning_count- warning_count += 1- elif layer["ows:Metadata"] is not None:- for item in layer["ows:Metadata"]:- schema_version = item["@xlink:role"]- if schema_version == "http://earthdata.nasa.gov/gibs/metadata-type/colormap/1.3":- colormap_link = item["@xlink:href"]- colormap_file = os.path.basename(colormap_link)- colormap_id = os.path.splitext(colormap_file)[0]- colormaps[colormap_id] = colormap_link-- elif schema_version == "http://earthdata.nasa.gov/gibs/metadata-type/mapbox-gl-style/1.0":- vector_style_link = item["@xlink:href"]- vector_style_file = os.path.basename(vector_style_link)- vector_style_id = os.path.splitext(vector_style_file)[0]- vectorstyles[vector_style_id] = vector_style_link--async def process_remote(client, entry):- try:- url = entry["from"]- print("%s: %s" % (prog, url))- response = await client.get(url)- contents = response.text- output_file = os.path.join(output_dir, entry["to"])-- # Write GetCapabilities responses to XML files- with open(output_file, "w", encoding="utf-8") as fp:- fp.write(contents)- gc = xmltodict.parse(contents)-- # Find all colormaps and vectorstyles in GetCapabilities responses and store them in memory- if gc["Capabilities"]["Contents"] is None:- print(('error: %s: no layers' % url))- return-- layers = gc["Capabilities"]["Contents"]["Layer"]--- if(type(layers) is OrderedDict):- process_layer(layers)- process_vector_data(layers)- else:- [process_layer(layer) for layer in layers]- [process_vector_data(layer) for layer in layers]-- except Exception as e:- print(('ERROR: %s: %s' % (url, str(e))))--async def process_metadata(client, link, dir, ext):- try:- response = await client.get(link)- contents = response.text- if link.endswith(ext):- output_file = os.path.join(dir, os.path.basename(link))- with open(output_file, "w", encoding="utf-8") as fp:- fp.write(contents)- except Exception as e:- handle_exception(e, link)--async def gather_process(type, typeStr, client, dir, ext):- print("%s: Fetching %d %s" % (prog, len(type), typeStr))- sys.stdout.flush()- if not os.path.exists(dir):- os.makedirs(dir)- await asyncio.gather(*[process_metadata(client, link, dir, ext) for link in list(type.values())])--async def main():- if "wv-options-fetch" in config:- limits = httpx.Limits(max_keepalive_connections=10, max_connections=10)- async with httpx.AsyncClient(limits=limits, timeout=20.0) as client:- await asyncio.gather(*[process_remote(client, entry) for entry in config["wv-options-fetch"]])- if colormaps:- await gather_process(colormaps, 'colormaps', client, colormaps_dir, '.xml')- if vectorstyles:- await gather_process(vectorstyles, 'vectorstyles', client, vectorstyles_dir, '.json')- if vectordata:- await gather_process(vectordata, 'vectordata', client, vectordata_dir, '.json')--asyncio.run(main())--print("%s: %d error(s)" % (prog, error_count))--if error_count > 0:- sys.exit(1)diff --git a/tasks/python3/getVisMetadata.py b/tasks/python3/getVisMetadata.pydeleted file mode 100755index dab91ac68..000000000--- a/tasks/python3/getVisMetadata.py+++ /dev/null@@ -1,103 +0,0 @@-#!/usr/bin/env python--from optparse import OptionParser-import os-import json-import httpx-import asyncio--prog = os.path.basename(__file__)-parser = OptionParser(usage="Usage: %s <config> <output_dir>" % prog)-(options, args) = parser.parse_args()-prog = os.path.basename(__file__)--features_file = args[0]-layer_order_file = args[1]-output_file = args[2]--# These are alias or otherwise layers that don't exist in GIBS-skip_layers = [- 'Land_Water_Map',- 'Land_Mask',- 'World_Database_on_Protected_Areas',-]--# NOTE: Only using these properties at this time-use_keys = [- 'conceptIds',- 'dataCenter',- 'daynight',- 'orbitTracks',- 'orbitDirection',- 'ongoing',- 'layerPeriod',- 'title',- 'subtitle'-]-layer_metadata = {}-daacMap = {}-failed_requests = []--def get_daac(metadata):- if metadata.get('conceptIds', None) is None:- return metadata- for collection in metadata.get("conceptIds"):- origDataCenter = collection.get("dataCenter", None)- dataCenter = daacMap.get(origDataCenter, None)- collection.pop('dataCenter', None)- if dataCenter is None:- continue- if metadata.get("dataCenter", None) is None:- metadata["dataCenter"] = [dataCenter]- elif dataCenter not in metadata["dataCenter"]:- metadata["dataCenter"].append(dataCenter)- return metadata--async def get_metadata(client, layer_id, base_url):- try:- response = await client.get(base_url + layer_id + '.json')- metadata = response.json()- layer_metadata[layer_id] = get_daac(metadata)-- # Remove any props we don't expect to use- metadata_keys = dict(layer_metadata[layer_id]).keys()- for key in metadata_keys:- if key not in use_keys:- layer_metadata[layer_id].pop(key, None)-- except Exception as e:- global failed_requests- if layer_id not in failed_requests:- failed_requests.append(layer_id)- print('%s WARNING: Failed to retrieve metadata config for [%s], will retry...' % (prog, layer_id))- else:- print('%s WARNING: Failed to retrieve metadata config for [%s] on second attempt' % (prog, layer_id))--async def main(url):- global skip_layers-- with open(layer_order_file, 'rt', encoding="utf-8") as layer_order:- layer_ids = json.load(layer_order).get('layerOrder')- filtered_ids = list(filter(lambda l: l not in skip_layers, layer_ids))- print('%s: Pulling vis metadata for %s layers... ' % (prog, len(filtered_ids)))-- limits = httpx.Limits(max_keepalive_connections=10, max_connections=10)- async with httpx.AsyncClient(limits=limits, timeout=10.0) as client:- await asyncio.gather(*[get_metadata(client, layer_id, url) for layer_id in filtered_ids])- await asyncio.gather(*[get_metadata(client, layer_id, url) for layer_id in failed_requests])-- with open(output_file, "w", encoding="utf-8") as fp:- # Format of this object will determine how this data is combined into wv.json- json.dump({ 'layers': layer_metadata}, fp, indent=2, sort_keys=True)--#MAIN-if __name__ == "__main__":- with open(features_file, 'rt', encoding="utf-8") as features:- metadata_config = json.load(features).get('features').get('vismetadata')- if metadata_config is not None:- url = metadata_config.get('url')- daacMap = metadata_config.get('daacMap', {})- asyncio.run(main(url))- else:- print('%s: Visualization metadata not configured. Exiting.' % (prog))-diff --git a/tasks/python3/mergeConfig.py b/tasks/python3/mergeConfig.pydeleted file mode 100755index 74b10c764..000000000--- a/tasks/python3/mergeConfig.py+++ /dev/null@@ -1,53 +0,0 @@-#!/usr/bin/env python--from util import dict_merge-from optparse import OptionParser-import os-import sys-import json--prog = os.path.basename(__file__)-base_dir = os.path.join(os.path.dirname(__file__), "..")-version = "2.0.0"-help_description = """\-Concatenates all configuration items a directory into one configuration file.-"""--# MAIN-parser = OptionParser(usage="Usage: %s <input_dir> <output_file>" % prog,- version="%s version %s" % (prog, version),- epilog=help_description)--(options, args) = parser.parse_args()-if len(args) != 2:- parser.error("Invalid number of arguments")--input_dir = args[0]-output_file = args[1]--conf = {}-file_count = 0-for root, dirs, files in os.walk(input_dir):- dirs.sort()- for file in sorted(files):- try:- if not file.endswith(".json"):- continue- file_count += 1- with open(os.path.join(root, file), "r", encoding="utf-8") as fp:- data = json.load(fp)- dict_merge(conf, data)- except Exception as e:- sys.stderr.write("ERROR: %s: %s\n" %- (os.path.join(root, file), str(e)))- sys.exit(1)--json_options = {}-json_options["indent"] = 2-json_options["separators"] = (',', ': ')--with open(output_file, "w", encoding="utf-8") as fp:- json.dump(conf, fp, **json_options)--print("%s: %s file(s) merged into %s" % (prog, file_count,- os.path.basename(output_file)))diff --git a/tasks/python3/mergeConfigWithWMTS.py b/tasks/python3/mergeConfigWithWMTS.pydeleted file mode 100755index da4510e29..000000000--- a/tasks/python3/mergeConfigWithWMTS.py+++ /dev/null@@ -1,84 +0,0 @@-#!/usr/bin/env python--from util import dict_merge-from copy import deepcopy-from optparse import OptionParser-import os-import sys-import json--prog = os.path.basename(__file__)-base_dir = os.path.join(os.path.dirname(__file__), "..")-version = "2.0.0"-help_description = """\-Concatenates all configuration items a directory into one configuration file.-"""--def wmts_dict_merge(target, *args):- # Merge multiple dicts- # Recursively merge dicts and set non-dict values- obj = args[0]['layers']- conf = args[1]['layers']-- if not isinstance(obj, dict):- return obj- for k, v in obj.items():- if k in conf and isinstance(conf[k], dict):- foundSourceMisMatch = False- if 'projections' in v and 'projections' in conf[k]:- conf_projections = conf[k]['projections']- for projectionKey, projection in v['projections'].items():- source = projection['source']- if projectionKey in conf_projections:- if 'source' in conf_projections[projectionKey]:- if source != conf_projections[projectionKey]['source']:- foundSourceMisMatch = True- continue- if foundSourceMisMatch:- continue- if k in target and isinstance(target[k], dict):- dict_merge(target[k], v, conf[k])- else:- target[k] = dict_merge(v, conf[k])- return target--# MAIN-parser = OptionParser(usage="Usage: %s <input_dir> <output_file>" % prog,- version="%s version %s" % (prog, version),- epilog=help_description)--(options, args) = parser.parse_args()-if len(args) != 2:- parser.error("Invalid number of arguments")-input_dir = args[0]-output_file = args[1]-new_conf = {}-new_conf["layers"] = {}-new_conf["sources"] = {}--with open(output_file, "r", encoding="utf-8") as fp:- output_data = json.load(fp)-file_count = 0-for file in os.listdir(input_dir):- try:- if not file.endswith(".json"):- continue- file_count += 1- with open(os.path.join(input_dir, file), "r", encoding="utf-8") as fp:- data = json.load(fp)- new_conf['layers'] = wmts_dict_merge(new_conf['layers'], data, output_data)- new_conf["sources"] = dict_merge(new_conf["sources"], data['sources'], output_data['sources'])- except Exception as e:- sys.stderr.write("ERROR: %s: %s\n" %- (os.path.join(input_dir, file), str(e)))- sys.exit(1)-new_conf = dict_merge(new_conf, output_data)-json_options = {}-json_options["indent"] = 2-json_options["separators"] = (',', ': ')--with open(output_file, "w", encoding="utf-8") as fp:- json.dump(new_conf, fp, **json_options)--print("%s: %s file(s) merged into %s" % (prog, file_count,- os.path.basename(output_file)))diff --git a/tasks/python3/processColormap.py b/tasks/python3/processColormap.pydeleted file mode 100755index 5d15709e9..000000000--- a/tasks/python3/processColormap.py+++ /dev/null@@ -1,200 +0,0 @@-#!/usr/bin/env python--from concurrent.futures import ProcessPoolExecutor-from optparse import OptionParser-import os-import re-import sys-import json-import xmltodict--prog = os.path.basename(__file__)-base_dir = os.path.join(os.path.dirname(__file__), "..")-version = "3.0.0"-help_description = """\-Converts colormaps to JSON files-"""--parser = OptionParser(usage="Usage: %s <config_file> <input_dir> <output_dir>" % prog,- version="%s version %s" % (prog, version),- epilog=help_description)--(options, args) = parser.parse_args()-if len(args) != 3:- parser.error("Invalid number of arguments")--config_file = args[0]-input_dir = args[1]-output_dir = args[2]-with open(config_file, "r", encoding="utf-8") as fp:- config = json.load(fp)-skips = config.get("skipPalettes", [])--def to_list(v):- return v if isinstance(v, list) else [v]--def match_legend(entry, legends):- try:- matched = "false"- for legend in legends:- if "@id" not in legend:- raise KeyError("No legend IDs")- if legend["@id"] == entry["@ref"]:- matched = legend- return matched- except ValueError as e:- raise ValueError("Invalid reference: %s" % entry["@ref"])--def process_entries(colormap):- entries = to_list(colormap["Entries"]["ColorMapEntry"])-- transparent_map = "true"-- for entry in entries:- if entry["@transparent"] == "false":- transparent_map = "false"- if (transparent_map == "true"):- return "transparent"-- if "Legend" not in colormap:- raise KeyError("No Legend")- else:- map_type = colormap["Legend"]["@type"]- legends = to_list(colormap["Legend"]["LegendEntry"])- colors = []- values = []- ticks = []- tooltips = []- legend_colors = []- refs_list = []- ref_skip_list = []- color_format = "{0:02x}{1:02x}{2:02x}{3:02x}"- for index, entry in enumerate(entries):- legend = match_legend(entry, legends)- if (legend == "false"):- ref_skip_list += [entry['@ref']]- continue- r,g,b = entry["@rgb"].split(",")- a = 0 if entry.get("@transparent", "false") == "true" else 255- if a == 0:- ref_skip_list += [entry['@ref']]- continue- if "@ref" not in entry:- raise KeyError("No ref in legend")- refs_list += [entry['@ref']]- colors += [color_format.format(int(r), int(g), int(b), a)]- if (map_type == "continuous") or (map_type == "discrete"):- items = re.sub(r"[\(\)\[\]]", "", entry["@value"]).split(",")- try:- new_items = []- for item in items:- v = float(item)- if v == float("inf"):- v = sys.float_info.max- if v == float("-inf"):- v = sys.float_info.min- new_items += [v]- values += [new_items]- except ValueError as e:- raise ValueError("Invalid value: %s" % entry["@value"])- skip_index = 0- id_list = []- for index, entry in enumerate(legends):- if entry['@id'] in ref_skip_list:- skip_index = skip_index + 1- continue- r,g,b = entry["@rgb"].split(",")- legend_colors += [color_format.format(int(r), int(g), int(b), 255)]- if "@tooltip" not in entry:- raise KeyError("No tooltips in legend")- tooltips += [entry["@tooltip"]]- if "@id" not in entry:- raise KeyError("No id in legend")- id_list += [entry['@id']]- if "@showTick" in entry:- ticks += [index - skip_index]-- result = {- "type": map_type,- "entries": {- "type": map_type,- "colors": colors,- "refs": refs_list- },- "legend": {- "colors": legend_colors,- "type": map_type,- "tooltips": tooltips,- "ticks": ticks,- "refs": id_list- }- }- if (map_type == "continuous") or (map_type == "discrete"):- result["entries"]["values"] = values-- return result--def read_file(file):- id = os.path.splitext(os.path.basename(file))[0]- if id in skips:- sys.stderr.write("%s: WARN: [%s] %s\n" % (prog, file), "Skipping")- return- with open(file, "r", encoding="utf-8") as fp:- xml = fp.read()- return (id, xml)--def process_file(id, xml):- document = xmltodict.parse(xml)- colormaps = to_list(document["ColorMaps"]["ColorMap"])- maps = []- for colormap in colormaps:- result = process_entries(colormap)- if result == "transparent":- continue- result["title"] = colormap["@title"]- result["entries"]["title"] = colormap["@title"]- result["legend"]["title"] = colormap["@title"]- result["legend"]["id"] = id + "_" + str(len(maps)) + "_legend"- if "@minLabel" in colormap["Legend"]:- result["legend"]["minLabel"] = colormap["Legend"]["@minLabel"]- if "@maxLabel" in colormap["Legend"]:- result["legend"]["maxLabel"] = colormap["Legend"]["@maxLabel"]- if "@units" in colormap:- result["legend"]["units"] = colormap["@units"]- maps += [result]-- data = {- "id": id,- "maps": maps- }- json_options = {}- json_options["indent"] = 2- json_options["separators"] = (',', ': ')-- output_file = os.path.join(output_dir, id + ".json")- with open(output_file, "w", encoding="utf-8") as fp:- json.dump(data, fp, **json_options)--# Main-file_count = 0-error_count = 0-futures = []-if __name__ == "__main__":- with ProcessPoolExecutor() as readExecutor:- for root, dirs, files in os.walk(input_dir):- for file in files:- input_file = os.path.join(root, file)- futures.append(readExecutor.submit(read_file, input_file))- with ProcessPoolExecutor() as writeExecutor:- for future in futures:- try:- file_count += 1- (id, xml) = future.result()- writeExecutor.submit(process_file, id, xml)- except Exception as e:- sys.stderr.write("%s: ERROR: [%s] %s\n" % (prog, id, str(e)))- error_count += 1- print("%s: %d error(s), %d file(s)" % (prog, error_count, file_count))--if error_count > 0:- sys.exit(1)diff --git a/tasks/python3/processTemporalLayer.py b/tasks/python3/processTemporalLayer.pydeleted file mode 100755index 626ccea7c..000000000--- a/tasks/python3/processTemporalLayer.py+++ /dev/null@@ -1,79 +0,0 @@-from datetime import datetime-import isodate-import re-import traceback--def to_list(val):- return [val] if not hasattr(val, 'reverse') else val--# Add duration to end date using-# ISO 8601 duration keys-def determine_end_date(key, date):- return date + isodate.parse_duration(key)--# This method takes a layer and a temporal-# value and tranlates it to start and end dates-def process_temporal(wv_layer, value):- dateFormat = "%Y-%m-%d"- timeFormat = "%H:%M:%S"- dateTimeFormat = "%Y-%m-%d %H:%M:%S"- try:- ranges = to_list(value)- if "T" in ranges[0]:- wv_layer["period"] = "subdaily"-- else:- if ranges[0].endswith("Y"):- wv_layer["period"] = "yearly"- elif ranges[0].endswith("M"):- wv_layer["period"] = "monthly"- else:- wv_layer["period"] = "daily"- start_date = datetime.max- end_date = datetime.min- date_range_start, date_range_end, range_interval = [], [], []-- for range in ranges:- start, end, interval = range.split('/')- if wv_layer["period"] == "daily" \- or wv_layer["period"] == "monthly" \- or wv_layer["period"] == "yearly":- start_date = min(start_date, datetime.strptime(start, dateFormat))- end_date = max(end_date, datetime.strptime(end, dateFormat))- if start_date:- startDateParse = datetime.strptime(start, dateFormat)- date_range_start.append(startDateParse.strftime(dateFormat) + "T" + startDateParse.strftime(timeFormat) + "Z")- if end_date:- endDateParse = datetime.strptime(end, dateFormat)- date_range_end.append(endDateParse.strftime(dateFormat) + "T" + endDateParse.strftime(timeFormat) + "Z")- if interval != "P1D":- end_date = determine_end_date(interval, end_date)- range_interval.append(re.search(r'\d+', interval).group())-- # Subdaily Layers- else:- startTime = start.replace('T', ' ').replace('Z', '')- endTime = end.replace('T', ' ').replace('Z', '')- start_date = min(start_date, datetime.strptime(startTime, dateTimeFormat))- end_date = max(end_date, datetime.strptime(endTime, dateTimeFormat))-- if start_date:- startTimeParse = datetime.strptime(startTime, dateTimeFormat)- date_range_start.append(startTimeParse.strftime(dateFormat) + "T" + startTimeParse.strftime(timeFormat) + "Z")- if end_date:- endTimeParse = datetime.strptime(endTime, dateTimeFormat)- date_range_end.append(endTimeParse.strftime(dateFormat) + "T" + endTimeParse.strftime(timeFormat) + "Z")-- range_interval.append(re.search(r'\d+', interval).group())-- wv_layer["startDate"] = start_date.strftime(dateFormat) + "T" + start_date.strftime(timeFormat) + "Z"- if end_date != datetime.min:- wv_layer["endDate"] = end_date.strftime(dateFormat) + "T" + end_date.strftime(timeFormat) + "Z"- if date_range_start and date_range_end:- wv_layer["dateRanges"] = [{"startDate": s, "endDate": e, "dateInterval": i} for s, e, i in zip(date_range_start, date_range_end, range_interval)]- except ValueError:- raise Exception("Invalid time: {0}".format(range))- except Exception as e:- print(traceback.format_exc())- raise Exception("Error processing temporal layer: {0}".format(e))- return wv_layerdiff --git a/tasks/python3/processVectorData.py b/tasks/python3/processVectorData.pydeleted file mode 100755index 3c77d7bec..000000000--- a/tasks/python3/processVectorData.py+++ /dev/null@@ -1,63 +0,0 @@-#!/usr/bin/env python--from optparse import OptionParser-import os-import sys-import json-import shutil--prog = os.path.basename(__file__)-base_dir = os.path.join(os.path.dirname(__file__), "..")-version = "3.0.0"-help_description = """\-Converts vector data to JSON files-"""--parser = OptionParser(usage="Usage: %s <config_file> <vectordata_input_dir> <output_dir>" % prog,- version="%s version %s" % (prog, version),- epilog=help_description)--(options, args) = parser.parse_args()-if len(args) != 3:- parser.error("Invalid number of arguments")--config_file = args[0]-vectordata_input_dir = args[1]-output_dir = args[2]--with open(config_file, "r", encoding="utf-8") as fp:- config = json.load(fp)--def copy_file(file):- input_file = os.path.join(root, file)- if input_file.endswith('.json'):- response_data = {}- vector_layer_filename = file;- vector_layer_id = vector_layer_filename.split(".json", 1)[0]- response_data["vectorData"] = {}- response_data["vectorData"][vector_layer_id] = {}- with open(input_file, "r", encoding="utf-8") as json_file:- initial_data = json.load(json_file)- for i in initial_data:- response_data["vectorData"][vector_layer_id][i] = initial_data[i]- with open(input_file, "w", encoding="utf-8") as json_file:- json.dump(response_data, json_file,indent=2)- shutil.copy(input_file, output_dir)--# Main-file_count = 0-error_count = 0--for root, dirs, files in os.walk(vectordata_input_dir):- for file in files:- try:- file_count += 1- copy_file(file)- except Exception as e:- sys.stderr.write("%s: ERROR: [%s] %s\n" % (prog, file, str(e)))- error_count += 1--print("%s: %d error(s), %d file(s)" % (prog, error_count, file_count))--if error_count > 0:- sys.exit(1)diff --git a/tasks/python3/processVectorStyles.py b/tasks/python3/processVectorStyles.pydeleted file mode 100755index 57c29a70d..000000000--- a/tasks/python3/processVectorStyles.py+++ /dev/null@@ -1,63 +0,0 @@-#!/usr/bin/env python--from optparse import OptionParser-import os-import sys-import json-import shutil--prog = os.path.basename(__file__)-base_dir = os.path.join(os.path.dirname(__file__), "..")-version = "3.0.0"-help_description = """\-Converts vector data to JSON files-"""--parser = OptionParser(usage="Usage: %s <config_file> <vectordata_input_dir> <output_dir>" % prog,- version="%s version %s" % (prog, version),- epilog=help_description)--(options, args) = parser.parse_args()-if len(args) != 3:- parser.error("Invalid number of arguments")--config_file = args[0]-vectordata_input_dir = args[1]-output_dir = args[2]--with open(config_file, "r", encoding="utf-8") as fp:- config = json.load(fp)--def copy_file(file):- input_file = os.path.join(root, file)- if input_file.endswith('.json'):- response_data = {}- vector_layer_filename = file;- vector_layer_id = vector_layer_filename.split(".json", 1)[0]- response_data["vectorStyles"] = {}- response_data["vectorStyles"][vector_layer_id] = {}- with open(input_file, "r", encoding="utf-8") as json_file:- initial_data = json.load(json_file)- for i in initial_data:- response_data["vectorStyles"][vector_layer_id][i] = initial_data[i]- with open(input_file, "w", encoding="utf-8") as json_file:- json.dump(response_data, json_file,indent=2)- shutil.copy(input_file, output_dir)--# Main-file_count = 0-error_count = 0--for root, dirs, files in os.walk(vectordata_input_dir):- for file in files:- try:- file_count += 1- copy_file(file)- except Exception as e:- sys.stderr.write("%s: ERROR: [%s] %s\n" % (prog, file, str(e)))- error_count += 1--print("%s: %d error(s), %d file(s)" % (prog, error_count, file_count))--if error_count > 0:- sys.exit(1)diff --git a/tasks/python3/validateConfigs.py b/tasks/python3/validateConfigs.pydeleted file mode 100755index 3b3872fdf..000000000--- a/tasks/python3/validateConfigs.py+++ /dev/null@@ -1,48 +0,0 @@-#!/usr/bin/env python--from jsonschema import Draft7Validator-from optparse import OptionParser-import os-import json-import glob-import sys---prog = os.path.basename(__file__)-parser = OptionParser(usage="Usage: %s <input_directory> <schema_file>" % prog)-(options, args) = parser.parse_args()-input_directory = args[0]-schema_file = args[1]--layerConfigFiles = []-invalidJsonFiles = []--print('%s: Validating layer configs...' % (prog))--layerConfigSchema = json.load(open(schema_file))--files = glob.glob(input_directory + '/**/*.json', recursive=True)--for file_path in files:- configJson = json.load(open(file_path))- # Validate against the JSON schema- val = Draft7Validator(layerConfigSchema)- errors = sorted(val.iter_errors(configJson), key=lambda e: e.path)- split_path = file_path.split('/')- name_index = len(split_path) - 1- file_name = split_path[name_index]-- if len(errors):- invalidJsonFiles.append(file_name)- print('%s: ERROR: %s' % (prog, file_name))-- for err in errors:- print('%s: - %s' % (prog, err.message))--if len(invalidJsonFiles) > 0:- print('%s: FAILED: %s layer configs failed validation.' % (prog, len(invalidJsonFiles)))- sys.exit(1)-else:- print('%s: PASSED: All layer configs passed validation!' % (prog))--diff --git a/tasks/python3/validateOptions.py b/tasks/python3/validateOptions.pydeleted file mode 100755index 51434a153..000000000--- a/tasks/python3/validateOptions.py+++ /dev/null@@ -1,197 +0,0 @@-#!/usr/bin/env python--from datetime import datetime, date, timedelta-import json-from optparse import OptionParser-import os-import sys-import time-from processTemporalLayer import process_temporal--prog = os.path.basename(__file__)-base_dir = os.path.join(os.path.dirname(__file__), "..")-version = "1.0.0"-help_description = """\-Validates and corrects the configuration files.-"""--parser = OptionParser(usage="Usage: %s <config_json> <config_dir>" % prog,- version="%s version %s" % (prog, version),- epilog=help_description)-(options, args) = parser.parse_args()--if len(args) != 2:- parser.error("Invalid number of arguments")--options_file = args[0]-config_dir = args[1]--error_count = 0-warning_count = 0-removed_layers = []--main_config_file = os.path.join(config_dir, "wv.json")-with open(main_config_file, "r", encoding="utf-8") as fp:- wv = json.load(fp)-with open(options_file, "r", encoding="utf-8") as fp:- opt = json.load(fp)--tolerant = opt.get("tolerant", False)--def fail(message):- global error_count- sys.stderr.write("%s: ERROR: %s\n" % (prog, message))- error_count += 1--def error(message):- warn(message) if tolerant else fail(message)--def warn(message):- global warning_count- sys.stderr.write("%s: WARN: %s\n" % (prog, message))- warning_count += 1--def remove_layer(wv, layer_id):- global removed_layers- removed_layers.append(layer_id);- del wv["layers"][layer_id]- if layer_id in wv["layerOrder"]: wv["layerOrder"].remove(layer_id)--def isDateTimeFormat(input):- try:- datetime.strptime(input, '%Y-%m-%d %H:%M:%S')- return True- except ValueError:- return False--if tolerant:- warn("Validation enforcement disabled")--start_date = datetime.max--for layer_id in list(wv["layers"].keys()):- layer = wv["layers"][layer_id]-- if layer_id != layer.get('id'):- error("[%s] layer id does not match id of %s" % (layer_id, layer.get('id')))- if layer_id not in wv["layerOrder"]:- if layer_id in opt.get("layerOrderExceptions", []):- remove_layer(wv, layer_id)- continue- elif tolerant or opt.get("ignoreLayerOrder", False):- wv["layerOrder"] += [layer_id]- else:- remove_layer(wv, layer_id)- continue- if "projections" not in layer or len(layer["projections"]) == 0:- error("[%s] No projections defined or not found in GC documents" %- layer_id)- remove_layer(wv, layer_id)- continue- if "type" not in layer:- error("[" + layer_id + "] No type defined. Possible to be expecting " +- "configuration via GC document but was not found")- remove_layer(wv, layer_id)- continue- if "palette" in layer and "id" not in layer["palette"]:- error("[%s] No palette definition" % (layer_id))- elif "palette" in layer:- palette_id = layer["palette"]["id"]- if not os.path.exists(os.path.join(config_dir, "palettes",- palette_id + ".json")):- error("[%s] Unknown palette: %s" % (layer_id, palette_id))- del layer["palette"]- if "vectorStyle" in layer and "id" not in layer["vectorStyle"]:- error("[%s] No vectorStyle definition" % (layer_id))- if "group" not in layer and opt.get("warnOnUnexpectedLayer"):- error("[%s] Possible unexpected layer, no group defined" % layer_id)- remove_layer(wv, layer_id)- continue- elif "group" not in layer:- remove_layer(wv, layer_id)- continue- for proj_id in list(layer["projections"].keys()):- projection = layer["projections"][proj_id]- if "matrixSet" in projection:- source = projection["source"]- matrix_set = projection["matrixSet"]- if source not in wv["sources"]:- error("[%s:%s] Invalid source: %s" %- (layer_id, proj_id, source))- del layer["projections"][proj_id]- elif "matrixSets" not in wv["sources"][source]:- error("[%s:%s] No matrix sets for projection" %- (layer_id, proj_id))- del layer["projections"][proj_id]- elif matrix_set not in wv["sources"][source]["matrixSets"]:- error("[%s:%s] Invalid matrix set: %s" %- (layer_id, proj_id, matrix_set))- del layer["projections"][proj_id]- if "temporal" in layer:- warn("[%s] GC Layer temporal values overwritten by Options" % layer_id)- layer = process_temporal(layer, layer["temporal"])- if layer.get("futureTime"):- if "endDate" in layer:- del layer["endDate"]-- if layer.get("futureTime", False):- pass- else:- if "endDate" in layer and layer.get("ongoing", True):- del layer["endDate"]- if "startDate" in layer:- startTime = layer["startDate"].replace('T', ' ').replace('Z', '')- if isDateTimeFormat(startTime):- d = datetime.strptime(startTime, "%Y-%m-%d %H:%M:%S")- else:- d = datetime.strptime(layer["startDate"], "%Y-%m-%d")- start_date = min(start_date, d)--if start_date != datetime.max:- wv["startDate"] = start_date.strftime("%Y-%m-%d") + "T" + start_date.strftime("%H:%M:%S") + "Z"--for layer_id in wv["layerOrder"]:- if layer_id not in wv["layers"]:- error("[%s] In layer order but no definition" % layer_id)--startingLayers = []-for startingLayer in wv["defaults"]["startingLayers"]:- if startingLayer["id"] not in wv["layers"]:- error("Invalid starting layer: %s" % (startingLayer["id"]))- else:- startingLayers += [startingLayer]-wv["defaults"]["startingLayers"] = startingLayers-wv["buildDate"] = int(round(time.time() * 1000))--for projection, projectionValue in wv["naturalEvents"]["layers"].items():- for eventType, eventTypeLayerList in projectionValue.items():- for layerObject in eventTypeLayerList:- if layerObject[0] not in wv["layers"]:- error("The %s layer in the Natural events %s %s config does not have a matching ID in the layer config" %- (layerObject[0], projection, eventType))--for measurement in list(wv["measurements"].values()):- if "sources" not in measurement:- continue- for source in list(measurement["sources"].values()):- if "settings" not in source:- continue- for setting in source["settings"]:- if setting not in wv["layers"]:- error("In measurement %s, source %s, layer not found: %s" %- (measurement["id"], source["id"], setting))--remove_count = len(removed_layers)-print("%s: %d error(s), %d warning(s), %d removed" % (prog, error_count, warning_count, remove_count))-if remove_count > 0:- print(removed_layers)--json_options = {}-json_options["indent"] = 2-json_options["separators"] = (',', ': ')--with open(main_config_file, "w", encoding="utf-8") as fp:- json.dump(wv, fp)--if error_count > 0:- sys.exit(1)