Interactive interface for browsing global, full-resolution satellite imagery
add explicit utf-8 encoding
Ed Plato26f001db046b
diff --git a/tasks/python3/copyVectorStyles.py b/tasks/python3/copyVectorStyles.pyindex 50ea56652..f913702d1 100755--- a/tasks/python3/copyVectorStyles.py+++ b/tasks/python3/copyVectorStyles.py@@ -25,7 +25,7 @@ config_file = args[0]vectorstyles_input_dir = args[1]output_dir = args[2]-with open(config_file) as fp:+with open(config_file, "r", encoding="utf-8") as fp:config = json.load(fp)skips = config.get("skipPalettes", [])diff --git a/tasks/python3/extractConfigFromWMTS.py b/tasks/python3/extractConfigFromWMTS.pyindex c7e8d6705..155e563c9 100755--- a/tasks/python3/extractConfigFromWMTS.py+++ b/tasks/python3/extractConfigFromWMTS.py@@ -29,7 +29,7 @@ config_file = args[0]input_dir = args[1]output_dir = args[2]-with open(config_file) as fp:+with open(config_file, "r", encoding="utf-8") as fp:config = json.load(fp)tolerant = config.get("tolerant", False)@@ -241,7 +241,7 @@ def process_entry(entry):process_matrix_set(gc_matrix_set)output_file = os.path.join(output_dir, entry["to"])- with open(output_file, "w") as fp:+ with open(output_file, "w", encoding="utf-8") as fp:json.dump(wv, fp, **json_options)print("%s: %d error(s), %d warning(s), %d layers for %s" % (prog,error_count, warning_count, layer_count, entry["source"]))diff --git a/tasks/python3/fetchPreviewSnapshots.py b/tasks/python3/fetchPreviewSnapshots.pyindex e076615d2..e993dbb0f 100755--- a/tasks/python3/fetchPreviewSnapshots.py+++ b/tasks/python3/fetchPreviewSnapshots.py@@ -189,16 +189,16 @@ def get_snapshots(layer):if __name__ == "__main__":# Check to see if this feature is enabled in features.json before continuing- with open(features_file, 'rt') as features_json:+ with open(features_file, 'rt', encoding="utf-8") as features_json:features_dict = json.load(features_json)if features_dict['features']['previewSnapshots'] is False:sys.exit();# Allow manual configuration of layer ID to specific date to generate desired preview- with open(overrides_file, 'rt') as overrides_json:+ with open(overrides_file, 'rt', encoding="utf-8") as overrides_json:override_dates_dict = json.load(overrides_json)- with open(wv_json_file, 'rt') as wv_json:+ with open(wv_json_file, 'rt', encoding="utf-8") as wv_json:wv_json_dict = json.load(wv_json)layers = wv_json_dict['layers']snapshots_url = wv_json_dict['features']['imageDownload']['url']diff --git a/tasks/python3/generateCategoryGroupOrder.py b/tasks/python3/generateCategoryGroupOrder.pyindex 21bde7f95..be3bd6427 100755--- a/tasks/python3/generateCategoryGroupOrder.py+++ b/tasks/python3/generateCategoryGroupOrder.py@@ -17,7 +17,7 @@ category_dict = {}for root, dirs, files in os.walk(category_directory):for file in files:file_path = os.path.join(root, file)- with open(file_path) as category_config:+ with open(file_path, "r", encoding="utf-8") as category_config:config = json.load(category_config)try:category = list(config['categories'])[0]@@ -26,6 +26,6 @@ for root, dirs, files in os.walk(category_directory):print("%s ERROR: Could not read category config. Check formatting of: %s" % (prog, file))sys.exit(1)-with open(output_dir + '/categoryGroupOrder.json', "w") as fp:+with open(output_dir + '/categoryGroupOrder.json', "w", encoding="utf-8") as fp:json.dump({ 'categoryGroupOrder': list(category_dict) }, fp, indent=2, sort_keys=True)print("%s Successfully generated categoryGroupOrder.json with these categories: %s" % (prog, list(category_dict)))\ No newline at end of filediff --git a/tasks/python3/getCapabilities.py b/tasks/python3/getCapabilities.pyindex a9af73ea6..6db79ec13 100755--- a/tasks/python3/getCapabilities.py+++ b/tasks/python3/getCapabilities.py@@ -48,7 +48,7 @@ http = urllib3.PoolManager(if not os.path.exists(output_dir):os.makedirs(output_dir)-with open(config_file) as fp:+with open(config_file, "r", encoding="utf-8") as fp:config = json.load(fp)def process_vector_data(layer):@@ -93,7 +93,7 @@ def process_remote(entry):output_file = os.path.join(output_dir, entry["to"])# Write GetCapabilities responses to XML files- with open(output_file, "w") as fp:+ with open(output_file, "w", encoding="utf-8") as fp:fp.write(contents.decode('utf-8'))gc = xmltodict.parse(contents)@@ -121,7 +121,7 @@ def process_single_colormap(link):response = http.request("GET", link)contents = response.dataoutput_file = os.path.join(colormaps_dir, os.path.basename(link))- with open(output_file, "w") as fp:+ with open(output_file, "w", encoding='utf-8') as fp:fp.write(contents.decode('utf-8'))except Exception as e:sys.stderr.write("%s: WARN: Unable to fetch %s: %s\n" %@@ -151,7 +151,7 @@ def process_vectorstyles():contents = response.dataif link.endswith('.json'):output_file = os.path.join(vectorstyles_dir, os.path.basename(link))- with open(output_file, "w") as fp:+ with open(output_file, "w", encoding="utf-8") as fp:fp.write(contents.decode('utf-8'))except Exception as e:sys.stderr.write("%s: WARN: Unable to fetch %s: %s" %@@ -171,7 +171,7 @@ def process_vectordata():contents = response.dataif link.endswith('.json'):output_file = os.path.join(vectordata_dir, os.path.basename(link))- with open(output_file, "w") as fp:+ with open(output_file, "w", encoding="utf-8") as fp:fp.write(contents.decode('utf-8'))except Exception as e:sys.stderr.write("%s: WARN: Unable to fetch %s: %s" %diff --git a/tasks/python3/getVisMetadata.py b/tasks/python3/getVisMetadata.pyindex aee7eeae4..803a2f512 100755--- a/tasks/python3/getVisMetadata.py+++ b/tasks/python3/getVisMetadata.py@@ -33,7 +33,7 @@ def get_metadata(layer_id, base_url):layer_metadata[layer_id].pop(key, None)def main(url):- with open(input_file, 'rt') as layer_order:+ with open(input_file, 'rt', encoding="utf-8") as layer_order:layer_ids = json.load(layer_order).get('layerOrder')print('%s: Pulling vis metadata for %s layers... ' % (prog, len(layer_ids)))@@ -48,13 +48,13 @@ def main(url):except Exception as e:print("%s:" % (e))- with open(output_file, "w") as fp:+ with open(output_file, "w", encoding="utf-8") as fp:# Format of this object will determine how this data is combined into wv.jsonjson.dump({ 'layers': layer_metadata}, fp, indent=2, sort_keys=True)#MAINif __name__ == "__main__":- with open(features_file, 'rt') as features:+ with open(features_file, 'rt', encoding="utf-8") as features:metadata_config = json.load(features).get('features').get('vismetadata')if metadata_config is not None:url = metadata_config.get('url')diff --git a/tasks/python3/mergeConfig.py b/tasks/python3/mergeConfig.pyindex 89dadd5c6..74b10c764 100755--- a/tasks/python3/mergeConfig.py+++ b/tasks/python3/mergeConfig.py@@ -34,7 +34,7 @@ for root, dirs, files in os.walk(input_dir):if not file.endswith(".json"):continuefile_count += 1- with open(os.path.join(root, file)) as fp:+ with open(os.path.join(root, file), "r", encoding="utf-8") as fp:data = json.load(fp)dict_merge(conf, data)except Exception as e:@@ -46,7 +46,7 @@ json_options = {}json_options["indent"] = 2json_options["separators"] = (',', ': ')-with open(output_file, "w") as fp:+with open(output_file, "w", encoding="utf-8") as fp:json.dump(conf, fp, **json_options)print("%s: %s file(s) merged into %s" % (prog, file_count,diff --git a/tasks/python3/mergeConfigWithWMTS.py b/tasks/python3/mergeConfigWithWMTS.pyindex 358b59821..da4510e29 100755--- a/tasks/python3/mergeConfigWithWMTS.py+++ b/tasks/python3/mergeConfigWithWMTS.py@@ -56,7 +56,7 @@ new_conf = {}new_conf["layers"] = {}new_conf["sources"] = {}-with open(output_file) as fp:+with open(output_file, "r", encoding="utf-8") as fp:output_data = json.load(fp)file_count = 0for file in os.listdir(input_dir):@@ -64,7 +64,7 @@ for file in os.listdir(input_dir):if not file.endswith(".json"):continuefile_count += 1- with open(os.path.join(input_dir, file)) as fp:+ with open(os.path.join(input_dir, file), "r", encoding="utf-8") as fp:data = json.load(fp)new_conf['layers'] = wmts_dict_merge(new_conf['layers'], data, output_data)new_conf["sources"] = dict_merge(new_conf["sources"], data['sources'], output_data['sources'])@@ -77,7 +77,7 @@ json_options = {}json_options["indent"] = 2json_options["separators"] = (',', ': ')-with open(output_file, "w") as fp:+with open(output_file, "w", encoding="utf-8") as fp:json.dump(new_conf, fp, **json_options)print("%s: %s file(s) merged into %s" % (prog, file_count,diff --git a/tasks/python3/processColormap.py b/tasks/python3/processColormap.pyindex a19b9c203..5d15709e9 100755--- a/tasks/python3/processColormap.py+++ b/tasks/python3/processColormap.py@@ -26,7 +26,7 @@ if len(args) != 3:config_file = args[0]input_dir = args[1]output_dir = args[2]-with open(config_file) as fp:+with open(config_file, "r", encoding="utf-8") as fp:config = json.load(fp)skips = config.get("skipPalettes", [])@@ -139,7 +139,7 @@ def read_file(file):if id in skips:sys.stderr.write("%s: WARN: [%s] %s\n" % (prog, file), "Skipping")return- with open(file) as fp:+ with open(file, "r", encoding="utf-8") as fp:xml = fp.read()return (id, xml)@@ -172,7 +172,7 @@ def process_file(id, xml):json_options["separators"] = (',', ': ')output_file = os.path.join(output_dir, id + ".json")- with open(output_file, "w") as fp:+ with open(output_file, "w", encoding="utf-8") as fp:json.dump(data, fp, **json_options)# Maindiff --git a/tasks/python3/processVectorData.py b/tasks/python3/processVectorData.pyindex 07ba42a9d..3c77d7bec 100755--- a/tasks/python3/processVectorData.py+++ b/tasks/python3/processVectorData.py@@ -25,7 +25,7 @@ config_file = args[0]vectordata_input_dir = args[1]output_dir = args[2]-with open(config_file) as fp:+with open(config_file, "r", encoding="utf-8") as fp:config = json.load(fp)def copy_file(file):@@ -36,11 +36,11 @@ def copy_file(file):vector_layer_id = vector_layer_filename.split(".json", 1)[0]response_data["vectorData"] = {}response_data["vectorData"][vector_layer_id] = {}- with open(input_file) as json_file:+ with open(input_file, "r", encoding="utf-8") as json_file:initial_data = json.load(json_file)for i in initial_data:response_data["vectorData"][vector_layer_id][i] = initial_data[i]- with open(input_file, 'w') as json_file:+ with open(input_file, "w", encoding="utf-8") as json_file:json.dump(response_data, json_file,indent=2)shutil.copy(input_file, output_dir)diff --git a/tasks/python3/processVectorStyles.py b/tasks/python3/processVectorStyles.pyindex 59dd84020..57c29a70d 100755--- a/tasks/python3/processVectorStyles.py+++ b/tasks/python3/processVectorStyles.py@@ -25,7 +25,7 @@ config_file = args[0]vectordata_input_dir = args[1]output_dir = args[2]-with open(config_file) as fp:+with open(config_file, "r", encoding="utf-8") as fp:config = json.load(fp)def copy_file(file):@@ -36,11 +36,11 @@ def copy_file(file):vector_layer_id = vector_layer_filename.split(".json", 1)[0]response_data["vectorStyles"] = {}response_data["vectorStyles"][vector_layer_id] = {}- with open(input_file) as json_file:+ with open(input_file, "r", encoding="utf-8") as json_file:initial_data = json.load(json_file)for i in initial_data:response_data["vectorStyles"][vector_layer_id][i] = initial_data[i]- with open(input_file, 'w') as json_file:+ with open(input_file, "w", encoding="utf-8") as json_file:json.dump(response_data, json_file,indent=2)shutil.copy(input_file, output_dir)diff --git a/tasks/python3/validateOptions.py b/tasks/python3/validateOptions.pyindex 2ece0930a..16831d58a 100755--- a/tasks/python3/validateOptions.py+++ b/tasks/python3/validateOptions.py@@ -31,9 +31,9 @@ warning_count = 0remove_count = 0main_config_file = os.path.join(config_dir, "wv.json")-with open(main_config_file) as fp:+with open(main_config_file, "r", encoding="utf-8") as fp:wv = json.load(fp)-with open(options_file) as fp:+with open(options_file, "r", encoding="utf-8") as fp:opt = json.load(fp)tolerant = opt.get("tolerant", False)@@ -71,7 +71,7 @@ start_date = datetime.maxfor layer_id in list(wv["layers"].keys()):layer = wv["layers"][layer_id]-+if layer_id != layer.get('id'):error("[%s] layer id does not match id of %s" % (layer_id, layer.get('id')))if layer_id not in wv["layerOrder"]:@@ -193,7 +193,7 @@ json_options = {}json_options["indent"] = 2json_options["separators"] = (',', ': ')-with open(main_config_file, "w") as fp:+with open(main_config_file, "w", encoding="utf-8") as fp:json.dump(wv, fp)if error_count > 0: