Skip to content

Interactive interface for browsing global, full-resolution satellite imagery

add explicit utf-8 encoding

Ed Plato26f001db046b
diff --git a/tasks/python3/copyVectorStyles.py b/tasks/python3/copyVectorStyles.py
index 50ea56652..f913702d1 100755
--- a/tasks/python3/copyVectorStyles.py
+++ b/tasks/python3/copyVectorStyles.py
@@ -25,7 +25,7 @@ config_file = args[0]
vectorstyles_input_dir = args[1]
output_dir = args[2]
-with open(config_file) as fp:
+with open(config_file, "r", encoding="utf-8") as fp:
config = json.load(fp)
skips = config.get("skipPalettes", [])
diff --git a/tasks/python3/extractConfigFromWMTS.py b/tasks/python3/extractConfigFromWMTS.py
index c7e8d6705..155e563c9 100755
--- a/tasks/python3/extractConfigFromWMTS.py
+++ b/tasks/python3/extractConfigFromWMTS.py
@@ -29,7 +29,7 @@ config_file = args[0]
input_dir = args[1]
output_dir = args[2]
-with open(config_file) as fp:
+with open(config_file, "r", encoding="utf-8") as fp:
config = json.load(fp)
tolerant = config.get("tolerant", False)
@@ -241,7 +241,7 @@ def process_entry(entry):
process_matrix_set(gc_matrix_set)
output_file = os.path.join(output_dir, entry["to"])
- with open(output_file, "w") as fp:
+ with open(output_file, "w", encoding="utf-8") as fp:
json.dump(wv, fp, **json_options)
print("%s: %d error(s), %d warning(s), %d layers for %s" % (prog,
error_count, warning_count, layer_count, entry["source"]))
diff --git a/tasks/python3/fetchPreviewSnapshots.py b/tasks/python3/fetchPreviewSnapshots.py
index e076615d2..e993dbb0f 100755
--- a/tasks/python3/fetchPreviewSnapshots.py
+++ b/tasks/python3/fetchPreviewSnapshots.py
@@ -189,16 +189,16 @@ def get_snapshots(layer):
if __name__ == "__main__":
# Check to see if this feature is enabled in features.json before continuing
- with open(features_file, 'rt') as features_json:
+ with open(features_file, 'rt', encoding="utf-8") as features_json:
features_dict = json.load(features_json)
if features_dict['features']['previewSnapshots'] is False:
sys.exit();
# Allow manual configuration of layer ID to specific date to generate desired preview
- with open(overrides_file, 'rt') as overrides_json:
+ with open(overrides_file, 'rt', encoding="utf-8") as overrides_json:
override_dates_dict = json.load(overrides_json)
- with open(wv_json_file, 'rt') as wv_json:
+ with open(wv_json_file, 'rt', encoding="utf-8") as wv_json:
wv_json_dict = json.load(wv_json)
layers = wv_json_dict['layers']
snapshots_url = wv_json_dict['features']['imageDownload']['url']
diff --git a/tasks/python3/generateCategoryGroupOrder.py b/tasks/python3/generateCategoryGroupOrder.py
index 21bde7f95..be3bd6427 100755
--- a/tasks/python3/generateCategoryGroupOrder.py
+++ b/tasks/python3/generateCategoryGroupOrder.py
@@ -17,7 +17,7 @@ category_dict = {}
for root, dirs, files in os.walk(category_directory):
for file in files:
file_path = os.path.join(root, file)
- with open(file_path) as category_config:
+ with open(file_path, "r", encoding="utf-8") as category_config:
config = json.load(category_config)
try:
category = list(config['categories'])[0]
@@ -26,6 +26,6 @@ for root, dirs, files in os.walk(category_directory):
print("%s ERROR: Could not read category config. Check formatting of: %s" % (prog, file))
sys.exit(1)
-with open(output_dir + '/categoryGroupOrder.json', "w") as fp:
+with open(output_dir + '/categoryGroupOrder.json', "w", encoding="utf-8") as fp:
json.dump({ 'categoryGroupOrder': list(category_dict) }, fp, indent=2, sort_keys=True)
print("%s Successfully generated categoryGroupOrder.json with these categories: %s" % (prog, list(category_dict)))
\ No newline at end of file
diff --git a/tasks/python3/getCapabilities.py b/tasks/python3/getCapabilities.py
index a9af73ea6..6db79ec13 100755
--- a/tasks/python3/getCapabilities.py
+++ b/tasks/python3/getCapabilities.py
@@ -48,7 +48,7 @@ http = urllib3.PoolManager(
if not os.path.exists(output_dir):
os.makedirs(output_dir)
-with open(config_file) as fp:
+with open(config_file, "r", encoding="utf-8") as fp:
config = json.load(fp)
def process_vector_data(layer):
@@ -93,7 +93,7 @@ def process_remote(entry):
output_file = os.path.join(output_dir, entry["to"])
# Write GetCapabilities responses to XML files
- with open(output_file, "w") as fp:
+ with open(output_file, "w", encoding="utf-8") as fp:
fp.write(contents.decode('utf-8'))
gc = xmltodict.parse(contents)
@@ -121,7 +121,7 @@ def process_single_colormap(link):
response = http.request("GET", link)
contents = response.data
output_file = os.path.join(colormaps_dir, os.path.basename(link))
- with open(output_file, "w") as fp:
+ with open(output_file, "w", encoding='utf-8') as fp:
fp.write(contents.decode('utf-8'))
except Exception as e:
sys.stderr.write("%s: WARN: Unable to fetch %s: %s\n" %
@@ -151,7 +151,7 @@ def process_vectorstyles():
contents = response.data
if link.endswith('.json'):
output_file = os.path.join(vectorstyles_dir, os.path.basename(link))
- with open(output_file, "w") as fp:
+ with open(output_file, "w", encoding="utf-8") as fp:
fp.write(contents.decode('utf-8'))
except Exception as e:
sys.stderr.write("%s: WARN: Unable to fetch %s: %s" %
@@ -171,7 +171,7 @@ def process_vectordata():
contents = response.data
if link.endswith('.json'):
output_file = os.path.join(vectordata_dir, os.path.basename(link))
- with open(output_file, "w") as fp:
+ with open(output_file, "w", encoding="utf-8") as fp:
fp.write(contents.decode('utf-8'))
except Exception as e:
sys.stderr.write("%s: WARN: Unable to fetch %s: %s" %
diff --git a/tasks/python3/getVisMetadata.py b/tasks/python3/getVisMetadata.py
index aee7eeae4..803a2f512 100755
--- a/tasks/python3/getVisMetadata.py
+++ b/tasks/python3/getVisMetadata.py
@@ -33,7 +33,7 @@ def get_metadata(layer_id, base_url):
layer_metadata[layer_id].pop(key, None)
def main(url):
- with open(input_file, 'rt') as layer_order:
+ with open(input_file, 'rt', encoding="utf-8") as layer_order:
layer_ids = json.load(layer_order).get('layerOrder')
print('%s: Pulling vis metadata for %s layers... ' % (prog, len(layer_ids)))
@@ -48,13 +48,13 @@ def main(url):
except Exception as e:
print("%s:" % (e))
- with open(output_file, "w") as fp:
+ with open(output_file, "w", encoding="utf-8") as fp:
# Format of this object will determine how this data is combined into wv.json
json.dump({ 'layers': layer_metadata}, fp, indent=2, sort_keys=True)
#MAIN
if __name__ == "__main__":
- with open(features_file, 'rt') as features:
+ with open(features_file, 'rt', encoding="utf-8") as features:
metadata_config = json.load(features).get('features').get('vismetadata')
if metadata_config is not None:
url = metadata_config.get('url')
diff --git a/tasks/python3/mergeConfig.py b/tasks/python3/mergeConfig.py
index 89dadd5c6..74b10c764 100755
--- a/tasks/python3/mergeConfig.py
+++ b/tasks/python3/mergeConfig.py
@@ -34,7 +34,7 @@ for root, dirs, files in os.walk(input_dir):
if not file.endswith(".json"):
continue
file_count += 1
- with open(os.path.join(root, file)) as fp:
+ with open(os.path.join(root, file), "r", encoding="utf-8") as fp:
data = json.load(fp)
dict_merge(conf, data)
except Exception as e:
@@ -46,7 +46,7 @@ json_options = {}
json_options["indent"] = 2
json_options["separators"] = (',', ': ')
-with open(output_file, "w") as fp:
+with open(output_file, "w", encoding="utf-8") as fp:
json.dump(conf, fp, **json_options)
print("%s: %s file(s) merged into %s" % (prog, file_count,
diff --git a/tasks/python3/mergeConfigWithWMTS.py b/tasks/python3/mergeConfigWithWMTS.py
index 358b59821..da4510e29 100755
--- a/tasks/python3/mergeConfigWithWMTS.py
+++ b/tasks/python3/mergeConfigWithWMTS.py
@@ -56,7 +56,7 @@ new_conf = {}
new_conf["layers"] = {}
new_conf["sources"] = {}
-with open(output_file) as fp:
+with open(output_file, "r", encoding="utf-8") as fp:
output_data = json.load(fp)
file_count = 0
for file in os.listdir(input_dir):
@@ -64,7 +64,7 @@ for file in os.listdir(input_dir):
if not file.endswith(".json"):
continue
file_count += 1
- with open(os.path.join(input_dir, file)) as fp:
+ with open(os.path.join(input_dir, file), "r", encoding="utf-8") as fp:
data = json.load(fp)
new_conf['layers'] = wmts_dict_merge(new_conf['layers'], data, output_data)
new_conf["sources"] = dict_merge(new_conf["sources"], data['sources'], output_data['sources'])
@@ -77,7 +77,7 @@ json_options = {}
json_options["indent"] = 2
json_options["separators"] = (',', ': ')
-with open(output_file, "w") as fp:
+with open(output_file, "w", encoding="utf-8") as fp:
json.dump(new_conf, fp, **json_options)
print("%s: %s file(s) merged into %s" % (prog, file_count,
diff --git a/tasks/python3/processColormap.py b/tasks/python3/processColormap.py
index a19b9c203..5d15709e9 100755
--- a/tasks/python3/processColormap.py
+++ b/tasks/python3/processColormap.py
@@ -26,7 +26,7 @@ if len(args) != 3:
config_file = args[0]
input_dir = args[1]
output_dir = args[2]
-with open(config_file) as fp:
+with open(config_file, "r", encoding="utf-8") as fp:
config = json.load(fp)
skips = config.get("skipPalettes", [])
@@ -139,7 +139,7 @@ def read_file(file):
if id in skips:
sys.stderr.write("%s: WARN: [%s] %s\n" % (prog, file), "Skipping")
return
- with open(file) as fp:
+ with open(file, "r", encoding="utf-8") as fp:
xml = fp.read()
return (id, xml)
@@ -172,7 +172,7 @@ def process_file(id, xml):
json_options["separators"] = (',', ': ')
output_file = os.path.join(output_dir, id + ".json")
- with open(output_file, "w") as fp:
+ with open(output_file, "w", encoding="utf-8") as fp:
json.dump(data, fp, **json_options)
# Main
diff --git a/tasks/python3/processVectorData.py b/tasks/python3/processVectorData.py
index 07ba42a9d..3c77d7bec 100755
--- a/tasks/python3/processVectorData.py
+++ b/tasks/python3/processVectorData.py
@@ -25,7 +25,7 @@ config_file = args[0]
vectordata_input_dir = args[1]
output_dir = args[2]
-with open(config_file) as fp:
+with open(config_file, "r", encoding="utf-8") as fp:
config = json.load(fp)
def copy_file(file):
@@ -36,11 +36,11 @@ def copy_file(file):
vector_layer_id = vector_layer_filename.split(".json", 1)[0]
response_data["vectorData"] = {}
response_data["vectorData"][vector_layer_id] = {}
- with open(input_file) as json_file:
+ with open(input_file, "r", encoding="utf-8") as json_file:
initial_data = json.load(json_file)
for i in initial_data:
response_data["vectorData"][vector_layer_id][i] = initial_data[i]
- with open(input_file, 'w') as json_file:
+ with open(input_file, "w", encoding="utf-8") as json_file:
json.dump(response_data, json_file,indent=2)
shutil.copy(input_file, output_dir)
diff --git a/tasks/python3/processVectorStyles.py b/tasks/python3/processVectorStyles.py
index 59dd84020..57c29a70d 100755
--- a/tasks/python3/processVectorStyles.py
+++ b/tasks/python3/processVectorStyles.py
@@ -25,7 +25,7 @@ config_file = args[0]
vectordata_input_dir = args[1]
output_dir = args[2]
-with open(config_file) as fp:
+with open(config_file, "r", encoding="utf-8") as fp:
config = json.load(fp)
def copy_file(file):
@@ -36,11 +36,11 @@ def copy_file(file):
vector_layer_id = vector_layer_filename.split(".json", 1)[0]
response_data["vectorStyles"] = {}
response_data["vectorStyles"][vector_layer_id] = {}
- with open(input_file) as json_file:
+ with open(input_file, "r", encoding="utf-8") as json_file:
initial_data = json.load(json_file)
for i in initial_data:
response_data["vectorStyles"][vector_layer_id][i] = initial_data[i]
- with open(input_file, 'w') as json_file:
+ with open(input_file, "w", encoding="utf-8") as json_file:
json.dump(response_data, json_file,indent=2)
shutil.copy(input_file, output_dir)
diff --git a/tasks/python3/validateOptions.py b/tasks/python3/validateOptions.py
index 2ece0930a..16831d58a 100755
--- a/tasks/python3/validateOptions.py
+++ b/tasks/python3/validateOptions.py
@@ -31,9 +31,9 @@ warning_count = 0
remove_count = 0
main_config_file = os.path.join(config_dir, "wv.json")
-with open(main_config_file) as fp:
+with open(main_config_file, "r", encoding="utf-8") as fp:
wv = json.load(fp)
-with open(options_file) as fp:
+with open(options_file, "r", encoding="utf-8") as fp:
opt = json.load(fp)
tolerant = opt.get("tolerant", False)
@@ -71,7 +71,7 @@ start_date = datetime.max
for layer_id in list(wv["layers"].keys()):
layer = wv["layers"][layer_id]
-
+
if layer_id != layer.get('id'):
error("[%s] layer id does not match id of %s" % (layer_id, layer.get('id')))
if layer_id not in wv["layerOrder"]:
@@ -193,7 +193,7 @@ json_options = {}
json_options["indent"] = 2
json_options["separators"] = (',', ': ')
-with open(main_config_file, "w") as fp:
+with open(main_config_file, "w", encoding="utf-8") as fp:
json.dump(wv, fp)
if error_count > 0: