Interactive interface for browsing global, full-resolution satellite imagery
WV-2209: Faster concurrent requests during build (#3901)
Jason Kenta22d41f8f561
diff --git a/Pipfile b/Pipfileindex 00ccbb9a3..39bee2928 100644--- a/Pipfile+++ b/Pipfile@@ -10,6 +10,7 @@ urllib3 = {version = "==1.26.5", extras = ["secure"]}requests = "==2.26.0"jsonschema = "==3.2.0"isodate2 = "==0.7.0"+httpx = {extras = ["http2"], version = "*"}[dev-packages]diff --git a/Pipfile.lock b/Pipfile.lockindex b383f5d21..2c86322ae 100644--- a/Pipfile.lock+++ b/Pipfile.lock@@ -1,7 +1,7 @@{"_meta": {"hash": {- "sha256": "3ecf28a52ee4426bf83f5d4930828da0f21dd04f7e8a88ea75b2d944c9c86dbc"+ "sha256": "ab3961359262db4272e848cb92cfe5c38b60a54e33f86843aec092d02d440714"},"pipfile-spec": 6,"requires": {@@ -16,13 +16,21 @@]},"default": {+ "anyio": {+ "hashes": [+ "sha256:a0aeffe2fb1fdf374a8e4b471444f0f3ac4fb9f5a5b542b48824475e0042a5a6",+ "sha256:b5fa16c5ff93fa1046f2eeb5bbff2dad4d3514d6cda61d02816dba34fa8c3c2e"+ ],+ "markers": "python_full_version >= '3.6.2'",+ "version": "==3.5.0"+ },"attrs": {"hashes": [- "sha256:149e90d6d8ac20db7a955ad60cf0e6881a3f20d37096140088356da6c716b0b1",- "sha256:ef6aaac3ca6cd92904cdd0d83f629a15f18053ec84e6432106f7a4d04ae4f5fb"+ "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4",+ "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd"],"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",- "version": "==21.2.0"+ "version": "==21.4.0"},"certifi": {"hashes": [@@ -88,36 +96,88 @@},"charset-normalizer": {"hashes": [- "sha256:e019de665e2bcf9c2b64e2e5aa025fa991da8720daa3c1138cadd2fd1856aed0",- "sha256:f7af805c321bfa1ce6714c51f254e0d5bb5e5834039bc17db7ebe3a4cec9492b"+ "sha256:2857e29ff0d34db842cd7ca3230549d1a697f96ee6d3fb071cfa6c7393832597",+ "sha256:6881edbebdb17b39b4eaaa821b438bf6eddffb4468cf344f09f89def34a8b1df"],"markers": "python_version >= '3'",- "version": "==2.0.7"+ "version": "==2.0.12"},"cryptography": {"hashes": [- "sha256:07bb7fbfb5de0980590ddfc7f13081520def06dc9ed214000ad4372fb4e3c7f6",- "sha256:18d90f4711bf63e2fb21e8c8e51ed8189438e6b35a6d996201ebd98a26abbbe6",- "sha256:1ed82abf16df40a60942a8c211251ae72858b25b7421ce2497c2eb7a1cee817c",- "sha256:22a38e96118a4ce3b97509443feace1d1011d0571fae81fc3ad35f25ba3ea999",- "sha256:2d69645f535f4b2c722cfb07a8eab916265545b3475fdb34e0be2f4ee8b0b15e",- "sha256:4a2d0e0acc20ede0f06ef7aa58546eee96d2592c00f450c9acb89c5879b61992",- "sha256:54b2605e5475944e2213258e0ab8696f4f357a31371e538ef21e8d61c843c28d",- "sha256:7075b304cd567694dc692ffc9747f3e9cb393cc4aa4fb7b9f3abd6f5c4e43588",- "sha256:7b7ceeff114c31f285528ba8b390d3e9cfa2da17b56f11d366769a807f17cbaa",- "sha256:7eba2cebca600a7806b893cb1d541a6e910afa87e97acf2021a22b32da1df52d",- "sha256:928185a6d1ccdb816e883f56ebe92e975a262d31cc536429041921f8cb5a62fd",- "sha256:9933f28f70d0517686bd7de36166dda42094eac49415459d9bdf5e7df3e0086d",- "sha256:a688ebcd08250eab5bb5bca318cc05a8c66de5e4171a65ca51db6bd753ff8953",- "sha256:abb5a361d2585bb95012a19ed9b2c8f412c5d723a9836418fab7aaa0243e67d2",- "sha256:c10c797ac89c746e488d2ee92bd4abd593615694ee17b2500578b63cad6b93a8",- "sha256:ced40344e811d6abba00295ced98c01aecf0c2de39481792d87af4fa58b7b4d6",- "sha256:d57e0cdc1b44b6cdf8af1d01807db06886f10177469312fbde8f44ccbb284bc9",- "sha256:d99915d6ab265c22873f1b4d6ea5ef462ef797b4140be4c9d8b179915e0985c6",- "sha256:eb80e8a1f91e4b7ef8b33041591e6d89b2b8e122d787e87eeb2b08da71bb16ad",- "sha256:ebeddd119f526bcf323a89f853afb12e225902a24d29b55fe18dd6fcb2838a76"- ],- "version": "==35.0.0"+ "sha256:093cb351031656d3ee2f4fa1be579a8c69c754cf874206be1d4cf3b542042804",+ "sha256:0cc20f655157d4cfc7bada909dc5cc228211b075ba8407c46467f63597c78178",+ "sha256:1b9362d34363f2c71b7853f6251219298124aa4cc2075ae2932e64c91a3e2717",+ "sha256:1f3bfbd611db5cb58ca82f3deb35e83af34bb8cf06043fa61500157d50a70982",+ "sha256:2bd1096476aaac820426239ab534b636c77d71af66c547b9ddcd76eb9c79e004",+ "sha256:31fe38d14d2e5f787e0aecef831457da6cec68e0bb09a35835b0b44ae8b988fe",+ "sha256:3b8398b3d0efc420e777c40c16764d6870bcef2eb383df9c6dbb9ffe12c64452",+ "sha256:3c81599befb4d4f3d7648ed3217e00d21a9341a9a688ecdd615ff72ffbed7336",+ "sha256:419c57d7b63f5ec38b1199a9521d77d7d1754eb97827bbb773162073ccd8c8d4",+ "sha256:46f4c544f6557a2fefa7ac8ac7d1b17bf9b647bd20b16decc8fbcab7117fbc15",+ "sha256:471e0d70201c069f74c837983189949aa0d24bb2d751b57e26e3761f2f782b8d",+ "sha256:59b281eab51e1b6b6afa525af2bd93c16d49358404f814fe2c2410058623928c",+ "sha256:731c8abd27693323b348518ed0e0705713a36d79fdbd969ad968fbef0979a7e0",+ "sha256:95e590dd70642eb2079d280420a888190aa040ad20f19ec8c6e097e38aa29e06",+ "sha256:a68254dd88021f24a68b613d8c51d5c5e74d735878b9e32cc0adf19d1f10aaf9",+ "sha256:a7d5137e556cc0ea418dca6186deabe9129cee318618eb1ffecbd35bee55ddc1",+ "sha256:aeaba7b5e756ea52c8861c133c596afe93dd716cbcacae23b80bc238202dc023",+ "sha256:dc26bb134452081859aa21d4990474ddb7e863aa39e60d1592800a8865a702de",+ "sha256:e53258e69874a306fcecb88b7534d61820db8a98655662a3dd2ec7f1afd9132f",+ "sha256:ef15c2df7656763b4ff20a9bc4381d8352e6640cfeb95c2972c38ef508e75181",+ "sha256:f224ad253cc9cea7568f49077007d2263efa57396a2f2f78114066fd54b5c68e",+ "sha256:f8ec91983e638a9bcd75b39f1396e5c0dc2330cbd9ce4accefe68717e6779e0a"+ ],+ "version": "==37.0.2"+ },+ "h11": {+ "hashes": [+ "sha256:36a3cb8c0a032f56e2da7084577878a035d3b61d104230d4bd49c0c6b555a9c6",+ "sha256:47222cb6067e4a307d535814917cd98fd0a57b6788ce715755fa2b6c28b56042"+ ],+ "markers": "python_version >= '3.6'",+ "version": "==0.12.0"+ },+ "h2": {+ "hashes": [+ "sha256:03a46bcf682256c95b5fd9e9a99c1323584c3eec6440d379b9903d709476bc6d",+ "sha256:a83aca08fbe7aacb79fec788c9c0bac936343560ed9ec18b82a13a12c28d2abb"+ ],+ "version": "==4.1.0"+ },+ "hpack": {+ "hashes": [+ "sha256:84a076fad3dc9a9f8063ccb8041ef100867b1878b25ef0ee63847a5d53818a6c",+ "sha256:fc41de0c63e687ebffde81187a948221294896f6bdc0ae2312708df339430095"+ ],+ "markers": "python_full_version >= '3.6.1'",+ "version": "==4.0.0"+ },+ "httpcore": {+ "hashes": [+ "sha256:47d772f754359e56dd9d892d9593b6f9870a37aeb8ba51e9a88b09b3d68cfade",+ "sha256:7503ec1c0f559066e7e39bc4003fd2ce023d01cf51793e3c173b864eb456ead1"+ ],+ "markers": "python_version >= '3.6'",+ "version": "==0.14.7"+ },+ "httpx": {+ "extras": [+ "http2"+ ],+ "hashes": [+ "sha256:d8e778f76d9bbd46af49e7f062467e3157a5a3d2ae4876a4bbfd8a51ed9c9cb4",+ "sha256:e35e83d1d2b9b2a609ef367cc4c1e66fd80b750348b20cc9e19d1952fc2ca3f6"+ ],+ "index": "pypi",+ "version": "==0.22.0"+ },+ "hyperframe": {+ "hashes": [+ "sha256:0ec6bafd80d8ad2195c4f03aacba3a8265e57bc4cff261e802bf39970ed02a15",+ "sha256:ae510046231dc8e9ecb1a6586f63d2347bf4c8905914aa84ba585ae85f28a914"+ ],+ "markers": "python_full_version >= '3.6.1'",+ "version": "==6.0.1"},"idna": {"hashes": [@@ -144,45 +204,45 @@},"pycparser": {"hashes": [- "sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0",- "sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705"+ "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9",+ "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"],"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",- "version": "==2.20"+ "version": "==2.21"},"pyopenssl": {"hashes": [- "sha256:5e2d8c5e46d0d865ae933bef5230090bdaf5506281e9eec60fa250ee80600cb3",- "sha256:8935bd4920ab9abfebb07c41a4f58296407ed77f04bd1a92914044b848ba1ed6"+ "sha256:660b1b1425aac4a1bea1d94168a85d99f0b3144c869dd4390d27629d0087f1bf",+ "sha256:ea252b38c87425b64116f808355e8da644ef9b07e429398bfece610f893ee2e0"],- "version": "==21.0.0"+ "version": "==22.0.0"},"pyrsistent": {"hashes": [- "sha256:097b96f129dd36a8c9e33594e7ebb151b1515eb52cceb08474c10a5479e799f2",- "sha256:2aaf19dc8ce517a8653746d98e962ef480ff34b6bc563fc067be6401ffb457c7",- "sha256:404e1f1d254d314d55adb8d87f4f465c8693d6f902f67eb6ef5b4526dc58e6ea",- "sha256:48578680353f41dca1ca3dc48629fb77dfc745128b56fc01096b2530c13fd426",- "sha256:4916c10896721e472ee12c95cdc2891ce5890898d2f9907b1b4ae0f53588b710",- "sha256:527be2bfa8dc80f6f8ddd65242ba476a6c4fb4e3aedbf281dfbac1b1ed4165b1",- "sha256:58a70d93fb79dc585b21f9d72487b929a6fe58da0754fa4cb9f279bb92369396",- "sha256:5e4395bbf841693eaebaa5bb5c8f5cdbb1d139e07c975c682ec4e4f8126e03d2",- "sha256:6b5eed00e597b5b5773b4ca30bd48a5774ef1e96f2a45d105db5b4ebb4bca680",- "sha256:73ff61b1411e3fb0ba144b8f08d6749749775fe89688093e1efef9839d2dcc35",- "sha256:772e94c2c6864f2cd2ffbe58bb3bdefbe2a32afa0acb1a77e472aac831f83427",- "sha256:773c781216f8c2900b42a7b638d5b517bb134ae1acbebe4d1e8f1f41ea60eb4b",- "sha256:a0c772d791c38bbc77be659af29bb14c38ced151433592e326361610250c605b",- "sha256:b29b869cf58412ca5738d23691e96d8aff535e17390128a1a52717c9a109da4f",- "sha256:c1a9ff320fa699337e05edcaae79ef8c2880b52720bc031b219e5b5008ebbdef",- "sha256:cd3caef37a415fd0dae6148a1b6957a8c5f275a62cca02e18474608cb263640c",- "sha256:d5ec194c9c573aafaceebf05fc400656722793dac57f254cd4741f3c27ae57b4",- "sha256:da6e5e818d18459fa46fac0a4a4e543507fe1110e808101277c5a2b5bab0cd2d",- "sha256:e79d94ca58fcafef6395f6352383fa1a76922268fa02caa2272fff501c2fdc78",- "sha256:f3ef98d7b76da5eb19c37fda834d50262ff9167c65658d1d8f974d2e4d90676b",- "sha256:f4c8cabb46ff8e5d61f56a037974228e978f26bfefce4f61a4b1ac0ba7a2ab72"+ "sha256:0e3e1fcc45199df76053026a51cc59ab2ea3fc7c094c6627e93b7b44cdae2c8c",+ "sha256:1b34eedd6812bf4d33814fca1b66005805d3640ce53140ab8bbb1e2651b0d9bc",+ "sha256:4ed6784ceac462a7d6fcb7e9b663e93b9a6fb373b7f43594f9ff68875788e01e",+ "sha256:5d45866ececf4a5fff8742c25722da6d4c9e180daa7b405dc0a2a2790d668c26",+ "sha256:636ce2dc235046ccd3d8c56a7ad54e99d5c1cd0ef07d9ae847306c91d11b5fec",+ "sha256:6455fc599df93d1f60e1c5c4fe471499f08d190d57eca040c0ea182301321286",+ "sha256:6bc66318fb7ee012071b2792024564973ecc80e9522842eb4e17743604b5e045",+ "sha256:7bfe2388663fd18bd8ce7db2c91c7400bf3e1a9e8bd7d63bf7e77d39051b85ec",+ "sha256:7ec335fc998faa4febe75cc5268a9eac0478b3f681602c1f27befaf2a1abe1d8",+ "sha256:914474c9f1d93080338ace89cb2acee74f4f666fb0424896fcfb8d86058bf17c",+ "sha256:b568f35ad53a7b07ed9b1b2bae09eb15cdd671a5ba5d2c66caee40dbf91c68ca",+ "sha256:cdfd2c361b8a8e5d9499b9082b501c452ade8bbf42aef97ea04854f4a3f43b22",+ "sha256:d1b96547410f76078eaf66d282ddca2e4baae8964364abb4f4dcdde855cd123a",+ "sha256:d4d61f8b993a7255ba714df3aca52700f8125289f84f704cf80916517c46eb96",+ "sha256:d7a096646eab884bf8bed965bad63ea327e0d0c38989fc83c5ea7b8a87037bfc",+ "sha256:df46c854f490f81210870e509818b729db4488e1f30f2a1ce1698b2295a878d1",+ "sha256:e24a828f57e0c337c8d8bb9f6b12f09dfdf0273da25fda9e314f0b684b415a07",+ "sha256:e4f3149fd5eb9b285d6bfb54d2e5173f6a116fe19172686797c056672689daf6",+ "sha256:e92a52c166426efbe0d1ec1332ee9119b6d32fc1f0bbfd55d5c1088070e7fc1b",+ "sha256:f87cc2863ef33c709e237d4b5f4502a62a00fab450c9e020892e8e2ede5847f5",+ "sha256:fd8da6d0124efa2f67d86fa70c851022f87c98e205f0594e1fae044e7119a5a6"],- "markers": "python_version >= '3.6'",- "version": "==0.18.0"+ "markers": "python_version >= '3.7'",+ "version": "==0.18.1"},"requests": {"hashes": [@@ -192,6 +252,16 @@"index": "pypi","version": "==2.26.0"},+ "rfc3986": {+ "extras": [+ "idna2008"+ ],+ "hashes": [+ "sha256:270aaf10d87d0d4e095063c65bf3ddbc6ee3d0b226328ce21e036f946e421835",+ "sha256:a86d6e1f5b1dc238b218b012df0aa79409667bb209e58da56d0b94704e712a97"+ ],+ "version": "==1.5.0"+ },"six": {"hashes": ["sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9",@@ -200,6 +270,14 @@"index": "pypi","version": "==1.11.0"},+ "sniffio": {+ "hashes": [+ "sha256:471b71698eac1c2112a40ce2752bb2f4a4814c22a54a3eed3676bc0f5ca9f663",+ "sha256:c4666eecec1d3f50960c6bdf61ab7bc350648da6c126e3cf6898d8cd4ddcd3de"+ ],+ "markers": "python_version >= '3.5'",+ "version": "==1.2.0"+ },"urllib3": {"extras": ["secure"diff --git a/tasks/buildOptions.sh b/tasks/buildOptions.shindex d49b09d17..a98f56c4d 100755--- a/tasks/buildOptions.sh+++ b/tasks/buildOptions.sh@@ -46,8 +46,6 @@ fimkdir -p "$DEST_DIR/config"mkdir -p "$BUILD_DIR/colormaps"-mkdir -p "$BUILD_DIR/vectorstyles"-mkdir -p "$BUILD_DIR/vectordata"# If $FETCH_GC is set, make various API requestsif [ "$FETCH_GC" ] ; then@@ -76,36 +74,19 @@ if [ -e "$BUILD_DIR/config.json" ] ; then"$BUILD_DIR/_wmts"fi-# # Run copyVectorStyles.py and move vectorstyles where we want them-if [ -e "$BUILD_DIR/vectorstyles" ] ; then- mkdir -p "$BUILD_DIR"/config/vectorstyles- if [ -d "$BUILD_DIR"/gc/vectorstyles ] ; then- cp -r "$BUILD_DIR"/gc/vectorstyles "$BUILD_DIR"/vectorstyles/gc- fi- "$PYTHON_SCRIPTS_DIR/copyVectorStyles.py" "$OPT_DIR/$OPT_SUBDIR/config.json" \- "$BUILD_DIR/vectorstyles" \- "$BUILD_DIR/config/vectorstyles"-fi--# Run processVectorStyles.py and move vectordata where we want them-if [ -e "$BUILD_DIR/vectorstyles" ] ; then+# Run processVectorStyles.py and move vectorstyles where we want them+if [ -e "$BUILD_DIR/gc/vectorstyles" ] ; thenmkdir -p "$BUILD_DIR"/config/wv.json/vectorstyles- if [ -d "$BUILD_DIR"/gc/vectorstyles ] ; then- cp -r "$BUILD_DIR"/gc/vectorstyles "$BUILD_DIR"- fi"$PYTHON_SCRIPTS_DIR/processVectorStyles.py" "$OPT_DIR/$OPT_SUBDIR/config.json" \- "$BUILD_DIR/config/vectorstyles" \+ "$BUILD_DIR/gc/vectorstyles" \"$BUILD_DIR/config/wv.json/vectorstyles"fi# Run processVectorData.py and move vectordata where we want them-if [ -e "$BUILD_DIR/vectordata" ] ; then+if [ -e "$BUILD_DIR/gc/vectordata" ] ; thenmkdir -p "$BUILD_DIR"/config/wv.json/vectordata- if [ -d "$BUILD_DIR"/gc/vectordata ] ; then- cp -r "$BUILD_DIR"/gc/vectordata "$BUILD_DIR"- fi"$PYTHON_SCRIPTS_DIR/processVectorData.py" "$OPT_DIR/$OPT_SUBDIR/config.json" \- "$BUILD_DIR/vectordata" \+ "$BUILD_DIR/gc/vectordata" \"$BUILD_DIR/config/wv.json/vectordata"fi@@ -156,7 +137,7 @@ cp "$BUILD_DIR/brand.json" "$DEST_DIR"# Validate the options build"$PYTHON_SCRIPTS_DIR/validateOptions.py" "$BUILD_DIR/config.json" "$DEST_DIR/config"-# Fetch preview imagese from WV Snapshots for any layers which they are missing+# Fetch preview images from WV Snapshots for any layers which they are missing"$PYTHON_SCRIPTS_DIR/fetchPreviewSnapshots.py" "$DEST_DIR/config/wv.json" \"$OPT_DIR/common/previewLayerOverrides.json" "$BUILD_DIR/features.json"diff --git a/tasks/python3/copyVectorStyles.py b/tasks/python3/copyVectorStyles.pydeleted file mode 100755index f913702d1..000000000--- a/tasks/python3/copyVectorStyles.py+++ /dev/null@@ -1,64 +0,0 @@-#!/usr/bin/env python--from optparse import OptionParser-import os-import sys-import json-import shutil--prog = os.path.basename(__file__)-base_dir = os.path.join(os.path.dirname(__file__), "..")-version = "3.0.0"-help_description = """\-Converts vector styles to JSON files-"""--parser = OptionParser(usage="Usage: %s <config_file> <vectorstyles_input_dir> <output_dir>" % prog,- version="%s version %s" % (prog, version),- epilog=help_description)--(options, args) = parser.parse_args()-if len(args) != 3:- parser.error("Invalid number of arguments")--config_file = args[0]-vectorstyles_input_dir = args[1]-output_dir = args[2]--with open(config_file, "r", encoding="utf-8") as fp:- config = json.load(fp)-skips = config.get("skipPalettes", [])--def copy_file(file):- input_file = os.path.join(root, file)- id = os.path.splitext(os.path.basename(input_file))[0]- if id in skips:- sys.stderr.write("%s: WARN: [%s] %s\n" % (prog, input_file,- "Skipping"))- return-- if input_file.endswith('.json'):- shutil.copy(input_file, output_dir)--# Main-file_count = 0-error_count = 0-file_list = []--for root, dirs, files in os.walk(vectorstyles_input_dir):- for file in files:- try:- if file in file_list:- print(f'{file} is present in the vectorStyle List')- else:- file_count += 1- copy_file(file)- file_list.append(file)- except Exception as e:- sys.stderr.write("%s: ERROR: [%s] %s\n" % (prog, file, str(e)))- error_count += 1--print("%s: %d error(s), %d file(s)" % (prog, error_count, file_count))--if error_count > 0:- sys.exit(1)diff --git a/tasks/python3/getCapabilities.py b/tasks/python3/getCapabilities.pyindex 6db79ec13..75a570ef0 100755--- a/tasks/python3/getCapabilities.py+++ b/tasks/python3/getCapabilities.py@@ -6,11 +6,10 @@ from collections import OrderedDictimport osimport sysimport json-import urllib3-import urllib3.contrib.pyopensslimport xmltodictimport traceback-import certifi+import httpx+import asyncioprog = os.path.basename(__file__)base_dir = os.path.join(os.path.dirname(__file__), "..")@@ -35,22 +34,20 @@ vectordata = {}colormaps_dir = os.path.join(output_dir, "colormaps")vectorstyles_dir = os.path.join(output_dir, "vectorstyles")vectordata_dir = os.path.join(output_dir, "vectordata")-remote_count = 0error_count = 0warning_count = 0-urllib3.contrib.pyopenssl.inject_into_urllib3()-http = urllib3.PoolManager(- cert_reqs='CERT_REQUIRED',- ca_certs=certifi.where()-)-if not os.path.exists(output_dir):os.makedirs(output_dir)with open(config_file, "r", encoding="utf-8") as fp:config = json.load(fp)+def handle_exception(e, link):+ sys.stderr.write("\n %s: WARN: Unable to fetch %s %s" % (prog, link, str(e)))+ global warning_count+ warning_count += 1+def process_vector_data(layer):if "ows:Metadata" in layer and layer["ows:Metadata"] is not None:for item in layer["ows:Metadata"]:@@ -65,8 +62,7 @@ def process_layer(layer):ident = layer["ows:Identifier"]if "ows:Metadata" in layer:if ident in config.get("skipPalettes", []):- sys.stderr.write("%s: WARN: Skipping palette for %s\n" %- prog, ident)+ sys.stderr.write("%s: WARN: Skipping palette for %s\n" % prog, ident)global warning_countwarning_count += 1elif layer["ows:Metadata"] is not None:@@ -74,7 +70,6 @@ def process_layer(layer):schema_version = item["@xlink:role"]if schema_version == "http://earthdata.nasa.gov/gibs/metadata-type/colormap/1.3":colormap_link = item["@xlink:href"]- #colormap_link = layer["ows:Metadata"]["@xlink:href"]colormap_file = os.path.basename(colormap_link)colormap_id = os.path.splitext(colormap_file)[0]colormaps[colormap_id] = colormap_link@@ -85,126 +80,71 @@ def process_layer(layer):vector_style_id = os.path.splitext(vector_style_file)[0]vectorstyles[vector_style_id] = vector_style_link-def process_remote(entry):- url = entry["from"]- print("%s: %s" % (prog, url))- response = http.request('GET', url)- contents = response.data- output_file = os.path.join(output_dir, entry["to"])+async def process_remote(client, entry):+ try:+ url = entry["from"]+ print("%s: %s" % (prog, url))+ response = await client.get(url)+ contents = response.text+ output_file = os.path.join(output_dir, entry["to"])- # Write GetCapabilities responses to XML files- with open(output_file, "w", encoding="utf-8") as fp:- fp.write(contents.decode('utf-8'))- gc = xmltodict.parse(contents)+ # Write GetCapabilities responses to XML files+ with open(output_file, "w", encoding="utf-8") as fp:+ fp.write(contents)+ gc = xmltodict.parse(contents)- # Find all colormaps and vectorstyles in GetCapabilities responses and store them in memory- if gc["Capabilities"]["Contents"] is None:- print(('error: %s: no layers' % url))- return+ # Find all colormaps and vectorstyles in GetCapabilities responses and store them in memory+ if gc["Capabilities"]["Contents"] is None:+ print(('error: %s: no layers' % url))+ return- try:- if(type(gc["Capabilities"]["Contents"]["Layer"]) is OrderedDict):- process_layer(gc["Capabilities"]["Contents"]["Layer"])- process_vector_data(gc["Capabilities"]["Contents"]["Layer"])+ layers = gc["Capabilities"]["Contents"]["Layer"]+++ if(type(layers) is OrderedDict):+ process_layer(layers)+ process_vector_data(layers)else:- for layer in gc["Capabilities"]["Contents"]["Layer"]:- process_layer(layer)- process_vector_data(layer)+ [process_layer(layer) for layer in layers]+ [process_vector_data(layer) for layer in layers]except Exception as e:- print(('error: %s: %s' % (url, str(e))))+ print(('ERROR: %s: %s' % (url, str(e))))print((str(traceback.format_exc())))-# Fetch a single colormap and write to file-def process_single_colormap(link):+async def process_metadata(client, link, dir, ext):try:- response = http.request("GET", link)- contents = response.data- output_file = os.path.join(colormaps_dir, os.path.basename(link))- with open(output_file, "w", encoding='utf-8') as fp:- fp.write(contents.decode('utf-8'))- except Exception as e:- sys.stderr.write("%s: WARN: Unable to fetch %s: %s\n" %- (prog, link, str(e)))- global warning_count- warning_count += 1--# Fetch every colormap from the API and write response to file system-def process_colormaps():- print("%s: Fetching %d colormaps..." % (prog, len(colormaps)))- sys.stdout.flush()- if not os.path.exists(colormaps_dir):- os.makedirs(colormaps_dir)- with ThreadPoolExecutor() as executor:- for link in colormaps.values():- executor.submit(process_single_colormap, link)--# Fetch every vectorstyle from the API and write response to file system-def process_vectorstyles():- print("%s: Fetching %d vectorstyles" % (prog, len(vectorstyles)))- sys.stdout.flush()- if not os.path.exists(vectorstyles_dir):- os.makedirs(vectorstyles_dir)- for link in list(vectorstyles.values()):- try:- response = http.request("GET", link)- contents = response.data- if link.endswith('.json'):- output_file = os.path.join(vectorstyles_dir, os.path.basename(link))+ response = await client.get(link)+ contents = response.text+ if link.endswith(ext):+ output_file = os.path.join(dir, os.path.basename(link))with open(output_file, "w", encoding="utf-8") as fp:- fp.write(contents.decode('utf-8'))- except Exception as e:- sys.stderr.write("%s: WARN: Unable to fetch %s: %s" %- (prog, link, str(e)))- global warning_count- warning_count += 1+ fp.write(contents)+ except Exception as e:+ handle_exception(e, link)-# Fetch every vectordata from the API and write response to file system-def process_vectordata():- print("%s: Fetching %d vectordata" % (prog, len(vectordata)))+async def gather_process(type, typeStr, client, dir, ext):+ print("%s: Fetching %d %s" % (prog, len(type), typeStr))sys.stdout.flush()- if not os.path.exists(vectordata_dir):- os.makedirs(vectordata_dir)- for link in list(vectordata.values()):- try:- response = http.request("GET", link)- contents = response.data- if link.endswith('.json'):- output_file = os.path.join(vectordata_dir, os.path.basename(link))- with open(output_file, "w", encoding="utf-8") as fp:- fp.write(contents.decode('utf-8'))- except Exception as e:- sys.stderr.write("%s: WARN: Unable to fetch %s: %s" %- (prog, link, str(e)))- global warning_count- warning_count += 1+ if not os.path.exists(dir):+ os.makedirs(dir)+ await asyncio.gather(*[process_metadata(client, link, dir, ext) for link in list(type.values())])-futures = []-tolerant = config.get("tolerant", False)-if __name__ == "__main__":+async def main():if "wv-options-fetch" in config:- with ThreadPoolExecutor() as executor:- for entry in config["wv-options-fetch"]:- futures.append(executor.submit(process_remote, entry))- for future in futures:- try:- remote_count += 1- future.result()- except Exception as e:- if tolerant:- warning_count += 1- sys.stderr.write("%s: WARN: %s\n" % (prog, str(e)))- else:- error_count += 1- sys.stderr.write("%s: ERROR: %s\n" % (prog, str(e)))- if colormaps:- process_colormaps()- if vectorstyles:- process_vectorstyles()- if vectordata:- process_vectordata()--print("%s: %d error(s), %d remote(s)" % (prog, error_count, remote_count))+ limits = httpx.Limits(max_keepalive_connections=10, max_connections=10)+ async with httpx.AsyncClient(limits=limits) as client:+ await asyncio.gather(*[process_remote(client, entry) for entry in config["wv-options-fetch"]])+ if colormaps:+ await gather_process(colormaps, 'colormaps', client, colormaps_dir, '.xml')+ if vectorstyles:+ await gather_process(vectorstyles, 'vectorstyles', client, vectorstyles_dir, '.json')+ if vectordata:+ await gather_process(vectordata, 'vectordata', client, vectordata_dir, '.json')++asyncio.run(main())++print("%s: %d error(s)" % (prog, error_count))if error_count > 0:sys.exit(1)diff --git a/tasks/python3/getVisMetadata.py b/tasks/python3/getVisMetadata.pyindex b12dc7935..007214961 100755--- a/tasks/python3/getVisMetadata.py+++ b/tasks/python3/getVisMetadata.py@@ -1,10 +1,10 @@#!/usr/bin/env python-from concurrent.futures import ThreadPoolExecutorfrom optparse import OptionParserimport osimport json-import requests+import httpx+import asyncioprog = os.path.basename(__file__)parser = OptionParser(usage="Usage: %s <config> <output_dir>" % prog)@@ -35,39 +35,30 @@ def get_daac(metadata):metadata["dataCenter"].append(dataCenter)return metadata-def get_metadata(layer_id, base_url):- response = requests.get(base_url + layer_id + '.json')- if (response.status_code != 200):- print('%s WARNING: No metadata config found for [%s]' % (prog, layer_id))- return- metadata = response.json()- layer_metadata[layer_id] = get_daac(metadata)+async def get_metadata(client, layer_id, base_url):+ try:+ response = await client.get(base_url + layer_id + '.json')+ metadata = response.json()+ layer_metadata[layer_id] = get_daac(metadata)- # Remove any props we don't expect to use- metadata_keys = dict(layer_metadata[layer_id]).keys()- for key in metadata_keys:- if key not in use_keys:- layer_metadata[layer_id].pop(key, None)+ # Remove any props we don't expect to use+ metadata_keys = dict(layer_metadata[layer_id]).keys()+ for key in metadata_keys:+ if key not in use_keys:+ layer_metadata[layer_id].pop(key, None)+ except Exception as e:+ print('%s WARNING: Failed to retrieve metadata config for [%s]' % (prog, layer_id))-def main(url):+async def main(url):with open(input_file, 'rt', encoding="utf-8") as layer_order:layer_ids = json.load(layer_order).get('layerOrder')print('%s: Pulling vis metadata for %s layers... ' % (prog, len(layer_ids)))- futures = []- with ThreadPoolExecutor() as executor:- for layer in layer_ids:- futures.append(executor.submit(get_metadata, layer, url))- for f in futures:- try:- # Need to call result() on each future to catch any raised exceptions- f.result()- except Exception as e:- print("%s:" % (e))-- with open(output_file, "w", encoding="utf-8") as fp:- # Format of this object will determine how this data is combined into wv.json- json.dump({ 'layers': layer_metadata}, fp, indent=2, sort_keys=True)+ async with httpx.AsyncClient() as client:+ await asyncio.gather(*[get_metadata(client, layer, url) for layer in layer_ids])+ with open(output_file, "w", encoding="utf-8") as fp:+ # Format of this object will determine how this data is combined into wv.json+ json.dump({ 'layers': layer_metadata}, fp, indent=2, sort_keys=True)#MAINif __name__ == "__main__":@@ -76,7 +67,7 @@ if __name__ == "__main__":if metadata_config is not None:url = metadata_config.get('url')daacMap = metadata_config.get('daacMap', {})- main(url)+ asyncio.run(main(url))else:print('%s: Visualization metadata not configured. Exiting.' % (prog))diff --git a/tasks/python3/validateOptions.py b/tasks/python3/validateOptions.pyindex e6e85d308..055cc1fb2 100755--- a/tasks/python3/validateOptions.py+++ b/tasks/python3/validateOptions.py@@ -103,12 +103,6 @@ for layer_id in list(wv["layers"].keys()):del layer["palette"]if "vectorStyle" in layer and "id" not in layer["vectorStyle"]:error("[%s] No vectorStyle definition" % (layer_id))- elif "vectorStyle" in layer:- vector_style_id = layer["vectorStyle"]["id"]- if not os.path.exists(os.path.join(config_dir, "vectorstyles",- vector_style_id + ".json")):- error("[%s] Unknown vector style: %s" % (layer_id, vector_style_id))- del layer["vectorStyle"]if "group" not in layer and opt.get("warnOnUnexpectedLayer"):error("[%s] Possible unexpected layer, no group defined" % layer_id)remove_layer(wv, layer_id)