Skip to content

Interactive interface for browsing global, full-resolution satellite imagery

WV-2209: Faster concurrent requests during build (#3901)

Jason Kenta22d41f8f561
diff --git a/Pipfile b/Pipfile
index 00ccbb9a3..39bee2928 100644
--- a/Pipfile
+++ b/Pipfile
@@ -10,6 +10,7 @@ urllib3 = {version = "==1.26.5", extras = ["secure"]}
requests = "==2.26.0"
jsonschema = "==3.2.0"
isodate2 = "==0.7.0"
+httpx = {extras = ["http2"], version = "*"}
[dev-packages]
diff --git a/Pipfile.lock b/Pipfile.lock
index b383f5d21..2c86322ae 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
{
"_meta": {
"hash": {
- "sha256": "3ecf28a52ee4426bf83f5d4930828da0f21dd04f7e8a88ea75b2d944c9c86dbc"
+ "sha256": "ab3961359262db4272e848cb92cfe5c38b60a54e33f86843aec092d02d440714"
},
"pipfile-spec": 6,
"requires": {
@@ -16,13 +16,21 @@
]
},
"default": {
+ "anyio": {
+ "hashes": [
+ "sha256:a0aeffe2fb1fdf374a8e4b471444f0f3ac4fb9f5a5b542b48824475e0042a5a6",
+ "sha256:b5fa16c5ff93fa1046f2eeb5bbff2dad4d3514d6cda61d02816dba34fa8c3c2e"
+ ],
+ "markers": "python_full_version >= '3.6.2'",
+ "version": "==3.5.0"
+ },
"attrs": {
"hashes": [
- "sha256:149e90d6d8ac20db7a955ad60cf0e6881a3f20d37096140088356da6c716b0b1",
- "sha256:ef6aaac3ca6cd92904cdd0d83f629a15f18053ec84e6432106f7a4d04ae4f5fb"
+ "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4",
+ "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd"
],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
- "version": "==21.2.0"
+ "version": "==21.4.0"
},
"certifi": {
"hashes": [
@@ -88,36 +96,88 @@
},
"charset-normalizer": {
"hashes": [
- "sha256:e019de665e2bcf9c2b64e2e5aa025fa991da8720daa3c1138cadd2fd1856aed0",
- "sha256:f7af805c321bfa1ce6714c51f254e0d5bb5e5834039bc17db7ebe3a4cec9492b"
+ "sha256:2857e29ff0d34db842cd7ca3230549d1a697f96ee6d3fb071cfa6c7393832597",
+ "sha256:6881edbebdb17b39b4eaaa821b438bf6eddffb4468cf344f09f89def34a8b1df"
],
"markers": "python_version >= '3'",
- "version": "==2.0.7"
+ "version": "==2.0.12"
},
"cryptography": {
"hashes": [
- "sha256:07bb7fbfb5de0980590ddfc7f13081520def06dc9ed214000ad4372fb4e3c7f6",
- "sha256:18d90f4711bf63e2fb21e8c8e51ed8189438e6b35a6d996201ebd98a26abbbe6",
- "sha256:1ed82abf16df40a60942a8c211251ae72858b25b7421ce2497c2eb7a1cee817c",
- "sha256:22a38e96118a4ce3b97509443feace1d1011d0571fae81fc3ad35f25ba3ea999",
- "sha256:2d69645f535f4b2c722cfb07a8eab916265545b3475fdb34e0be2f4ee8b0b15e",
- "sha256:4a2d0e0acc20ede0f06ef7aa58546eee96d2592c00f450c9acb89c5879b61992",
- "sha256:54b2605e5475944e2213258e0ab8696f4f357a31371e538ef21e8d61c843c28d",
- "sha256:7075b304cd567694dc692ffc9747f3e9cb393cc4aa4fb7b9f3abd6f5c4e43588",
- "sha256:7b7ceeff114c31f285528ba8b390d3e9cfa2da17b56f11d366769a807f17cbaa",
- "sha256:7eba2cebca600a7806b893cb1d541a6e910afa87e97acf2021a22b32da1df52d",
- "sha256:928185a6d1ccdb816e883f56ebe92e975a262d31cc536429041921f8cb5a62fd",
- "sha256:9933f28f70d0517686bd7de36166dda42094eac49415459d9bdf5e7df3e0086d",
- "sha256:a688ebcd08250eab5bb5bca318cc05a8c66de5e4171a65ca51db6bd753ff8953",
- "sha256:abb5a361d2585bb95012a19ed9b2c8f412c5d723a9836418fab7aaa0243e67d2",
- "sha256:c10c797ac89c746e488d2ee92bd4abd593615694ee17b2500578b63cad6b93a8",
- "sha256:ced40344e811d6abba00295ced98c01aecf0c2de39481792d87af4fa58b7b4d6",
- "sha256:d57e0cdc1b44b6cdf8af1d01807db06886f10177469312fbde8f44ccbb284bc9",
- "sha256:d99915d6ab265c22873f1b4d6ea5ef462ef797b4140be4c9d8b179915e0985c6",
- "sha256:eb80e8a1f91e4b7ef8b33041591e6d89b2b8e122d787e87eeb2b08da71bb16ad",
- "sha256:ebeddd119f526bcf323a89f853afb12e225902a24d29b55fe18dd6fcb2838a76"
- ],
- "version": "==35.0.0"
+ "sha256:093cb351031656d3ee2f4fa1be579a8c69c754cf874206be1d4cf3b542042804",
+ "sha256:0cc20f655157d4cfc7bada909dc5cc228211b075ba8407c46467f63597c78178",
+ "sha256:1b9362d34363f2c71b7853f6251219298124aa4cc2075ae2932e64c91a3e2717",
+ "sha256:1f3bfbd611db5cb58ca82f3deb35e83af34bb8cf06043fa61500157d50a70982",
+ "sha256:2bd1096476aaac820426239ab534b636c77d71af66c547b9ddcd76eb9c79e004",
+ "sha256:31fe38d14d2e5f787e0aecef831457da6cec68e0bb09a35835b0b44ae8b988fe",
+ "sha256:3b8398b3d0efc420e777c40c16764d6870bcef2eb383df9c6dbb9ffe12c64452",
+ "sha256:3c81599befb4d4f3d7648ed3217e00d21a9341a9a688ecdd615ff72ffbed7336",
+ "sha256:419c57d7b63f5ec38b1199a9521d77d7d1754eb97827bbb773162073ccd8c8d4",
+ "sha256:46f4c544f6557a2fefa7ac8ac7d1b17bf9b647bd20b16decc8fbcab7117fbc15",
+ "sha256:471e0d70201c069f74c837983189949aa0d24bb2d751b57e26e3761f2f782b8d",
+ "sha256:59b281eab51e1b6b6afa525af2bd93c16d49358404f814fe2c2410058623928c",
+ "sha256:731c8abd27693323b348518ed0e0705713a36d79fdbd969ad968fbef0979a7e0",
+ "sha256:95e590dd70642eb2079d280420a888190aa040ad20f19ec8c6e097e38aa29e06",
+ "sha256:a68254dd88021f24a68b613d8c51d5c5e74d735878b9e32cc0adf19d1f10aaf9",
+ "sha256:a7d5137e556cc0ea418dca6186deabe9129cee318618eb1ffecbd35bee55ddc1",
+ "sha256:aeaba7b5e756ea52c8861c133c596afe93dd716cbcacae23b80bc238202dc023",
+ "sha256:dc26bb134452081859aa21d4990474ddb7e863aa39e60d1592800a8865a702de",
+ "sha256:e53258e69874a306fcecb88b7534d61820db8a98655662a3dd2ec7f1afd9132f",
+ "sha256:ef15c2df7656763b4ff20a9bc4381d8352e6640cfeb95c2972c38ef508e75181",
+ "sha256:f224ad253cc9cea7568f49077007d2263efa57396a2f2f78114066fd54b5c68e",
+ "sha256:f8ec91983e638a9bcd75b39f1396e5c0dc2330cbd9ce4accefe68717e6779e0a"
+ ],
+ "version": "==37.0.2"
+ },
+ "h11": {
+ "hashes": [
+ "sha256:36a3cb8c0a032f56e2da7084577878a035d3b61d104230d4bd49c0c6b555a9c6",
+ "sha256:47222cb6067e4a307d535814917cd98fd0a57b6788ce715755fa2b6c28b56042"
+ ],
+ "markers": "python_version >= '3.6'",
+ "version": "==0.12.0"
+ },
+ "h2": {
+ "hashes": [
+ "sha256:03a46bcf682256c95b5fd9e9a99c1323584c3eec6440d379b9903d709476bc6d",
+ "sha256:a83aca08fbe7aacb79fec788c9c0bac936343560ed9ec18b82a13a12c28d2abb"
+ ],
+ "version": "==4.1.0"
+ },
+ "hpack": {
+ "hashes": [
+ "sha256:84a076fad3dc9a9f8063ccb8041ef100867b1878b25ef0ee63847a5d53818a6c",
+ "sha256:fc41de0c63e687ebffde81187a948221294896f6bdc0ae2312708df339430095"
+ ],
+ "markers": "python_full_version >= '3.6.1'",
+ "version": "==4.0.0"
+ },
+ "httpcore": {
+ "hashes": [
+ "sha256:47d772f754359e56dd9d892d9593b6f9870a37aeb8ba51e9a88b09b3d68cfade",
+ "sha256:7503ec1c0f559066e7e39bc4003fd2ce023d01cf51793e3c173b864eb456ead1"
+ ],
+ "markers": "python_version >= '3.6'",
+ "version": "==0.14.7"
+ },
+ "httpx": {
+ "extras": [
+ "http2"
+ ],
+ "hashes": [
+ "sha256:d8e778f76d9bbd46af49e7f062467e3157a5a3d2ae4876a4bbfd8a51ed9c9cb4",
+ "sha256:e35e83d1d2b9b2a609ef367cc4c1e66fd80b750348b20cc9e19d1952fc2ca3f6"
+ ],
+ "index": "pypi",
+ "version": "==0.22.0"
+ },
+ "hyperframe": {
+ "hashes": [
+ "sha256:0ec6bafd80d8ad2195c4f03aacba3a8265e57bc4cff261e802bf39970ed02a15",
+ "sha256:ae510046231dc8e9ecb1a6586f63d2347bf4c8905914aa84ba585ae85f28a914"
+ ],
+ "markers": "python_full_version >= '3.6.1'",
+ "version": "==6.0.1"
},
"idna": {
"hashes": [
@@ -144,45 +204,45 @@
},
"pycparser": {
"hashes": [
- "sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0",
- "sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705"
+ "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9",
+ "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"
],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
- "version": "==2.20"
+ "version": "==2.21"
},
"pyopenssl": {
"hashes": [
- "sha256:5e2d8c5e46d0d865ae933bef5230090bdaf5506281e9eec60fa250ee80600cb3",
- "sha256:8935bd4920ab9abfebb07c41a4f58296407ed77f04bd1a92914044b848ba1ed6"
+ "sha256:660b1b1425aac4a1bea1d94168a85d99f0b3144c869dd4390d27629d0087f1bf",
+ "sha256:ea252b38c87425b64116f808355e8da644ef9b07e429398bfece610f893ee2e0"
],
- "version": "==21.0.0"
+ "version": "==22.0.0"
},
"pyrsistent": {
"hashes": [
- "sha256:097b96f129dd36a8c9e33594e7ebb151b1515eb52cceb08474c10a5479e799f2",
- "sha256:2aaf19dc8ce517a8653746d98e962ef480ff34b6bc563fc067be6401ffb457c7",
- "sha256:404e1f1d254d314d55adb8d87f4f465c8693d6f902f67eb6ef5b4526dc58e6ea",
- "sha256:48578680353f41dca1ca3dc48629fb77dfc745128b56fc01096b2530c13fd426",
- "sha256:4916c10896721e472ee12c95cdc2891ce5890898d2f9907b1b4ae0f53588b710",
- "sha256:527be2bfa8dc80f6f8ddd65242ba476a6c4fb4e3aedbf281dfbac1b1ed4165b1",
- "sha256:58a70d93fb79dc585b21f9d72487b929a6fe58da0754fa4cb9f279bb92369396",
- "sha256:5e4395bbf841693eaebaa5bb5c8f5cdbb1d139e07c975c682ec4e4f8126e03d2",
- "sha256:6b5eed00e597b5b5773b4ca30bd48a5774ef1e96f2a45d105db5b4ebb4bca680",
- "sha256:73ff61b1411e3fb0ba144b8f08d6749749775fe89688093e1efef9839d2dcc35",
- "sha256:772e94c2c6864f2cd2ffbe58bb3bdefbe2a32afa0acb1a77e472aac831f83427",
- "sha256:773c781216f8c2900b42a7b638d5b517bb134ae1acbebe4d1e8f1f41ea60eb4b",
- "sha256:a0c772d791c38bbc77be659af29bb14c38ced151433592e326361610250c605b",
- "sha256:b29b869cf58412ca5738d23691e96d8aff535e17390128a1a52717c9a109da4f",
- "sha256:c1a9ff320fa699337e05edcaae79ef8c2880b52720bc031b219e5b5008ebbdef",
- "sha256:cd3caef37a415fd0dae6148a1b6957a8c5f275a62cca02e18474608cb263640c",
- "sha256:d5ec194c9c573aafaceebf05fc400656722793dac57f254cd4741f3c27ae57b4",
- "sha256:da6e5e818d18459fa46fac0a4a4e543507fe1110e808101277c5a2b5bab0cd2d",
- "sha256:e79d94ca58fcafef6395f6352383fa1a76922268fa02caa2272fff501c2fdc78",
- "sha256:f3ef98d7b76da5eb19c37fda834d50262ff9167c65658d1d8f974d2e4d90676b",
- "sha256:f4c8cabb46ff8e5d61f56a037974228e978f26bfefce4f61a4b1ac0ba7a2ab72"
+ "sha256:0e3e1fcc45199df76053026a51cc59ab2ea3fc7c094c6627e93b7b44cdae2c8c",
+ "sha256:1b34eedd6812bf4d33814fca1b66005805d3640ce53140ab8bbb1e2651b0d9bc",
+ "sha256:4ed6784ceac462a7d6fcb7e9b663e93b9a6fb373b7f43594f9ff68875788e01e",
+ "sha256:5d45866ececf4a5fff8742c25722da6d4c9e180daa7b405dc0a2a2790d668c26",
+ "sha256:636ce2dc235046ccd3d8c56a7ad54e99d5c1cd0ef07d9ae847306c91d11b5fec",
+ "sha256:6455fc599df93d1f60e1c5c4fe471499f08d190d57eca040c0ea182301321286",
+ "sha256:6bc66318fb7ee012071b2792024564973ecc80e9522842eb4e17743604b5e045",
+ "sha256:7bfe2388663fd18bd8ce7db2c91c7400bf3e1a9e8bd7d63bf7e77d39051b85ec",
+ "sha256:7ec335fc998faa4febe75cc5268a9eac0478b3f681602c1f27befaf2a1abe1d8",
+ "sha256:914474c9f1d93080338ace89cb2acee74f4f666fb0424896fcfb8d86058bf17c",
+ "sha256:b568f35ad53a7b07ed9b1b2bae09eb15cdd671a5ba5d2c66caee40dbf91c68ca",
+ "sha256:cdfd2c361b8a8e5d9499b9082b501c452ade8bbf42aef97ea04854f4a3f43b22",
+ "sha256:d1b96547410f76078eaf66d282ddca2e4baae8964364abb4f4dcdde855cd123a",
+ "sha256:d4d61f8b993a7255ba714df3aca52700f8125289f84f704cf80916517c46eb96",
+ "sha256:d7a096646eab884bf8bed965bad63ea327e0d0c38989fc83c5ea7b8a87037bfc",
+ "sha256:df46c854f490f81210870e509818b729db4488e1f30f2a1ce1698b2295a878d1",
+ "sha256:e24a828f57e0c337c8d8bb9f6b12f09dfdf0273da25fda9e314f0b684b415a07",
+ "sha256:e4f3149fd5eb9b285d6bfb54d2e5173f6a116fe19172686797c056672689daf6",
+ "sha256:e92a52c166426efbe0d1ec1332ee9119b6d32fc1f0bbfd55d5c1088070e7fc1b",
+ "sha256:f87cc2863ef33c709e237d4b5f4502a62a00fab450c9e020892e8e2ede5847f5",
+ "sha256:fd8da6d0124efa2f67d86fa70c851022f87c98e205f0594e1fae044e7119a5a6"
],
- "markers": "python_version >= '3.6'",
- "version": "==0.18.0"
+ "markers": "python_version >= '3.7'",
+ "version": "==0.18.1"
},
"requests": {
"hashes": [
@@ -192,6 +252,16 @@
"index": "pypi",
"version": "==2.26.0"
},
+ "rfc3986": {
+ "extras": [
+ "idna2008"
+ ],
+ "hashes": [
+ "sha256:270aaf10d87d0d4e095063c65bf3ddbc6ee3d0b226328ce21e036f946e421835",
+ "sha256:a86d6e1f5b1dc238b218b012df0aa79409667bb209e58da56d0b94704e712a97"
+ ],
+ "version": "==1.5.0"
+ },
"six": {
"hashes": [
"sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9",
@@ -200,6 +270,14 @@
"index": "pypi",
"version": "==1.11.0"
},
+ "sniffio": {
+ "hashes": [
+ "sha256:471b71698eac1c2112a40ce2752bb2f4a4814c22a54a3eed3676bc0f5ca9f663",
+ "sha256:c4666eecec1d3f50960c6bdf61ab7bc350648da6c126e3cf6898d8cd4ddcd3de"
+ ],
+ "markers": "python_version >= '3.5'",
+ "version": "==1.2.0"
+ },
"urllib3": {
"extras": [
"secure"
diff --git a/tasks/buildOptions.sh b/tasks/buildOptions.sh
index d49b09d17..a98f56c4d 100755
--- a/tasks/buildOptions.sh
+++ b/tasks/buildOptions.sh
@@ -46,8 +46,6 @@ fi
mkdir -p "$DEST_DIR/config"
mkdir -p "$BUILD_DIR/colormaps"
-mkdir -p "$BUILD_DIR/vectorstyles"
-mkdir -p "$BUILD_DIR/vectordata"
# If $FETCH_GC is set, make various API requests
if [ "$FETCH_GC" ] ; then
@@ -76,36 +74,19 @@ if [ -e "$BUILD_DIR/config.json" ] ; then
"$BUILD_DIR/_wmts"
fi
-# # Run copyVectorStyles.py and move vectorstyles where we want them
-if [ -e "$BUILD_DIR/vectorstyles" ] ; then
- mkdir -p "$BUILD_DIR"/config/vectorstyles
- if [ -d "$BUILD_DIR"/gc/vectorstyles ] ; then
- cp -r "$BUILD_DIR"/gc/vectorstyles "$BUILD_DIR"/vectorstyles/gc
- fi
- "$PYTHON_SCRIPTS_DIR/copyVectorStyles.py" "$OPT_DIR/$OPT_SUBDIR/config.json" \
- "$BUILD_DIR/vectorstyles" \
- "$BUILD_DIR/config/vectorstyles"
-fi
-
-# Run processVectorStyles.py and move vectordata where we want them
-if [ -e "$BUILD_DIR/vectorstyles" ] ; then
+# Run processVectorStyles.py and move vectorstyles where we want them
+if [ -e "$BUILD_DIR/gc/vectorstyles" ] ; then
mkdir -p "$BUILD_DIR"/config/wv.json/vectorstyles
- if [ -d "$BUILD_DIR"/gc/vectorstyles ] ; then
- cp -r "$BUILD_DIR"/gc/vectorstyles "$BUILD_DIR"
- fi
"$PYTHON_SCRIPTS_DIR/processVectorStyles.py" "$OPT_DIR/$OPT_SUBDIR/config.json" \
- "$BUILD_DIR/config/vectorstyles" \
+ "$BUILD_DIR/gc/vectorstyles" \
"$BUILD_DIR/config/wv.json/vectorstyles"
fi
# Run processVectorData.py and move vectordata where we want them
-if [ -e "$BUILD_DIR/vectordata" ] ; then
+if [ -e "$BUILD_DIR/gc/vectordata" ] ; then
mkdir -p "$BUILD_DIR"/config/wv.json/vectordata
- if [ -d "$BUILD_DIR"/gc/vectordata ] ; then
- cp -r "$BUILD_DIR"/gc/vectordata "$BUILD_DIR"
- fi
"$PYTHON_SCRIPTS_DIR/processVectorData.py" "$OPT_DIR/$OPT_SUBDIR/config.json" \
- "$BUILD_DIR/vectordata" \
+ "$BUILD_DIR/gc/vectordata" \
"$BUILD_DIR/config/wv.json/vectordata"
fi
@@ -156,7 +137,7 @@ cp "$BUILD_DIR/brand.json" "$DEST_DIR"
# Validate the options build
"$PYTHON_SCRIPTS_DIR/validateOptions.py" "$BUILD_DIR/config.json" "$DEST_DIR/config"
-# Fetch preview imagese from WV Snapshots for any layers which they are missing
+# Fetch preview images from WV Snapshots for any layers which they are missing
"$PYTHON_SCRIPTS_DIR/fetchPreviewSnapshots.py" "$DEST_DIR/config/wv.json" \
"$OPT_DIR/common/previewLayerOverrides.json" "$BUILD_DIR/features.json"
diff --git a/tasks/python3/copyVectorStyles.py b/tasks/python3/copyVectorStyles.py
deleted file mode 100755
index f913702d1..000000000
--- a/tasks/python3/copyVectorStyles.py
+++ /dev/null
@@ -1,64 +0,0 @@
-#!/usr/bin/env python
-
-from optparse import OptionParser
-import os
-import sys
-import json
-import shutil
-
-prog = os.path.basename(__file__)
-base_dir = os.path.join(os.path.dirname(__file__), "..")
-version = "3.0.0"
-help_description = """\
-Converts vector styles to JSON files
-"""
-
-parser = OptionParser(usage="Usage: %s <config_file> <vectorstyles_input_dir> <output_dir>" % prog,
- version="%s version %s" % (prog, version),
- epilog=help_description)
-
-(options, args) = parser.parse_args()
-if len(args) != 3:
- parser.error("Invalid number of arguments")
-
-config_file = args[0]
-vectorstyles_input_dir = args[1]
-output_dir = args[2]
-
-with open(config_file, "r", encoding="utf-8") as fp:
- config = json.load(fp)
-skips = config.get("skipPalettes", [])
-
-def copy_file(file):
- input_file = os.path.join(root, file)
- id = os.path.splitext(os.path.basename(input_file))[0]
- if id in skips:
- sys.stderr.write("%s: WARN: [%s] %s\n" % (prog, input_file,
- "Skipping"))
- return
-
- if input_file.endswith('.json'):
- shutil.copy(input_file, output_dir)
-
-# Main
-file_count = 0
-error_count = 0
-file_list = []
-
-for root, dirs, files in os.walk(vectorstyles_input_dir):
- for file in files:
- try:
- if file in file_list:
- print(f'{file} is present in the vectorStyle List')
- else:
- file_count += 1
- copy_file(file)
- file_list.append(file)
- except Exception as e:
- sys.stderr.write("%s: ERROR: [%s] %s\n" % (prog, file, str(e)))
- error_count += 1
-
-print("%s: %d error(s), %d file(s)" % (prog, error_count, file_count))
-
-if error_count > 0:
- sys.exit(1)
diff --git a/tasks/python3/getCapabilities.py b/tasks/python3/getCapabilities.py
index 6db79ec13..75a570ef0 100755
--- a/tasks/python3/getCapabilities.py
+++ b/tasks/python3/getCapabilities.py
@@ -6,11 +6,10 @@ from collections import OrderedDict
import os
import sys
import json
-import urllib3
-import urllib3.contrib.pyopenssl
import xmltodict
import traceback
-import certifi
+import httpx
+import asyncio
prog = os.path.basename(__file__)
base_dir = os.path.join(os.path.dirname(__file__), "..")
@@ -35,22 +34,20 @@ vectordata = {}
colormaps_dir = os.path.join(output_dir, "colormaps")
vectorstyles_dir = os.path.join(output_dir, "vectorstyles")
vectordata_dir = os.path.join(output_dir, "vectordata")
-remote_count = 0
error_count = 0
warning_count = 0
-urllib3.contrib.pyopenssl.inject_into_urllib3()
-http = urllib3.PoolManager(
- cert_reqs='CERT_REQUIRED',
- ca_certs=certifi.where()
-)
-
if not os.path.exists(output_dir):
os.makedirs(output_dir)
with open(config_file, "r", encoding="utf-8") as fp:
config = json.load(fp)
+def handle_exception(e, link):
+ sys.stderr.write("\n %s: WARN: Unable to fetch %s %s" % (prog, link, str(e)))
+ global warning_count
+ warning_count += 1
+
def process_vector_data(layer):
if "ows:Metadata" in layer and layer["ows:Metadata"] is not None:
for item in layer["ows:Metadata"]:
@@ -65,8 +62,7 @@ def process_layer(layer):
ident = layer["ows:Identifier"]
if "ows:Metadata" in layer:
if ident in config.get("skipPalettes", []):
- sys.stderr.write("%s: WARN: Skipping palette for %s\n" %
- prog, ident)
+ sys.stderr.write("%s: WARN: Skipping palette for %s\n" % prog, ident)
global warning_count
warning_count += 1
elif layer["ows:Metadata"] is not None:
@@ -74,7 +70,6 @@ def process_layer(layer):
schema_version = item["@xlink:role"]
if schema_version == "http://earthdata.nasa.gov/gibs/metadata-type/colormap/1.3":
colormap_link = item["@xlink:href"]
- #colormap_link = layer["ows:Metadata"]["@xlink:href"]
colormap_file = os.path.basename(colormap_link)
colormap_id = os.path.splitext(colormap_file)[0]
colormaps[colormap_id] = colormap_link
@@ -85,126 +80,71 @@ def process_layer(layer):
vector_style_id = os.path.splitext(vector_style_file)[0]
vectorstyles[vector_style_id] = vector_style_link
-def process_remote(entry):
- url = entry["from"]
- print("%s: %s" % (prog, url))
- response = http.request('GET', url)
- contents = response.data
- output_file = os.path.join(output_dir, entry["to"])
+async def process_remote(client, entry):
+ try:
+ url = entry["from"]
+ print("%s: %s" % (prog, url))
+ response = await client.get(url)
+ contents = response.text
+ output_file = os.path.join(output_dir, entry["to"])
- # Write GetCapabilities responses to XML files
- with open(output_file, "w", encoding="utf-8") as fp:
- fp.write(contents.decode('utf-8'))
- gc = xmltodict.parse(contents)
+ # Write GetCapabilities responses to XML files
+ with open(output_file, "w", encoding="utf-8") as fp:
+ fp.write(contents)
+ gc = xmltodict.parse(contents)
- # Find all colormaps and vectorstyles in GetCapabilities responses and store them in memory
- if gc["Capabilities"]["Contents"] is None:
- print(('error: %s: no layers' % url))
- return
+ # Find all colormaps and vectorstyles in GetCapabilities responses and store them in memory
+ if gc["Capabilities"]["Contents"] is None:
+ print(('error: %s: no layers' % url))
+ return
- try:
- if(type(gc["Capabilities"]["Contents"]["Layer"]) is OrderedDict):
- process_layer(gc["Capabilities"]["Contents"]["Layer"])
- process_vector_data(gc["Capabilities"]["Contents"]["Layer"])
+ layers = gc["Capabilities"]["Contents"]["Layer"]
+
+
+ if(type(layers) is OrderedDict):
+ process_layer(layers)
+ process_vector_data(layers)
else:
- for layer in gc["Capabilities"]["Contents"]["Layer"]:
- process_layer(layer)
- process_vector_data(layer)
+ [process_layer(layer) for layer in layers]
+ [process_vector_data(layer) for layer in layers]
except Exception as e:
- print(('error: %s: %s' % (url, str(e))))
+ print(('ERROR: %s: %s' % (url, str(e))))
print((str(traceback.format_exc())))
-# Fetch a single colormap and write to file
-def process_single_colormap(link):
+async def process_metadata(client, link, dir, ext):
try:
- response = http.request("GET", link)
- contents = response.data
- output_file = os.path.join(colormaps_dir, os.path.basename(link))
- with open(output_file, "w", encoding='utf-8') as fp:
- fp.write(contents.decode('utf-8'))
- except Exception as e:
- sys.stderr.write("%s: WARN: Unable to fetch %s: %s\n" %
- (prog, link, str(e)))
- global warning_count
- warning_count += 1
-
-# Fetch every colormap from the API and write response to file system
-def process_colormaps():
- print("%s: Fetching %d colormaps..." % (prog, len(colormaps)))
- sys.stdout.flush()
- if not os.path.exists(colormaps_dir):
- os.makedirs(colormaps_dir)
- with ThreadPoolExecutor() as executor:
- for link in colormaps.values():
- executor.submit(process_single_colormap, link)
-
-# Fetch every vectorstyle from the API and write response to file system
-def process_vectorstyles():
- print("%s: Fetching %d vectorstyles" % (prog, len(vectorstyles)))
- sys.stdout.flush()
- if not os.path.exists(vectorstyles_dir):
- os.makedirs(vectorstyles_dir)
- for link in list(vectorstyles.values()):
- try:
- response = http.request("GET", link)
- contents = response.data
- if link.endswith('.json'):
- output_file = os.path.join(vectorstyles_dir, os.path.basename(link))
+ response = await client.get(link)
+ contents = response.text
+ if link.endswith(ext):
+ output_file = os.path.join(dir, os.path.basename(link))
with open(output_file, "w", encoding="utf-8") as fp:
- fp.write(contents.decode('utf-8'))
- except Exception as e:
- sys.stderr.write("%s: WARN: Unable to fetch %s: %s" %
- (prog, link, str(e)))
- global warning_count
- warning_count += 1
+ fp.write(contents)
+ except Exception as e:
+ handle_exception(e, link)
-# Fetch every vectordata from the API and write response to file system
-def process_vectordata():
- print("%s: Fetching %d vectordata" % (prog, len(vectordata)))
+async def gather_process(type, typeStr, client, dir, ext):
+ print("%s: Fetching %d %s" % (prog, len(type), typeStr))
sys.stdout.flush()
- if not os.path.exists(vectordata_dir):
- os.makedirs(vectordata_dir)
- for link in list(vectordata.values()):
- try:
- response = http.request("GET", link)
- contents = response.data
- if link.endswith('.json'):
- output_file = os.path.join(vectordata_dir, os.path.basename(link))
- with open(output_file, "w", encoding="utf-8") as fp:
- fp.write(contents.decode('utf-8'))
- except Exception as e:
- sys.stderr.write("%s: WARN: Unable to fetch %s: %s" %
- (prog, link, str(e)))
- global warning_count
- warning_count += 1
+ if not os.path.exists(dir):
+ os.makedirs(dir)
+ await asyncio.gather(*[process_metadata(client, link, dir, ext) for link in list(type.values())])
-futures = []
-tolerant = config.get("tolerant", False)
-if __name__ == "__main__":
+async def main():
if "wv-options-fetch" in config:
- with ThreadPoolExecutor() as executor:
- for entry in config["wv-options-fetch"]:
- futures.append(executor.submit(process_remote, entry))
- for future in futures:
- try:
- remote_count += 1
- future.result()
- except Exception as e:
- if tolerant:
- warning_count += 1
- sys.stderr.write("%s: WARN: %s\n" % (prog, str(e)))
- else:
- error_count += 1
- sys.stderr.write("%s: ERROR: %s\n" % (prog, str(e)))
- if colormaps:
- process_colormaps()
- if vectorstyles:
- process_vectorstyles()
- if vectordata:
- process_vectordata()
-
-print("%s: %d error(s), %d remote(s)" % (prog, error_count, remote_count))
+ limits = httpx.Limits(max_keepalive_connections=10, max_connections=10)
+ async with httpx.AsyncClient(limits=limits) as client:
+ await asyncio.gather(*[process_remote(client, entry) for entry in config["wv-options-fetch"]])
+ if colormaps:
+ await gather_process(colormaps, 'colormaps', client, colormaps_dir, '.xml')
+ if vectorstyles:
+ await gather_process(vectorstyles, 'vectorstyles', client, vectorstyles_dir, '.json')
+ if vectordata:
+ await gather_process(vectordata, 'vectordata', client, vectordata_dir, '.json')
+
+asyncio.run(main())
+
+print("%s: %d error(s)" % (prog, error_count))
if error_count > 0:
sys.exit(1)
diff --git a/tasks/python3/getVisMetadata.py b/tasks/python3/getVisMetadata.py
index b12dc7935..007214961 100755
--- a/tasks/python3/getVisMetadata.py
+++ b/tasks/python3/getVisMetadata.py
@@ -1,10 +1,10 @@
#!/usr/bin/env python
-from concurrent.futures import ThreadPoolExecutor
from optparse import OptionParser
import os
import json
-import requests
+import httpx
+import asyncio
prog = os.path.basename(__file__)
parser = OptionParser(usage="Usage: %s <config> <output_dir>" % prog)
@@ -35,39 +35,30 @@ def get_daac(metadata):
metadata["dataCenter"].append(dataCenter)
return metadata
-def get_metadata(layer_id, base_url):
- response = requests.get(base_url + layer_id + '.json')
- if (response.status_code != 200):
- print('%s WARNING: No metadata config found for [%s]' % (prog, layer_id))
- return
- metadata = response.json()
- layer_metadata[layer_id] = get_daac(metadata)
+async def get_metadata(client, layer_id, base_url):
+ try:
+ response = await client.get(base_url + layer_id + '.json')
+ metadata = response.json()
+ layer_metadata[layer_id] = get_daac(metadata)
- # Remove any props we don't expect to use
- metadata_keys = dict(layer_metadata[layer_id]).keys()
- for key in metadata_keys:
- if key not in use_keys:
- layer_metadata[layer_id].pop(key, None)
+ # Remove any props we don't expect to use
+ metadata_keys = dict(layer_metadata[layer_id]).keys()
+ for key in metadata_keys:
+ if key not in use_keys:
+ layer_metadata[layer_id].pop(key, None)
+ except Exception as e:
+ print('%s WARNING: Failed to retrieve metadata config for [%s]' % (prog, layer_id))
-def main(url):
+async def main(url):
with open(input_file, 'rt', encoding="utf-8") as layer_order:
layer_ids = json.load(layer_order).get('layerOrder')
print('%s: Pulling vis metadata for %s layers... ' % (prog, len(layer_ids)))
- futures = []
- with ThreadPoolExecutor() as executor:
- for layer in layer_ids:
- futures.append(executor.submit(get_metadata, layer, url))
- for f in futures:
- try:
- # Need to call result() on each future to catch any raised exceptions
- f.result()
- except Exception as e:
- print("%s:" % (e))
-
- with open(output_file, "w", encoding="utf-8") as fp:
- # Format of this object will determine how this data is combined into wv.json
- json.dump({ 'layers': layer_metadata}, fp, indent=2, sort_keys=True)
+ async with httpx.AsyncClient() as client:
+ await asyncio.gather(*[get_metadata(client, layer, url) for layer in layer_ids])
+ with open(output_file, "w", encoding="utf-8") as fp:
+ # Format of this object will determine how this data is combined into wv.json
+ json.dump({ 'layers': layer_metadata}, fp, indent=2, sort_keys=True)
#MAIN
if __name__ == "__main__":
@@ -76,7 +67,7 @@ if __name__ == "__main__":
if metadata_config is not None:
url = metadata_config.get('url')
daacMap = metadata_config.get('daacMap', {})
- main(url)
+ asyncio.run(main(url))
else:
print('%s: Visualization metadata not configured. Exiting.' % (prog))
diff --git a/tasks/python3/validateOptions.py b/tasks/python3/validateOptions.py
index e6e85d308..055cc1fb2 100755
--- a/tasks/python3/validateOptions.py
+++ b/tasks/python3/validateOptions.py
@@ -103,12 +103,6 @@ for layer_id in list(wv["layers"].keys()):
del layer["palette"]
if "vectorStyle" in layer and "id" not in layer["vectorStyle"]:
error("[%s] No vectorStyle definition" % (layer_id))
- elif "vectorStyle" in layer:
- vector_style_id = layer["vectorStyle"]["id"]
- if not os.path.exists(os.path.join(config_dir, "vectorstyles",
- vector_style_id + ".json")):
- error("[%s] Unknown vector style: %s" % (layer_id, vector_style_id))
- del layer["vectorStyle"]
if "group" not in layer and opt.get("warnOnUnexpectedLayer"):
error("[%s] Possible unexpected layer, no group defined" % layer_id)
remove_layer(wv, layer_id)