Light, fluffy, and always free - The AWS Local Emulator alternative
#!/usr/bin/env python3
"""Regenerate src/main/resources/aws/partitions.json from AWS's published partition data.
Floci serves every AWS partition, and the Java side (`AwsPartitions`) reads one vendored
file for all of it: partition ids, DNS suffixes, region lists, the `<partition>-global`
pseudo-regions, which services each partition offers and which of them have a partition-wide
endpoint with its signing region. Nothing in that file is hand-typed; this script derives it
from two sources and a rule:
- botocore's `partitions.json` and `endpoints.json` (MIT, https://github.com/boto/botocore),
the same data every AWS SDK resolves endpoints from. Located, in order, at `--botocore-data`,
the `local/aws/botocore` checkout (the reference corpus everything else in this repo is
grounded on), then the installed `botocore` package, which `requirements.txt` pins to the
version the vendored file was generated from so CI regenerates the same bytes.
- `s3DualStackRegions` per partition: the regions whose S3 endpoint publishes a `dualstack`
variant in `endpoints.json` (the service defaults cover every region in the commercial, China
and GovCloud partitions; the ISO partitions list them per region and the EUSC one has none).
- botocore's per-service `service-2.json` metadata, for the services whose SigV4 signing name
differs from their `endpoints.json` key (`ecr` signs what `api.ecr` publishes, `bedrock`
covers `bedrock-runtime`). `signingNames` maps each such signing name to its endpoint
prefixes so a request's credential scope can be checked against a partition's service list.
A signing name is left out when a model whose own endpoint prefix is that signing name is
listed in no partition (`execute-api` is Connect Participant's signing name, but also the
endpoint prefix of API Gateway's WebSocket management API, which `endpoints.json` lists
nowhere): the data cannot say where such a name is served.
- botocore's per-service `endpoint-rule-set-1.json`, for the partitions a ruleset names in an
explicit branch that resolves to an endpoint (IAM's `iam.eusc-de-east-1.amazonaws.eu`).
`endpoints.json` omits some of those, so each partition's `services` adds them, recorded again
under `servicesFromRulesets`. Only a service `endpoints.json` lists in some partition is added:
a service it lists nowhere stays unknown everywhere rather than present in a few partitions.
- botocore's `signin` endpoint ruleset, for `signinHost`: the host AWS Sign-In's data plane is
served on in each partition (`{region}.signin.amazonaws.cn`), which no partition output can
build because the domain is not the DNS suffix. Only the plain branches count (no FIPS, no
dual-stack, not the control plane and not the OAuth endpoint); a partition the ruleset names in
none of them gets no `signinHost`.
- The AWS CDK's `region-info/lib/aws-entities.ts` (Apache-2.0), whose ordered region list
carries two rule markers: regions before `RULE_S3_WEBSITE_REGIONAL_SUBDOMAIN` use the
legacy `s3-website-<region>` endpoint form, and commercial regions after
`RULE_CLASSIC_PARTITION_BECOMES_OPT_IN` are opt-in (DescribeRegions omits them by default).
botocore publishes neither fact. Read from `--cdk` or `local/aws/aws-cdk`; when the checkout
is absent (CI), the two flags are carried over from the existing output file, and a region
with nothing to carry is an error rather than a guess.
Run from anywhere in the repo:
python3 tools/aws/regen_partitions.py # rewrite the vendored file in place
python3 tools/aws/regen_partitions.py --check # exit 1 when the vendored file is stale
"""
from __future__ import annotations
import argparse
import gzip
import importlib
import json
import re
import sys
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[2]
OUTPUT = REPO_ROOT / "src/main/resources/aws/partitions.json"
LOCAL_BOTOCORE_DATA = REPO_ROOT / "local/aws/botocore/botocore/data"
LOCAL_CDK = REPO_ROOT / "local/aws/aws-cdk"
CDK_ENTITIES = "packages/aws-cdk-lib/region-info/lib/aws-entities.ts"
RULE_S3_WEBSITE = "RULE_S3_WEBSITE_REGIONAL_SUBDOMAIN"
RULE_OPT_IN = "RULE_CLASSIC_PARTITION_BECOMES_OPT_IN"
COMMERCIAL = "aws"
ENTITY_REGION_RE = re.compile(r"^\s*'([a-z0-9-]+)',")
ENTITY_RULE_RE = re.compile(r"^\s*(RULE_[A-Z0-9_]+),")
# --------------------------------------------------------------------------- #
# Sources
# --------------------------------------------------------------------------- #
def resolve_botocore_data(explicit: Path | None) -> tuple[Path, str]:
"""The botocore data directory to read and a provenance label for it."""
if explicit is not None:
return explicit, f"--botocore-data {explicit}"
if (LOCAL_BOTOCORE_DATA / "partitions.json").exists():
return LOCAL_BOTOCORE_DATA, f"botocore {local_botocore_version()} (local/aws/botocore)"
try:
botocore = importlib.import_module("botocore")
data = Path(botocore.__file__).resolve().parent / "data"
if (data / "partitions.json").exists():
return data, f"botocore {botocore.__version__} (installed package)"
except ImportError:
pass
raise SystemExit("error: no botocore data found; check out local/aws/botocore, pass --botocore-data, "
"or pip install -r tools/aws/requirements.txt")
def local_botocore_version() -> str:
init = LOCAL_BOTOCORE_DATA.parent / "__init__.py"
if init.exists():
match = re.search(r"__version__\s*=\s*'([^']+)'", init.read_text(encoding="utf-8"))
if match:
return match.group(1)
return "unknown"
def load_json(path: Path) -> dict:
with path.open(encoding="utf-8") as handle:
return json.load(handle)
def service_model(version_dir: Path) -> Path | None:
"""A version directory's `service-2.json`, plain as the botocore checkout has it or gzipped
as the botocore wheel ships it, or None when the directory holds neither."""
for name in ("service-2.json", "service-2.json.gz"):
if (version_dir / name).is_file():
return version_dir / name
return None
def ruleset_file(version_dir: Path) -> Path | None:
"""A version directory's `endpoint-rule-set-1.json`, plain or gzipped, or None."""
for name in ("endpoint-rule-set-1.json", "endpoint-rule-set-1.json.gz"):
if (version_dir / name).is_file():
return version_dir / name
return None
def load_model(path: Path) -> dict:
if path.suffix == ".gz":
with gzip.open(path, "rt", encoding="utf-8") as handle:
return json.load(handle)
return load_json(path)
def parse_cdk_entities(text: str) -> list[str]:
"""The ordered `AWS_REGIONS_AND_RULES` entries: region ids and RULE_ markers, in file order."""
entries: list[str] = []
inside = False
for line in text.splitlines():
if "AWS_REGIONS_AND_RULES" in line and "=" in line and "[" in line:
inside = True
continue
if not inside:
continue
if line.strip().startswith("];"):
break
region = ENTITY_REGION_RE.match(line)
if region:
entries.append(region.group(1))
continue
rule = ENTITY_RULE_RE.match(line)
if rule:
entries.append(rule.group(1))
if RULE_S3_WEBSITE not in entries or RULE_OPT_IN not in entries:
raise ValueError("aws-entities.ts: AWS_REGIONS_AND_RULES lacks the two rule markers")
return entries
def cdk_flags(entries: list[str], region: str, partition: str) -> tuple[bool, bool]:
"""(optIn, s3WebsiteDashForm) exactly as CDK's generate-static-data.ts derives them.
`before(region, rule)` is false for a region the list does not know, so an unlisted
region is never dash-form and, in the commercial partition, is opt-in.
"""
website_rule = entries.index(RULE_S3_WEBSITE)
opt_in_rule = entries.index(RULE_OPT_IN)
try:
index = entries.index(region)
except ValueError:
index = -1
before_website = index != -1 and index < website_rule
before_opt_in = index != -1 and index < opt_in_rule
opt_in = partition == COMMERCIAL and not before_opt_in
return opt_in, before_website
# --------------------------------------------------------------------------- #
# Generation
# --------------------------------------------------------------------------- #
def collect_signing_names(botocore_data: Path) -> dict[str, list[str]]:
"""Signing name -> endpoint prefixes, for the services where the two differ."""
by_signing_name: dict[str, set[str]] = {}
for service_dir in sorted(p for p in botocore_data.iterdir() if p.is_dir()):
versions = sorted(v for v in service_dir.iterdir() if service_model(v) is not None)
if not versions:
continue
metadata = load_model(service_model(versions[-1])).get("metadata", {})
endpoint_prefix = metadata.get("endpointPrefix")
signing_name = metadata.get("signingName") or endpoint_prefix
if endpoint_prefix and signing_name != endpoint_prefix:
by_signing_name.setdefault(signing_name, set()).add(endpoint_prefix)
return {name: sorted(prefixes) for name, prefixes in sorted(by_signing_name.items())}
def collect_self_signed_prefixes(botocore_data: Path) -> set[str]:
"""Endpoint prefixes of the services that sign with their own endpoint prefix."""
prefixes: set[str] = set()
for service_dir in sorted(p for p in botocore_data.iterdir() if p.is_dir()):
versions = sorted(v for v in service_dir.iterdir() if service_model(v) is not None)
if not versions:
continue
metadata = load_model(service_model(versions[-1])).get("metadata", {})
endpoint_prefix = metadata.get("endpointPrefix")
if endpoint_prefix and (metadata.get("signingName") or endpoint_prefix) == endpoint_prefix:
prefixes.add(endpoint_prefix)
return prefixes
def partition_name_condition(condition: dict) -> str | None:
"""The partition id a `stringEquals(getAttr(<partition>, "name"), "<id>")` condition tests."""
if condition.get("fn") != "stringEquals":
return None
argv = condition.get("argv", [])
if len(argv) != 2:
return None
for attribute, value in ((argv[0], argv[1]), (argv[1], argv[0])):
if (isinstance(attribute, dict) and attribute.get("fn") == "getAttr"
and attribute.get("argv", [None, None])[1] == "name" and isinstance(value, str)):
return value
return None
def resolves_to_endpoint(rule: dict) -> bool:
if rule.get("type") == "endpoint":
return True
return rule.get("type") == "tree" and any(resolves_to_endpoint(child) for child in rule.get("rules", []))
def ruleset_partitions(rules: list[dict]) -> set[str]:
"""Partitions a ruleset names in a branch that resolves to an endpoint. A branch that only
raises an error, a negated test and the generic `{PartitionResult#dnsSuffix}` template name
no partition."""
named: set[str] = set()
for rule in rules:
if resolves_to_endpoint(rule):
for condition in rule.get("conditions", []):
partition_id = partition_name_condition(condition)
if partition_id:
named.add(partition_id)
if rule.get("type") == "tree":
named |= ruleset_partitions(rule.get("rules", []))
return named
def collect_ruleset_partitions(botocore_data: Path) -> dict[str, list[str]]:
"""Endpoint prefix -> the partitions its endpoint ruleset names explicitly."""
by_prefix: dict[str, set[str]] = {}
for service_dir in sorted(p for p in botocore_data.iterdir() if p.is_dir()):
versions = sorted(v for v in service_dir.iterdir()
if service_model(v) is not None and ruleset_file(v) is not None)
if not versions:
continue
endpoint_prefix = load_model(service_model(versions[-1])).get("metadata", {}).get("endpointPrefix")
named = ruleset_partitions(load_model(ruleset_file(versions[-1])).get("rules", []))
if endpoint_prefix and named:
by_prefix.setdefault(endpoint_prefix, set()).update(named)
return {prefix: sorted(partitions) for prefix, partitions in sorted(by_prefix.items())}
SIGNIN_SERVICE = "signin"
SIGNIN_URL_RE = re.compile(r"https://\{Region\}\.([a-z0-9-]+(?:\.[a-z0-9-]+)+)")
def boolean_condition(condition: dict) -> tuple[str, bool] | None:
"""The `(parameter, value)` a `booleanEquals(<parameter>, <literal>)` condition tests."""
if condition.get("fn") != "booleanEquals":
return None
argv = condition.get("argv", [])
if len(argv) != 2:
return None
for parameter, value in ((argv[0], argv[1]), (argv[1], argv[0])):
if isinstance(parameter, dict) and isinstance(parameter.get("ref"), str) and isinstance(value, bool):
return parameter["ref"], value
return None
def signin_hosts(rules: list[dict], inherited: tuple[dict, ...] = ()) -> dict[str, str]:
"""Partition id -> `{region}.signin.<domain>`, from the ruleset branches a plain data-plane
call resolves through: one partition named, FIPS and dual-stack both tested false, and a URL
that is the region followed by a literal domain. The control-plane and OAuth branches test
neither flag false, so they never match."""
hosts: dict[str, str] = {}
for rule in rules:
conditions = inherited + tuple(rule.get("conditions", []))
if rule.get("type") == "tree":
found = signin_hosts(rule.get("rules", []), conditions)
elif rule.get("type") == "endpoint":
url = rule.get("endpoint", {}).get("url")
match = SIGNIN_URL_RE.fullmatch(url) if isinstance(url, str) else None
named = {partition_name_condition(c) for c in conditions} - {None}
flags = dict(filter(None, (boolean_condition(c) for c in conditions)))
plain = flags.get("UseFIPS") is False and flags.get("UseDualStack") is False
found = {named.pop(): "{region}." + match.group(1)} if match and plain and len(named) == 1 else {}
else:
found = {}
for partition_id, host in found.items():
if hosts.setdefault(partition_id, host) != host:
raise ValueError(f"signin ruleset: {partition_id} resolves to both {hosts[partition_id]} and {host}")
return hosts
def collect_signin_hosts(botocore_data: Path) -> dict[str, str]:
"""Partition id -> AWS Sign-In data-plane host template, or {} when botocore has no model."""
service_dir = botocore_data / SIGNIN_SERVICE
if not service_dir.is_dir():
return {}
versions = sorted(v for v in service_dir.iterdir() if ruleset_file(v) is not None)
if not versions:
return {}
return signin_hosts(load_model(ruleset_file(versions[-1])).get("rules", []))
def build(partitions_doc: dict, endpoints_doc: dict, entities: list[str] | None,
carried: dict | None, provenance: str,
signing_names: dict[str, list[str]] | None = None,
rulesets: dict[str, list[str]] | None = None,
self_signed: set[str] | None = None,
signin: dict[str, str] | None = None) -> dict:
endpoint_partitions = {p["partition"]: p for p in endpoints_doc["partitions"]}
listed_anywhere = {service for p in endpoints_doc["partitions"] for service in p.get("services", {})}
if signing_names is not None and self_signed is not None:
signing_names = {name: prefixes for name, prefixes in signing_names.items()
if name not in self_signed or name in listed_anywhere}
carried_flags = {}
for partition in (carried or {}).get("partitions", []):
for region in partition.get("regions", []):
carried_flags[region["id"]] = (bool(region.get("optIn")), bool(region.get("s3WebsiteDashForm")))
result_partitions = []
for meta in partitions_doc["partitions"]:
partition_id = meta["id"]
outputs = meta["outputs"]
endpoints_partition = endpoint_partitions.get(partition_id)
if endpoints_partition is None:
raise ValueError(f"endpoints.json has no partition {partition_id}")
pseudo = f"{partition_id}-global"
descriptions = {}
for source in (meta.get("regions", {}), endpoints_partition.get("regions", {})):
for region_id, info in source.items():
if region_id == pseudo:
continue
descriptions.setdefault(region_id, info.get("description", ""))
regions = []
for region_id in sorted(descriptions):
if entities is not None:
opt_in, dash = cdk_flags(entities, region_id, partition_id)
elif region_id in carried_flags:
opt_in, dash = carried_flags[region_id]
else:
raise ValueError(f"{region_id}: no aws-cdk checkout to derive optIn/s3WebsiteDashForm from and "
f"nothing to carry over from {OUTPUT.name}; run with local/aws/aws-cdk present")
regions.append({
"id": region_id,
"description": descriptions[region_id],
"optIn": opt_in,
"s3WebsiteDashForm": dash,
})
services = endpoints_partition.get("services", {})
s3 = services.get("s3", {})
s3_defaults_dualstack = any("dualstack" in v.get("tags", []) for v in s3.get("defaults", {}).get("variants", []))
s3_dualstack_regions = sorted(
region_id for region_id, endpoint in s3.get("endpoints", {}).items()
if region_id in descriptions and (s3_defaults_dualstack
or any("dualstack" in v.get("tags", []) for v in endpoint.get("variants", []))))
global_services = {}
for service_key in sorted(services):
service = services[service_key]
partition_endpoint = service.get("partitionEndpoint")
if not partition_endpoint:
continue
endpoint = service.get("endpoints", {}).get(partition_endpoint, {})
hostname = endpoint.get("hostname") or (
endpoints_partition["defaults"]["hostname"]
.replace("{service}", service_key)
.replace("{region}", partition_endpoint)
.replace("{dnsSuffix}", outputs["dnsSuffix"]))
signing_region = endpoint.get("credentialScope", {}).get("region") or outputs["implicitGlobalRegion"]
global_services[service_key] = {
"hostname": hostname,
"signingRegion": signing_region,
"regionalized": service.get("isRegionalized", True) is not False,
}
from_rulesets = sorted(
prefix for prefix, partitions in (rulesets or {}).items()
if partition_id in partitions and prefix in listed_anywhere and prefix not in services)
result_partitions.append({
"id": partition_id,
"name": endpoints_partition.get("partitionName", partition_id),
"dnsSuffix": outputs["dnsSuffix"],
"dualStackDnsSuffix": outputs["dualStackDnsSuffix"],
"implicitGlobalRegion": outputs["implicitGlobalRegion"],
"regionRegex": meta["regionRegex"],
"globalPseudoRegion": pseudo if pseudo in meta.get("regions", {}) else None,
"supportsDualStack": bool(outputs.get("supportsDualStack", False)),
"supportsFips": bool(outputs.get("supportsFIPS", False)),
"regions": regions,
"services": sorted(set(services) | set(from_rulesets)),
"servicesFromRulesets": from_rulesets,
"globalServices": global_services,
"s3DualStackRegions": s3_dualstack_regions,
})
if partition_id in (signin or {}):
result_partitions[-1]["signinHost"] = signin[partition_id]
result_partitions.sort(key=lambda p: (p["id"] != COMMERCIAL, p["id"]))
return {
"_source": {
"generator": "tools/aws/regen_partitions.py",
"botocore": provenance,
"partitionsJsonVersion": partitions_doc.get("version"),
"endpointsJsonVersion": endpoints_doc.get("version"),
"regionFlags": "aws-cdk region-info/lib/aws-entities.ts rules" if entities is not None
else "carried over from the previous file (no aws-cdk checkout)",
},
"signingNames": signing_names if signing_names is not None else (carried or {}).get("signingNames", {}),
"partitions": result_partitions,
}
def render(document: dict) -> str:
return json.dumps(document, indent=2, ensure_ascii=False) + "\n"
def generate(botocore_data: Path, provenance: str, cdk_root: Path | None, existing: Path) -> dict:
partitions_doc = load_json(botocore_data / "partitions.json")
endpoints_doc = load_json(botocore_data / "endpoints.json")
entities = None
if cdk_root is not None and (cdk_root / CDK_ENTITIES).exists():
entities = parse_cdk_entities((cdk_root / CDK_ENTITIES).read_text(encoding="utf-8"))
carried = load_json(existing) if existing.exists() else None
return build(partitions_doc, endpoints_doc, entities, carried, provenance, collect_signing_names(botocore_data),
collect_ruleset_partitions(botocore_data), collect_self_signed_prefixes(botocore_data),
collect_signin_hosts(botocore_data))
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("--check", action="store_true", help="exit 1 when the vendored file differs from a fresh generation")
parser.add_argument("--botocore-data", type=Path, help="botocore data directory (partitions.json, endpoints.json)")
parser.add_argument("--cdk", type=Path, default=LOCAL_CDK, help="aws-cdk checkout root (default: local/aws/aws-cdk)")
parser.add_argument("--output", type=Path, default=OUTPUT, help=argparse.SUPPRESS)
args = parser.parse_args(argv)
botocore_data, provenance = resolve_botocore_data(args.botocore_data)
document = generate(botocore_data, provenance, args.cdk, args.output)
rendered = render(document)
rel = args.output.relative_to(REPO_ROOT) if args.output.is_relative_to(REPO_ROOT) else args.output
if args.check:
current = args.output.read_text(encoding="utf-8") if args.output.exists() else ""
if strip_source(current) != strip_source(rendered):
previous = json.loads(current).get("_source", {}).get("botocore", "unknown") if current.strip() else "nothing"
print(f"error: {rel} (generated from {previous}) differs from a fresh generation against {provenance}.")
print(" Run 'make aws-data-sync' and commit the result; if the two botocore versions differ, "
"align tools/aws/requirements.txt with the checkout first.")
return 1
print(f"{rel} is up to date ({len(document['partitions'])} partitions, "
f"{sum(len(p['regions']) for p in document['partitions'])} regions)")
return 0
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(rendered, encoding="utf-8")
print(f"wrote {rel} from {provenance}: {len(document['partitions'])} partitions, "
f"{sum(len(p['regions']) for p in document['partitions'])} regions")
return 0
def strip_source(text: str) -> str:
"""The document without its provenance block, so the check compares data, not labels."""
if not text.strip():
return ""
document = json.loads(text)
document.pop("_source", None)
return json.dumps(document, sort_keys=True)
if __name__ == "__main__":
sys.exit(main())