Open Infrastructure Map, a view of infrastructure data in OpenStreetMap
ISO country pages
Russ Garrett8869769fa0d4
diff --git a/schema/stats.sql b/schema/stats.sqlindex c682115..9e01166 100644--- a/schema/stats.sql+++ b/schema/stats.sql@@ -20,7 +20,7 @@ CREATE TABLE stats.power_plant (count INTEGER,output BIGINT);-CREATE INDEX power_plant_country ON stats.power_line(country);+CREATE INDEX power_plant_country ON stats.power_plant(country);CREATE TABLE stats.power_generator (id SERIAL PRIMARY KEY,@@ -30,7 +30,7 @@ CREATE TABLE stats.power_generator (count INTEGER,output BIGINT);-CREATE INDEX power_generator_country ON stats.power_line(country);+CREATE INDEX power_generator_country ON stats.power_generator(country);CREATE TABLE stats.substation (id SERIAL PRIMARY KEY,@@ -39,4 +39,4 @@ CREATE TABLE stats.substation (voltage INTEGER,count INTEGER);-CREATE INDEX power_substation_country ON stats.power_line(country);\ No newline at end of file+CREATE INDEX power_substation_country ON stats.substation(country);\ No newline at end of filediff --git a/web-backend/config.py b/web-backend/config.pyindex 4017551..adbc0e1 100644--- a/web-backend/config.py+++ b/web-backend/config.py@@ -4,6 +4,6 @@ from starlette.config import Configconfig = Config(".env")DATABASE_URL = config("DATABASE_URL")+DEBUG = config("DEBUG", cast=bool, default=False)database = Database(DATABASE_URL)-diff --git a/web-backend/data.py b/web-backend/data.pyindex 5a21b56..60657db 100644--- a/web-backend/data.py+++ b/web-backend/data.py@@ -1,3 +1,4 @@+import asyncioimport jsonimport loggingimport re@@ -67,51 +68,144 @@ async def get_plant_generator_summary(plant_id):return res-async def stats_power_line(country=None):- stats_date = (await database.fetch_one("SELECT max(time) FROM stats.power_line"))[0]+async def latest_stats_date():+ return (await database.fetch_one("SELECT max(time) FROM stats.power_line"))[0]+++async def stats_power_line(union=None, territory_iso3=None, date=None) -> dict:+ stats_date = date or await latest_stats_date()values = {"time": stats_date}country_clause = ""+ join_clause = ""- if country:+ if union:country_clause = " AND country = :country"- values["country"] = country+ values["country"] = union+ elif territory_iso3:+ join_clause = ", countries.country_eez AS eez"+ country_clause = """+ AND eez."union" = country+ AND eez.iso_sov1 = :iso3+ AND (eez.iso_ter1 IS NULL OR eez.iso_ter1 = eez.iso_sov1)+ """+ values["iso3"] = territory_iso3lines = {}- for low, high in windowed(chain(VOLTAGE_SCALE, [None]), 2):- low = low * 1000- query = (- "SELECT sum(length) FROM stats.power_line WHERE time = :time AND voltage >= :low"- + country_clause++ async with asyncio.TaskGroup() as tg:+ for low, high in windowed(chain(VOLTAGE_SCALE, [None]), 2):+ low = low * 1000+ query = f"""SELECT sum(length)+ FROM stats.power_line {join_clause}+ WHERE time = :time+ AND voltage >= :low+ {country_clause}+ """+ vals = values.copy()+ vals["low"] = low++ if high is not None:+ high = high * 1000+ query += " AND voltage < :high"+ vals["high"] = high++ res = tg.create_task(database.fetch_one(query, vals))+ lines[(low, high)] = res++ unspecified = tg.create_task(+ database.fetch_one(+ f"""SELECT sum(length)+ FROM stats.power_line {join_clause}+ WHERE time = :time+ AND voltage IS NULL+ {country_clause}+ """,+ values,+ ))- vals = values.copy()- vals["low"] = low- if high is not None:- high = high * 1000- query += " AND voltage < :high"- vals["high"] = high+ total = tg.create_task(+ database.fetch_one(+ f"""SELECT sum(length)+ FROM stats.power_line {join_clause}+ WHERE time = :time+ {country_clause}+ """,+ values,+ )+ )- res = await database.fetch_one(query, vals)- lines[(low, high)] = res[0] or 0+ lines = {(low, high): res.result()[0] or 0 for (low, high), res in lines.items()}- unspecified = await database.fetch_one(- "SELECT sum(length) FROM stats.power_line WHERE time = :time AND voltage IS NULL"- + country_clause,- values,- )+ return {+ "lines": lines,+ "total": total.result()[0] or 0.01,+ "unspecified": unspecified.result()[0] or 0,+ }+++async def plant_stats(country_gid: int = None, territory_iso3: str = None, date=None):+ where_clause = ""+ values = {+ "time": date or await latest_stats_date(),+ }- total = await database.fetch_one(- "SELECT sum(length) FROM stats.power_line WHERE time = :time" + country_clause,- values,+ if country_gid:+ where_clause = "eez.gid = :gid"+ values["gid"] = country_gid+ elif territory_iso3:+ where_clause = "eez.iso_sov1 = :iso3 AND (eez.iso_ter1 IS NULL OR eez.iso_ter1 = eez.iso_sov1)"+ values["iso3"] = territory_iso3++ result = await database.fetch_one(+ query=f"""SELECT sum(output * count) AS output, sum(count) AS count+ FROM stats.power_plant, countries.country_eez AS eez+ WHERE eez."union" = country+ AND {where_clause}+ AND time = :time+ """,+ values=values,)- data = {- "date": stats_date.date(),- "lines": lines,- "total": total[0] or 0.01,- "unspecified": unspecified[0] or 0,+ if not result:+ return {+ "output": 0,+ "count": 0,+ }++ return {+ "output": int(result[0] or 0),+ "count": result[1] or 0,+ }+++async def plant_source_stats(country_gid=None, territory_iso3=None, date=None):+ where_clause = ""+ values = {+ "time": date or await latest_stats_date(),}- return data++ if country_gid:+ where_clause = "eez.gid = :gid"+ values["gid"] = country_gid+ elif territory_iso3:+ where_clause = "eez.iso_sov1 = :iso3 AND (eez.iso_ter1 IS NULL OR eez.iso_ter1 = eez.iso_sov1)"+ values["iso3"] = territory_iso3++ return await database.fetch_all(+ query=f"""SELECT+ first_semi(source) AS source,+ sum(output * count) AS output,+ sum(count) AS count+ FROM stats.power_plant, countries.country_eez AS eez+ WHERE eez."union" = country+ AND {where_clause}+ AND time = :time+ GROUP BY first_semi(source)+ ORDER BY output DESC NULLS LAST+ """,+ values=values,+ )@alru_cache(maxsize=1000)diff --git a/web-backend/main.py b/web-backend/main.pyindex 3da274a..c2628e2 100644--- a/web-backend/main.py+++ b/web-backend/main.py@@ -23,7 +23,7 @@ from template_functions import (format_percent,format_external_url,)-from config import database, config+from config import database, DEBUGfrom util import cache_forfrom sitemap import sitemapfrom data import (@@ -32,7 +32,6 @@ from data import ()import charts-DEBUG = config("DEBUG", cast=bool, default=False)templates = Jinja2Templates(directory="templates")templates.env.filters["power"] = format_power@@ -146,3 +145,4 @@ async def stats_charts(request):import views.wikidata # noqaimport views.search # noqaimport views.area # noqa+import views.country # noqadiff --git a/web-backend/templates/_tables.html b/web-backend/templates/_tables.htmlindex 9ed773b..0751e30 100644--- a/web-backend/templates/_tables.html+++ b/web-backend/templates/_tables.html@@ -31,6 +31,5 @@</tr></tfoot></table>- <p class="footnote">Updated {{power_lines.date}}</p></div>{%- endmacro %}diff --git a/web-backend/templates/area.html b/web-backend/templates/area.htmlnew file mode 100644index 0000000..2b997de--- /dev/null+++ b/web-backend/templates/area.html@@ -0,0 +1,61 @@+{% extends "_base.html" %}+{% import "_tables.html" as tables %}+{% set breadcrumbs = [('Stats', url_for('stats')), (country, None)] %}+{% block title %}{{country}}{% endblock %}+{% block description -%}+Statistics on the electricity network in {{country|country_name}} from OpenStreetMap.+{%- endblock %}+{% block content -%}+<script src="https://cdn.bokeh.org/bokeh/release/bokeh-3.7.0.min.js"+ crossorigin="anonymous"></script>++ <p><strong>{{country|country_name(true)}}</strong> has+ {{plant_stats.count}} power plant{%if plant_stats.count != 1 %}s{% endif -%}+ {%- if plant_stats.output %}+ totalling {{plant_stats.output|power}}+ {%- endif %}+ and {{power_lines.total|distance}} of power lines mapped on OpenStreetMap.</p>++ <div class="table-wrapper">+ <table>+ <caption>Power plants in {{country|country_name}} by source</caption>+ <thead>+ <tr>+ <th>Source</th>+ <th>Output</th>+ <th>Count</th>+ <tr>+ </thead>+ <tbody>+ {% for row in plant_source_stats %}+ <tr>+ <td><a href="{{url_for('plants_region', region=country)}}?source={{row.source or ''}}">{{row.source or '[unspecified]'}}</a></td>+ <td>{{row.output|power}}</td>+ <td>{{row._mapping.count}}</td>+ </tr>+ {% endfor %}+ </tbody>+ <tfoot>+ <tr>+ <td><a href="{{url_for('plants_region', region=country)}}">All</a></td>+ <td>{{plant_stats.output|power}}</td>+ <td>{{plant_stats.count}}</td>+ </tr>+ </tfoot>+ </table>+ <p class="footnote">If multiple sources are listed for a power plant, only the first source is used in this breakdown.</p>+ </div>+ <p><a href="{{url_for('plants_construction_region', region=country)}}">Show plants under construction</a></p>+ <div id="plant-summary"></div>+ <script>+ Bokeh.embed.embed_item({{ plant_summary | safe }}, "plant-summary");+ </script>+ {% if power_lines.total > 1 %}+ {{ tables.power_lines_table(power_lines) }}+ <div id="country-grid-summary"></div>+ <script>+ Bokeh.embed.embed_item({{ country_grid_summary | safe }}, "country-grid-summary");+ </script>+ {% endif %}+ <p class="footnote">Stats updated {{stats_date.date()}}</p>+{% endblock %}diff --git a/web-backend/templates/country.html b/web-backend/templates/country.htmlindex bbc98c0..60a67f3 100644--- a/web-backend/templates/country.html+++ b/web-backend/templates/country.html@@ -1,60 +1,53 @@{% extends "_base.html" %}{% import "_tables.html" as tables %}-{% set breadcrumbs = [('Stats', url_for('stats')), (country, None)] %}-{% block title %}{{country}}{% endblock %}+{% set breadcrumbs = [('Stats', url_for('stats')), (country.name, None)] %}+{% block title %}{{country.name}}{% endblock %}{% block description -%}-Statistics on the electricity network in {{country|country_name}} from OpenStreetMap.+Statistics on the electricity network in {{country.name}} from OpenStreetMap.{%- endblock %}{% block content -%}-<script src="https://cdn.bokeh.org/bokeh/release/bokeh-3.7.0.min.js"- crossorigin="anonymous"></script>+<p>Stats for ISO territory {{country.name}} ({{country.alpha_3}}).</p>- <p><strong>{{country|country_name(true)}}</strong> has- {{plant_stats.count}} power plant{%if plant_stats.count != 1 %}s{% endif -%}- {%- if plant_stats.output %}- totalling {{plant_stats.output|power}}- {%- endif %}- and {{power_lines.total|distance}} of power lines mapped on OpenStreetMap.</p>+{% if lines.total > 1 %}+ {{ tables.power_lines_table(lines) }}+{% endif %}- <div class="table-wrapper">- <table>- <caption>Power plants in {{country|country_name}} by source</caption>- <thead>- <tr>- <th>Source</th>- <th>Output</th>- <th>Count</th>- <tr>- </thead>- <tbody>- {% for row in plant_source_stats %}- <tr>- <td><a href="{{url_for('plants_region', region=country)}}?source={{row.source or ''}}">{{row.source or '[unspecified]'}}</a></td>- <td>{{row.output|power}}</td>- <td>{{row._mapping.count}}</td>- </tr>- {% endfor %}- </tbody>- <tfoot>- <tr>- <td><a href="{{url_for('plants_region', region=country)}}">All</a></td>- <td>{{plant_stats.output|power}}</td>- <td>{{plant_stats.count}}</td>- </tr>- </tfoot>- </table>- <p class="footnote">If multiple sources are listed for a power plant, only the first source is used in this breakdown.</p>- </div>- <p><a href="{{url_for('plants_construction_region', region=country)}}">Show plants under construction</a></p>- <div id="plant-summary"></div>- <script>- Bokeh.embed.embed_item({{ plant_summary | safe }}, "plant-summary");- </script>- {% if power_lines.total > 1 %}- {{ tables.power_lines_table(power_lines) }}- <div id="country-grid-summary"></div>- <script>- Bokeh.embed.embed_item({{ country_grid_summary | safe }}, "country-grid-summary");- </script>- {% endif %}+<div class="table-wrapper">+ <table>+ <caption>Power plants in {{country.name}} by source</caption>+ <thead>+ <tr>+ <th>Source</th>+ <th>Output</th>+ <th>Count</th>+ <tr>+ </thead>+ <tbody>+ {% for row in plant_source_stats %}+ <tr>+ <td>{{row.source or '[unspecified]'}}</td>+ <td>{{row.output|power}}</td>+ <td>{{row._mapping.count}}</td>+ </tr>+ {% endfor %}+ </tbody>+ <tfoot>+ <tr>+ <td>All</td>+ <td>{{plant_stats.output|power}}</td>+ <td>{{plant_stats.count}}</td>+ </tr>+ </tfoot>+ </table>+ <p class="footnote">If multiple sources are listed for a power plant, only the first source is used in this breakdown.</p>+</div>++<h3>Parts</h3>+<p>This country has {{country_parts|length}} parts:</p>+<ul>+{% for part in country_parts %}+ <li><a href="{{url_for('region', region=part.union)}}">{{part.union}}</a></li>+{% endfor %}+</ul>+<p class="footnote">Stats updated {{stats_date.date()}}</p>{% endblock %}diff --git a/web-backend/util.py b/web-backend/util.pyindex 9f7a487..757137c 100644--- a/web-backend/util.py+++ b/web-backend/util.py@@ -1,8 +1,8 @@from functools import wrapsfrom starlette.exceptions import HTTPExceptionfrom urllib.parse import unquote_plus--from config import database+from datetime import timedelta+from config import database, DEBUGdef region_required(func):@@ -22,11 +22,23 @@ def region_required(func):return wrap_region-def cache_for(lifetime):+def cache_for(seconds: int = None, hours: int = None, days: int = None):def cache_for_inner(func):@wraps(func)async def wrap_cache(*args, **kwargs):response = await func(*args, **kwargs)+ if DEBUG:+ return response++ lifetime = timedelta(+ seconds=seconds or 0,+ hours=hours or 0,+ days=days or 0,+ ).total_seconds()++ if response.headers is None:+ return response+response.headers["Cache-Control"] = f"public, max-age={lifetime}"return responsediff --git a/web-backend/views/area.py b/web-backend/views/area.pyindex 309c99c..bad7e3e 100644--- a/web-backend/views/area.py+++ b/web-backend/views/area.py@@ -13,41 +13,22 @@ from data import (get_plant_generator_summary,get_wikidata,get_commons_thumbnail,+ plant_source_stats,+ plant_stats,+ latest_stats_date,)-import charts+import charts.country@app.route("/stats/area/{region}")@region_required-@cache_for(3600)+@cache_for(hours=1)async def region(request, region):+ stats_date = await latest_stats_date()async with asyncio.TaskGroup() as tg:- plant_stats = tg.create_task(- database.fetch_one(- query="""SELECT SUM(convert_power(output)) AS output, COUNT(*)- FROM power_plant, countries.country_eez_3857 AS eez- WHERE ST_Contains(eez.geom, geometry)- AND eez.gid = :gid- AND tags -> 'construction:power' IS NULL- """,- values={"gid": region["gid"]},- )- )-- plant_source_stats = tg.create_task(- database.fetch_all(- query="""SELECT first_semi(source) AS source, sum(convert_power(output)) AS output, count(*)- FROM power_plant, countries.country_eez_3857 AS eez- WHERE ST_Contains(eez.geom, geometry)- AND eez.gid = :gid- AND tags -> 'construction:power' IS NULL- GROUP BY first_semi(source)- ORDER BY SUM(convert_power(output)) DESC NULLS LAST""",- values={"gid": region["gid"]},- )- )-- power_lines = tg.create_task(stats_power_line(region["union"]))+ plants = tg.create_task(plant_stats(region["gid"], date=stats_date))+ sources = tg.create_task(plant_source_stats(region["gid"], date=stats_date))+ power_lines = tg.create_task(stats_power_line(region["union"], date=stats_date))grid_summary_chart = tg.create_task(charts.country.grid_summary(region["union"]))@@ -56,12 +37,13 @@ async def region(request, region):)return templates.TemplateResponse(- "country.html",+ "area.html",{"request": request,"country": region["union"],- "plant_stats": plant_stats.result()._mapping,- "plant_source_stats": plant_source_stats.result(),+ "stats_date": stats_date,+ "plant_stats": plants.result(),+ "plant_source_stats": sources.result(),"power_lines": power_lines.result(),"country_grid_summary": json.dumps(json_item(@@ -78,7 +60,7 @@ async def region(request, region):@app.route("/stats/area/{region}/plants")@region_required-@cache_for(3600)+@cache_for(hours=1)async def plants_region(request, region):gid = region[0]@@ -131,7 +113,7 @@ async def plants_region(request, region):@app.route("/stats/area/{region}/plants/construction")@region_required-@cache_for(3600)+@cache_for(hours=1)async def plants_construction_region(request, region):gid = region[0]@@ -162,7 +144,7 @@ async def plants_construction_region(request, region):@app.route("/stats/object/plant/{id}")-@cache_for(86400)+@cache_for(hours=1)async def stats_object(request):try:id = int(request.path_params["id"])@@ -185,7 +167,7 @@ async def stats_object(request):@app.route("/stats/area/{region}/plants/{id}")@region_required-@cache_for(3600)+@cache_for(hours=1)async def plant_detail(request, region):try:plant_id = int(request.path_params["id"])diff --git a/web-backend/views/country.py b/web-backend/views/country.pynew file mode 100644index 0000000..b0a4eb2--- /dev/null+++ b/web-backend/views/country.py@@ -0,0 +1,101 @@+import asyncio+from starlette.requests import Request+from starlette.responses import Response, JSONResponse+from starlette.exceptions import HTTPException+from starlette.responses import RedirectResponse++from main import app, database, templates+from util import cache_for+from data import stats_power_line, plant_source_stats, plant_stats, latest_stats_date+import pycountry+++def json_area_stats(lines: dict, plants, sources) -> dict:+ return {+ "lines": [+ {"min_voltage": k[0], "max_voltage": k[1], "length": v}+ for k, v in lines["lines"].items()+ ]+ + [{"min_voltage": None, "max_voltage": None, "length": lines["unspecified"]}],+ "plant_stats": plants,+ # "plant_source_stats": sources,+ }++[email protected]("/stats/country/{iso2}.json")+@cache_for(hours=1)+async def country_json(request: Request) -> Response:+ iso2 = request.path_params["iso2"]+ if iso2.upper() != iso2:+ return RedirectResponse(+ request.url.replace(path=f"/stats/country/{iso2.upper()}.json")+ )++ country = pycountry.countries.get(alpha_2=iso2)+ if not country:+ raise HTTPException(404, detail="Country not found")++ stats_date = await latest_stats_date()+ async with asyncio.TaskGroup() as tg:+ lines = tg.create_task(+ stats_power_line(territory_iso3=country.alpha_3, date=stats_date)+ )+ plants = tg.create_task(+ plant_stats(territory_iso3=country.alpha_3, date=stats_date)+ )+ sources = tg.create_task(+ plant_source_stats(territory_iso3=country.alpha_3, date=stats_date)+ )++ return JSONResponse(+ json_area_stats(lines.result(), plants.result(), sources.result())+ )++[email protected]("/stats/country/{iso2}")+@cache_for(hours=1)+async def country(request: Request) -> Response:+ iso2 = request.path_params["iso2"]+ if iso2.upper() != iso2:+ return RedirectResponse(+ request.url.replace(path=f"/stats/country/{iso2.upper()}")+ )++ country = pycountry.countries.get(alpha_2=iso2)+ if not country:+ raise HTTPException(404, detail="Country not found")++ stats_date = await latest_stats_date()+ async with asyncio.TaskGroup() as tg:+ country_parts = tg.create_task(+ database.fetch_all(+ """SELECT gid, "union" FROM countries.country_eez WHERE iso_sov1 = :iso3+ AND (iso_ter1 IS NULL OR iso_ter1 = iso_sov1)+ AND pol_type != 'Joint regime (EEZ)'+ """,+ values={"iso3": country.alpha_3},+ )+ )++ lines = tg.create_task(+ stats_power_line(territory_iso3=country.alpha_3, date=stats_date)+ )+ plants = tg.create_task(+ plant_stats(territory_iso3=country.alpha_3, date=stats_date)+ )+ sources = tg.create_task(+ plant_source_stats(territory_iso3=country.alpha_3, date=stats_date)+ )++ return templates.TemplateResponse(+ "country.html",+ {+ "stats_date": stats_date,+ "request": request,+ "country": country,+ "country_parts": country_parts.result(),+ "lines": lines.result(),+ "plant_stats": plants.result(),+ "plant_source_stats": sources.result(),+ },+ )