Skip to content

Open Infrastructure Map, a view of infrastructure data in OpenStreetMap

More languages

Russ Garrettaa3273c4a89b
diff --git a/schema/views.sql b/schema/views.sql
index a080a4c..4e6a897 100644
--- a/schema/views.sql
+++ b/schema/views.sql
@@ -1,32 +1,6 @@
--- Indexes for typeahead search
-CREATE INDEX CONCURRENTLY osm_power_substation_name
- ON osm_power_substation(substr(lower(tags->'name'), 1, 20) text_pattern_ops)
- WHERE tags->'name' IS NOT NULL;
-CREATE INDEX CONCURRENTLY osm_power_substation_name_en
- ON osm_power_substation(substr(lower(tags->'name:en'), 1, 20) text_pattern_ops)
- WHERE tags->'name:en' IS NOT NULL;
-
-CREATE INDEX CONCURRENTLY power_substation_relation_name
- ON power_substation_relation(substr(lower(name), 1, 20) text_pattern_ops)
- WHERE name IS NOT NULL;
-CREATE INDEX CONCURRENTLY power_substation_relation_name_en
- ON power_substation_relation(substr(lower(tags->'name:en'), 1, 20) text_pattern_ops)
- WHERE tags->'name:en' IS NOT NULL;
-
-CREATE INDEX CONCURRENTLY osm_power_plant_name
- ON osm_power_plant(substr(lower(tags->'name'), 1, 20) text_pattern_ops)
- WHERE tags->'name' IS NOT NULL;
-CREATE INDEX CONCURRENTLY osm_power_plant_name_en
- ON osm_power_plant(substr(lower(tags->'name:en'), 1, 20) text_pattern_ops)
- WHERE tags->'name:en' IS NOT NULL;
-
-CREATE INDEX CONCURRENTLY power_plant_relation_name
- ON power_plant_relation(substr(lower(name), 1, 20) text_pattern_ops)
- WHERE name IS NOT NULL;
-CREATE INDEX CONCURRENTLY power_plant_relation_name_en
- ON power_plant_relation(substr(lower(tags->'name:en'), 1, 20) text_pattern_ops)
- WHERE tags->'name:en' IS NOT NULL;
-
+---- Indexes for typeahead search
+-- You need to run the `create_indexes.py` script from the `web_backend` package
+-- to create language-specific indexes for the typeahead search.
---------------------------------------------------------
CREATE MATERIALIZED VIEW power_substation_relation AS
diff --git a/web-backend/create_indexes.py b/web-backend/create_indexes.py
new file mode 100644
index 0000000..5bc8751
--- /dev/null
+++ b/web-backend/create_indexes.py
@@ -0,0 +1,63 @@
+"""
+Programmatically create search indexes for tables. This has to be run manually and the
+`ADMIN_DATABASE_URL` must be set to a connection with the privileges to create indexes.
+"""
+
+from search import SEARCH_LANGUAGES
+from databases import Database
+from config import config
+
+MAIN_INDEXES = [
+ """CREATE INDEX CONCURRENTLY IF NOT EXISTS osm_power_substation_name
+ ON osm_power_substation(substr(lower(tags->'name'), 1, 20) text_pattern_ops)
+ WHERE tags->'name' IS NOT NULL""",
+ """CREATE INDEX CONCURRENTLY IF NOT EXISTS power_substation_relation_name
+ ON power_substation_relation(substr(lower(name), 1, 20) text_pattern_ops)
+ WHERE name IS NOT NULL""",
+ """CREATE INDEX CONCURRENTLY IF NOT EXISTS osm_power_plant_name
+ ON osm_power_plant(substr(lower(tags->'name'), 1, 20) text_pattern_ops)
+ WHERE tags->'name' IS NOT NULL""",
+ """CREATE INDEX CONCURRENTLY IF NOT EXISTS power_plant_relation_name
+ ON power_plant_relation(substr(lower(name), 1, 20) text_pattern_ops)
+ WHERE name IS NOT NULL""",
+]
+
+
+async def create_indexes():
+ database = Database(config("ADMIN_DATABASE_URL"))
+ await database.connect()
+ print("Creating main indexes...")
+ for index in MAIN_INDEXES:
+ print(index)
+ await database.execute(index)
+
+ for lang in SEARCH_LANGUAGES:
+ print(f"Creating indexes for language: {lang}...", end="", flush=True)
+ queries = [
+ f"""CREATE INDEX CONCURRENTLY IF NOT EXISTS osm_power_substation_name_{lang}
+ ON osm_power_substation(substr(lower(tags->'name:{lang}'), 1, 20) text_pattern_ops)
+ WHERE tags->'name:{lang}' IS NOT NULL""",
+ f"""CREATE INDEX CONCURRENTLY IF NOT EXISTS power_substation_relation_name_{lang}
+ ON power_substation_relation(substr(lower(tags->'name:{lang}'), 1, 20) text_pattern_ops)
+ WHERE tags->'name:{lang}' IS NOT NULL""",
+ f"""CREATE INDEX CONCURRENTLY IF NOT EXISTS osm_power_plant_name_{lang}
+ ON osm_power_plant(substr(lower(tags->'name:{lang}'), 1, 20) text_pattern_ops)
+ WHERE tags->'name:{lang}' IS NOT NULL""",
+ f"""CREATE INDEX CONCURRENTLY IF NOT EXISTS power_plant_relation_name_{lang}
+ ON power_plant_relation(substr(lower(tags->'name:{lang}'), 1, 20) text_pattern_ops)
+ WHERE tags->'name:{lang}' IS NOT NULL""",
+ ]
+ for query in queries:
+ print(".", end="", flush=True)
+ await database.execute(query)
+ print("")
+ print("Indexes created successfully.", flush=True)
+ await database.disconnect()
+
+
+if __name__ == "__main__":
+ import asyncio
+
+ loop = asyncio.new_event_loop()
+ loop.run_until_complete(create_indexes())
+ loop.close()
diff --git a/web-backend/search.py b/web-backend/search.py
index 6e17e9a..9fc3dde 100644
--- a/web-backend/search.py
+++ b/web-backend/search.py
@@ -10,8 +10,22 @@ from main import app
from config import database
from util import cache_for
-# Ensure languages in this list have an index in the database.
-SEARCH_LANGUAGES = {"en"}
+# Ensure the `create_indexes` script is run after changing this list.
+SEARCH_LANGUAGES = {
+ "en",
+ "es",
+ "el",
+ "de",
+ "fr",
+ "hi",
+ "ur",
+ "zh",
+ "ru",
+ "pt",
+ "ja",
+ "it",
+ "nl",
+}
async def search_substations(
@@ -122,8 +136,8 @@ async def search(request):
start = time.monotonic_ns()
async with asyncio.TaskGroup() as tg:
- substations = tg.create_task(search_substations(query, language))
- plants = tg.create_task(search_plants(query, language))
+ substations = tg.create_task(search_substations(query, language, limit))
+ plants = tg.create_task(search_plants(query, language, limit))
results = sorted(await substations + await plants, key=sort_key, reverse=True)
end = time.monotonic_ns()