Open Infrastructure Map, a view of infrastructure data in OpenStreetMap
More languages
Russ Garrettaa3273c4a89b
diff --git a/schema/views.sql b/schema/views.sqlindex a080a4c..4e6a897 100644--- a/schema/views.sql+++ b/schema/views.sql@@ -1,32 +1,6 @@--- Indexes for typeahead search-CREATE INDEX CONCURRENTLY osm_power_substation_name- ON osm_power_substation(substr(lower(tags->'name'), 1, 20) text_pattern_ops)- WHERE tags->'name' IS NOT NULL;-CREATE INDEX CONCURRENTLY osm_power_substation_name_en- ON osm_power_substation(substr(lower(tags->'name:en'), 1, 20) text_pattern_ops)- WHERE tags->'name:en' IS NOT NULL;--CREATE INDEX CONCURRENTLY power_substation_relation_name- ON power_substation_relation(substr(lower(name), 1, 20) text_pattern_ops)- WHERE name IS NOT NULL;-CREATE INDEX CONCURRENTLY power_substation_relation_name_en- ON power_substation_relation(substr(lower(tags->'name:en'), 1, 20) text_pattern_ops)- WHERE tags->'name:en' IS NOT NULL;--CREATE INDEX CONCURRENTLY osm_power_plant_name- ON osm_power_plant(substr(lower(tags->'name'), 1, 20) text_pattern_ops)- WHERE tags->'name' IS NOT NULL;-CREATE INDEX CONCURRENTLY osm_power_plant_name_en- ON osm_power_plant(substr(lower(tags->'name:en'), 1, 20) text_pattern_ops)- WHERE tags->'name:en' IS NOT NULL;--CREATE INDEX CONCURRENTLY power_plant_relation_name- ON power_plant_relation(substr(lower(name), 1, 20) text_pattern_ops)- WHERE name IS NOT NULL;-CREATE INDEX CONCURRENTLY power_plant_relation_name_en- ON power_plant_relation(substr(lower(tags->'name:en'), 1, 20) text_pattern_ops)- WHERE tags->'name:en' IS NOT NULL;-+---- Indexes for typeahead search+-- You need to run the `create_indexes.py` script from the `web_backend` package+-- to create language-specific indexes for the typeahead search.---------------------------------------------------------CREATE MATERIALIZED VIEW power_substation_relation ASdiff --git a/web-backend/create_indexes.py b/web-backend/create_indexes.pynew file mode 100644index 0000000..5bc8751--- /dev/null+++ b/web-backend/create_indexes.py@@ -0,0 +1,63 @@+"""+Programmatically create search indexes for tables. This has to be run manually and the+`ADMIN_DATABASE_URL` must be set to a connection with the privileges to create indexes.+"""++from search import SEARCH_LANGUAGES+from databases import Database+from config import config++MAIN_INDEXES = [+ """CREATE INDEX CONCURRENTLY IF NOT EXISTS osm_power_substation_name+ ON osm_power_substation(substr(lower(tags->'name'), 1, 20) text_pattern_ops)+ WHERE tags->'name' IS NOT NULL""",+ """CREATE INDEX CONCURRENTLY IF NOT EXISTS power_substation_relation_name+ ON power_substation_relation(substr(lower(name), 1, 20) text_pattern_ops)+ WHERE name IS NOT NULL""",+ """CREATE INDEX CONCURRENTLY IF NOT EXISTS osm_power_plant_name+ ON osm_power_plant(substr(lower(tags->'name'), 1, 20) text_pattern_ops)+ WHERE tags->'name' IS NOT NULL""",+ """CREATE INDEX CONCURRENTLY IF NOT EXISTS power_plant_relation_name+ ON power_plant_relation(substr(lower(name), 1, 20) text_pattern_ops)+ WHERE name IS NOT NULL""",+]+++async def create_indexes():+ database = Database(config("ADMIN_DATABASE_URL"))+ await database.connect()+ print("Creating main indexes...")+ for index in MAIN_INDEXES:+ print(index)+ await database.execute(index)++ for lang in SEARCH_LANGUAGES:+ print(f"Creating indexes for language: {lang}...", end="", flush=True)+ queries = [+ f"""CREATE INDEX CONCURRENTLY IF NOT EXISTS osm_power_substation_name_{lang}+ ON osm_power_substation(substr(lower(tags->'name:{lang}'), 1, 20) text_pattern_ops)+ WHERE tags->'name:{lang}' IS NOT NULL""",+ f"""CREATE INDEX CONCURRENTLY IF NOT EXISTS power_substation_relation_name_{lang}+ ON power_substation_relation(substr(lower(tags->'name:{lang}'), 1, 20) text_pattern_ops)+ WHERE tags->'name:{lang}' IS NOT NULL""",+ f"""CREATE INDEX CONCURRENTLY IF NOT EXISTS osm_power_plant_name_{lang}+ ON osm_power_plant(substr(lower(tags->'name:{lang}'), 1, 20) text_pattern_ops)+ WHERE tags->'name:{lang}' IS NOT NULL""",+ f"""CREATE INDEX CONCURRENTLY IF NOT EXISTS power_plant_relation_name_{lang}+ ON power_plant_relation(substr(lower(tags->'name:{lang}'), 1, 20) text_pattern_ops)+ WHERE tags->'name:{lang}' IS NOT NULL""",+ ]+ for query in queries:+ print(".", end="", flush=True)+ await database.execute(query)+ print("")+ print("Indexes created successfully.", flush=True)+ await database.disconnect()+++if __name__ == "__main__":+ import asyncio++ loop = asyncio.new_event_loop()+ loop.run_until_complete(create_indexes())+ loop.close()diff --git a/web-backend/search.py b/web-backend/search.pyindex 6e17e9a..9fc3dde 100644--- a/web-backend/search.py+++ b/web-backend/search.py@@ -10,8 +10,22 @@ from main import appfrom config import databasefrom util import cache_for-# Ensure languages in this list have an index in the database.-SEARCH_LANGUAGES = {"en"}+# Ensure the `create_indexes` script is run after changing this list.+SEARCH_LANGUAGES = {+ "en",+ "es",+ "el",+ "de",+ "fr",+ "hi",+ "ur",+ "zh",+ "ru",+ "pt",+ "ja",+ "it",+ "nl",+}async def search_substations(@@ -122,8 +136,8 @@ async def search(request):start = time.monotonic_ns()async with asyncio.TaskGroup() as tg:- substations = tg.create_task(search_substations(query, language))- plants = tg.create_task(search_plants(query, language))+ substations = tg.create_task(search_substations(query, language, limit))+ plants = tg.create_task(search_plants(query, language, limit))results = sorted(await substations + await plants, key=sort_key, reverse=True)end = time.monotonic_ns()