Build your personal knowledge base with Trilium Notes
TypeScript electronelectron-appknowledge-baseknowledge-graphknowledge-managementknowledge-management-graphlocal-firstnote-managmentnote-takernote-takingnotebooknotesnotes-apppersonal-knowledge-basepersonal-wikiscriptableself-hostedself-hostingwiki
feat(geomap): search places through Nominatim
Elian Dorance991855afc0
diff --git a/apps/client/src/widgets/collections/geomap/SearchBox.spec.tsx b/apps/client/src/widgets/collections/geomap/SearchBox.spec.tsxindex 59b0f1ab92..bc0258e634 100644--- a/apps/client/src/widgets/collections/geomap/SearchBox.spec.tsx+++ b/apps/client/src/widgets/collections/geomap/SearchBox.spec.tsx@@ -10,7 +10,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";import type FNote from "../../../entities/fnote";import { buildNote } from "../../../test/easy-froca";import { renderInto } from "../../../test/render";-import { GEOCODING_PROVIDERS } from "./geocoding";+import { DEFAULT_GEOCODING_PROVIDER_NAME, GEOCODING_PROVIDERS, type GeoSearchResult } from "./geocoding";import { ParentMap } from "./map";import SearchBox from "./SearchBox";@@ -20,6 +20,14 @@ vi.mock("../../../services/i18n", () => ({t: (key: string, vars?: Record<string, unknown>) => (vars ? `${key}(${Object.values(vars).join(",")})` : key)}));+const TOKYO: GeoSearchResult = { id: "1", label: "Tokyo, Japan", lat: 35.6762, lng: 139.6503 };++/** Stands in for the geocoder, which would otherwise reach for the network. */+function mockGeocoder(results: GeoSearchResult[]) {+ return vi.spyOn(GEOCODING_PROVIDERS[DEFAULT_GEOCODING_PROVIDER_NAME], "search")+ .mockResolvedValue(results);+}+/** A map that can be flown somewhere, which is all the bar uses. */function fakeMap() {return { flyTo: vi.fn() };@@ -94,6 +102,7 @@ afterEach(() => {describe("geo map SearchBox", () => {it("lists the map's own markers by title, skipping notes with nowhere to fly to", async () => {+ mockGeocoder([]);const container = renderSearchBox(fakeMap(), mapNotes());await type(container, "london");@@ -115,7 +124,7 @@ describe("geo map SearchBox", () => {});it("does not reach the geocoder until its row is picked", async () => {- const search = vi.spyOn(GEOCODING_PROVIDERS.dummy, "search");+ const search = mockGeocoder([ TOKYO ]);const container = renderSearchBox(fakeMap(), mapNotes());await type(container, "tokyo");@@ -130,6 +139,7 @@ describe("geo map SearchBox", () => {});it("flies to a marker, and closes the list rather than leaving it open over the map", async () => {+ mockGeocoder([]);const map = fakeMap();const container = renderSearchBox(map, mapNotes());@@ -142,6 +152,7 @@ describe("geo map SearchBox", () => {});it("flies to a geocoded place, further out than to a marker", async () => {+ mockGeocoder([ TOKYO ]);const map = fakeMap();const container = renderSearchBox(map, mapNotes());@@ -160,19 +171,21 @@ describe("geo map SearchBox", () => {});it("says so when the geocoder finds nothing, and when it cannot be reached", async () => {+ mockGeocoder([]);const container = renderSearchBox(fakeMap(), []);await type(container, "atlantis");await pick(0);expect(labels()).toEqual([ "geo-map.no-places-found" ]);- vi.spyOn(GEOCODING_PROVIDERS.dummy, "search").mockRejectedValue(new Error("rate limited"));+ mockGeocoder([]).mockRejectedValue(new Error("rate limited"));await type(container, "berlin");await pick(0);expect(labels()).toEqual([ "geo-map.search-failed" ]);});it("offers the geocoder again once the query moves on from what it answered", async () => {+ mockGeocoder([ { id: "2", label: "Berlin, Germany", lat: 52.52, lng: 13.405 } ]);const container = renderSearchBox(fakeMap(), []);await type(container, "berlin");diff --git a/apps/client/src/widgets/collections/geomap/geocoding.spec.ts b/apps/client/src/widgets/collections/geomap/geocoding.spec.tsnew file mode 100644index 0000000000..7edd0d22cd--- /dev/null+++ b/apps/client/src/widgets/collections/geomap/geocoding.spec.ts@@ -0,0 +1,104 @@+/**+ * The Nominatim geocoder: the request it makes, what it makes of the answer, and the rate its usage+ * policy holds it to.+ */+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";++import { DEFAULT_GEOCODING_PROVIDER_NAME, GEOCODING_PROVIDERS } from "./geocoding";++vi.mock("../../../services/i18n", () => ({ getCurrentLanguage: () => "pt_br" }));++const provider = GEOCODING_PROVIDERS[DEFAULT_GEOCODING_PROVIDER_NAME];++/** One entry as Nominatim's `jsonv2` answer carries it. */+function place(overrides: Record<string, unknown> = {}) {+ return {+ place_id: 240109189,+ display_name: "Berlin, Germany",+ lat: "52.5170365",+ lon: "13.3888599",+ ...overrides+ };+}++function respondWith(body: unknown, init: { ok?: boolean; status?: number } = {}) {+ const fetchMock = vi.fn(async (_url: string) => ({+ ok: init.ok ?? true,+ status: init.status ?? 200,+ statusText: "",+ json: async () => body+ }));+ vi.stubGlobal("fetch", fetchMock);+ return fetchMock;+}++/** Runs a search, letting the rate limiter's wait elapse rather than sitting through it. */+async function search(query: string) {+ const results = provider.search(query);+ await vi.runAllTimersAsync();+ return results;+}++function requestedUrl(fetchMock: ReturnType<typeof respondWith>) {+ return new URL(fetchMock.mock.calls[0][0]);+}++beforeEach(() => vi.useFakeTimers());+afterEach(() => {+ vi.useRealTimers();+ vi.unstubAllGlobals();+});++describe("Nominatim geocoding", () => {+ it("asks Nominatim for the query, capped, in the language the app runs in", async () => {+ const fetchMock = respondWith([]);++ await search("berlin");++ const url = requestedUrl(fetchMock);+ expect(url.origin + url.pathname).toBe("https://nominatim.openstreetmap.org/search");+ expect(url.searchParams.get("q")).toBe("berlin");+ expect(url.searchParams.get("format")).toBe("jsonv2");+ expect(Number(url.searchParams.get("limit"))).toBeGreaterThan(0);+ // Written as a language tag rather than as Trilium's own locale id.+ expect(url.searchParams.get("accept-language")).toBe("pt-br");+ });++ it("takes the places from the answer, and leaves out any without a readable position", async () => {+ respondWith([+ place(),+ place({ place_id: 2, display_name: "Nowhere", lat: "not a number" }),+ place({ place_id: 3, display_name: undefined }),+ place({ place_id: 4, display_name: "Paris, France", lat: "48.8566", lon: "2.3522" })+ ]);++ expect(await search("anywhere")).toEqual([+ { id: "240109189", label: "Berlin, Germany", lat: 52.5170365, lng: 13.3888599 },+ { id: "4", label: "Paris, France", lat: 48.8566, lng: 2.3522 }+ ]);+ });++ it("reports a refused request rather than answering with no places", async () => {+ respondWith([], { ok: false, status: 429 });++ // The expectation is attached before the timers run, so the rejection is never loose.+ const refused = expect(provider.search("berlin")).rejects.toThrow("429");+ await vi.runAllTimersAsync();+ await refused;+ });++ it("holds searches to the one request a second the usage policy allows", async () => {+ const requestedAt: number[] = [];+ vi.stubGlobal("fetch", vi.fn(async () => {+ requestedAt.push(Date.now());+ return { ok: true, status: 200, statusText: "", json: async () => [] };+ }));++ const first = provider.search("berlin");+ const second = provider.search("paris");+ await vi.runAllTimersAsync();+ await Promise.all([ first, second ]);++ expect(requestedAt[1] - requestedAt[0]).toBeGreaterThanOrEqual(1000);+ });+});diff --git a/apps/client/src/widgets/collections/geomap/geocoding.ts b/apps/client/src/widgets/collections/geomap/geocoding.tsindex 16ad20d8a4..46e839f325 100644--- a/apps/client/src/widgets/collections/geomap/geocoding.ts+++ b/apps/client/src/widgets/collections/geomap/geocoding.ts@@ -1,3 +1,5 @@+import { getCurrentLanguage } from "../../../services/i18n";+/** A single place returned by a geocoder. */export interface GeoSearchResult {/** Unique within one result set. */@@ -11,55 +13,112 @@ export interface GeoSearchResult {export interface GeocodingProvider {/** Display name, shown where the user picks a provider. */name: string;- /** Returns matches, best first, or an empty array when nothing matches. */+ /**+ * Returns matches, best first, or an empty array when nothing matches. Throws when the provider+ * cannot be reached or refuses the request.+ */search(query: string): Promise<GeoSearchResult[]>;}/*** The geocoders a map can search, keyed by the name a note selects one by.*- * Same shape as `MAP_LAYERS` in map_layer.ts, so both are read the same way. Only the dummy provider- * exists so far; a real one is a new entry here.+ * Same shape as `MAP_LAYERS` in map_layer.ts, so both are read the same way.*/export const GEOCODING_PROVIDERS: Record<string, GeocodingProvider> = {- "dummy": {- name: "Dummy",- search: searchDummyPlaces+ "nominatim": {+ name: "Nominatim (OpenStreetMap)",+ search: searchNominatim}};-export const DEFAULT_GEOCODING_PROVIDER_NAME: keyof typeof GEOCODING_PROVIDERS = "dummy";--/** The places `searchDummyPlaces()` matches against. */-const DUMMY_PLACES: GeoSearchResult[] = [- { id: "dummy-amsterdam", label: "Amsterdam, Netherlands", lat: 52.3676, lng: 4.9041 },- { id: "dummy-berlin", label: "Berlin, Germany", lat: 52.52, lng: 13.405 },- { id: "dummy-bucharest", label: "Bucharest, Romania", lat: 44.4268, lng: 26.1025 },- { id: "dummy-cluj", label: "Cluj-Napoca, Cluj, Romania", lat: 46.7712, lng: 23.6236 },- { id: "dummy-lisbon", label: "Lisbon, Portugal", lat: 38.7223, lng: -9.1393 },- { id: "dummy-london", label: "London, England, United Kingdom", lat: 51.5072, lng: -0.1276 },- { id: "dummy-london-on", label: "London, Ontario, Canada", lat: 42.9849, lng: -81.2453 },- { id: "dummy-new-york", label: "New York, New York, United States", lat: 40.7128, lng: -74.006 },- { id: "dummy-paris", label: "Paris, Île-de-France, France", lat: 48.8566, lng: 2.3522 },- { id: "dummy-reykjavik", label: "Reykjavík, Iceland", lat: 64.1466, lng: -21.9426 },- { id: "dummy-sydney", label: "Sydney, New South Wales, Australia", lat: -33.8688, lng: 151.2093 },- { id: "dummy-tokyo", label: "Tokyo, Japan", lat: 35.6762, lng: 139.6503 }-];+export const DEFAULT_GEOCODING_PROVIDER_NAME: keyof typeof GEOCODING_PROVIDERS = "nominatim";++const NOMINATIM_URL = "https://nominatim.openstreetmap.org/search";/** Caps how many results one search returns. */const MAX_RESULTS = 8;/**- * Matches `DUMMY_PLACES` by substring, so the search bar works before a real geocoder is wired up.- * Async like a real provider, so callers cannot depend on results arriving in the same tick.+ * Nominatim's usage policy allows at most one request per second, and blocks clients that exceed it.+ * See https://operations.osmfoundation.org/policies/nominatim/.+ */+const MIN_REQUEST_INTERVAL = 1000;++/** When the next request is allowed to go out, moved forward by every request that is queued. */+let nextRequestAt = 0;++/**+ * Looks a query up through Nominatim, the geocoder run by the OpenStreetMap Foundation.+ *+ * No API key, and it answers with `Access-Control-Allow-Origin: *`, so the browser can ask it+ * directly — which matters on desktop, where the page is served from `trilium-app://`. Its policy+ * also asks that clients identify themselves, which a browser can only do through the `Referer` it+ * sends of its own accord.+ *+ * The same policy rules out searching as the user types, which is why a search runs only when the+ * user asks for one (see SearchBox).*/-async function searchDummyPlaces(query: string): Promise<GeoSearchResult[]> {- const needle = query.trim().toLowerCase();- if (!needle) {- return [];+async function searchNominatim(query: string): Promise<GeoSearchResult[]> {+ const params = new URLSearchParams({+ q: query,+ format: "jsonv2",+ limit: String(MAX_RESULTS)+ });++ // Place names in the language the app runs in, where Nominatim holds one. Trilium writes some+ // locale ids with an underscore (`pt_br`), which is not what a language tag looks like.+ const language = getCurrentLanguage();+ if (language) {+ params.set("accept-language", language.replace("_", "-"));+ }++ await waitForRequestSlot();+ const response = await fetch(`${NOMINATIM_URL}?${params}`);+ if (!response.ok) {+ throw new Error(`Nominatim answered ${response.status} ${response.statusText}`);+ }++ const places: NominatimPlace[] = await response.json();+ return places.map(toSearchResult).filter((result) => result !== null);+}++/** One entry of a Nominatim `jsonv2` response, narrowed to the fields a result is built from. */+interface NominatimPlace {+ place_id?: number;+ display_name?: string;+ lat?: string;+ lon?: string;+}++/** A place as the app holds one, or `null` where the entry carries no readable position. */+function toSearchResult(place: NominatimPlace): GeoSearchResult | null {+ const lat = Number(place.lat);+ const lng = Number(place.lon);+ if (!place.display_name || !Number.isFinite(lat) || !Number.isFinite(lng)) {+ return null;}- return DUMMY_PLACES- .filter((place) => place.label.toLowerCase().includes(needle))- .slice(0, MAX_RESULTS);+ return {+ id: String(place.place_id ?? `${lat},${lng}`),+ label: place.display_name,+ lat,+ lng+ };+}++/**+ * Holds a request back until the interval the policy asks for has passed since the last one.+ *+ * The slot is claimed before the wait rather than after it, so two searches started at once queue+ * behind each other instead of both finding the same slot free.+ */+async function waitForRequestSlot() {+ const now = Date.now();+ const slot = Math.max(now, nextRequestAt);+ nextRequestAt = slot + MIN_REQUEST_INTERVAL;++ if (slot > now) {+ await new Promise((resolve) => setTimeout(resolve, slot - now));+ }}