The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Fix flaky GeoJS-dependent tests by mocking external service (#68623)
bryana45d9203f174
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex 887c6f4fa4c..d538652bcbb 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -1200,7 +1200,8 @@:api #{metabase.request.coremetabase.request.initmetabase.request.schema}- :uses #{api+ :uses #{analytics+ apiconfigembeddingpermissions ; FIXMEdiff --git a/src/metabase/analytics/prometheus.clj b/src/metabase/analytics/prometheus.cljindex 48823a9f772..4319dceb09e 100644--- a/src/metabase/analytics/prometheus.clj+++ b/src/metabase/analytics/prometheus.clj@@ -241,6 +241,10 @@{:description "Number of emails sent."})(prometheus/counter :metabase-email/message-errors{:description "Number of errors when sending emails."})+ (prometheus/counter :metabase-geocoding/requests+ {:description "Number of successful IP geocoding requests via GeoJS."})+ (prometheus/counter :metabase-geocoding/errors+ {:description "Number of errors when geocoding IP addresses via GeoJS."})(prometheus/counter :metabase-scim/response-ok{:description "Number of successful responses from SCIM endpoints"})(prometheus/counter :metabase-scim/response-errordiff --git a/src/metabase/request/util.clj b/src/metabase/request/util.cljindex 87300c3ec8b..65083615255 100644--- a/src/metabase/request/util.clj+++ b/src/metabase/request/util.clj@@ -4,6 +4,7 @@[clj-http.client :as http][clojure.string :as str][java-time.api :as t]+ [metabase.analytics.core :as analytics][metabase.config.core :as config][metabase.embedding.util :as embed.util][metabase.request.settings :as request.settings]@@ -152,11 +153,14 @@:socket-timeout gecode-ip-address-timeout-ms:connection-timeout gecode-ip-address-timeout-ms}):body- json/decode+kw)]- (into {} (for [info response]- [(:ip info) {:description (or (describe-location info)- "Unknown location")- :timezone (u/ignore-exceptions (some-> (:timezone info) t/zone-id))}])))+ json/decode+kw)+ result (into {} (for [info response]+ [(:ip info) {:description (or (describe-location info)+ "Unknown location")+ :timezone (u/ignore-exceptions (some-> (:timezone info) t/zone-id))}]))]+ (analytics/inc! :metabase-geocoding/requests)+ result)(catch Throwable e+ (analytics/inc! :metabase-geocoding/errors)(log/error e "Error geocoding IP addresses" {:url url})nil))))))diff --git a/test/metabase/login_history/api_test.clj b/test/metabase/login_history/api_test.cljindex adae6a891a5..4c74f931d91 100644--- a/test/metabase/login_history/api_test.clj+++ b/test/metabase/login_history/api_test.clj@@ -1,6 +1,8 @@(ns metabase.login-history.api-test(:require[clojure.test :refer :all]+ [java-time.api :as t]+ [metabase.request.util :as req.util][metabase.session.core :as session][metabase.test :as mt][metabase.util :as u]))@@ -48,46 +50,49 @@:device_id device-id:device_description windows-user-agent:ip_address "52.206.149.9"}]- ;; GeoJS is having issues. We have safe error handling so we expect either nil or the expected values.- ;; https://github.com/jloh/geojs/issues/48- ;; The timestamps will also need to be updated (to be in the TZ, not in Zulu time)- ;;- ;; A Slack reminder has been set to follow up on this- (is (malli= [:cat- ;; localhost ipv6- [:map- [:timestamp [:= "2021-03-18T20:55:50.955232Z"]]- [:device_description [:= "Mobile Browser (Mobile Safari/iOS)"]]- [:ip_address [:= "0:0:0:0:0:0:0:1"]]- [:active [:= false]]- [:location [:maybe [:= "Unknown location"]]]- [:timezone nil?]]- ;; localhost ipv4- [:map- [:timestamp [:= "2021-03-18T20:04:24.7273Z"]]- [:device_description [:= "Library (Apache-HttpClient/JVM (Java))"]]- [:ip_address [:= "127.0.0.1"]]- [:active [:= false]]- [:location [:maybe [:= "Unknown location"]]]- [:timezone nil?]]- ;; France- [:map- [:timestamp [:enum- "2021-03-18T20:52:41.808482+01:00"- "2021-03-18T19:52:41.808482Z"]]- [:device_description [:= "Browser (Chrome/Windows)"]]- [:ip_address [:= "185.233.100.23"]]- [:active [:= true]]- [:location [:maybe [:re #"France"]]]- [:timezone [:maybe [:= "CET"]]]]- ;; Virginia- [:map- [:timestamp [:enum- "2021-03-18T15:52:20.172351-04:00"- "2021-03-18T19:52:20.172351Z"]]- [:device_description [:= "Browser (Chrome/Windows)"]]- [:ip_address [:= "52.206.149.9"]]- [:active [:= false]]- [:location [:maybe [:re "Virginia, United States"]]]- [:timezone [:maybe [:= "ET"]]]]]- (mt/client session-key :get 200 "login-history/current")))))))+ ;; Mock geocoding to avoid external GeoJS dependency+ (with-redefs [req.util/geocode-ip-addresses+ (fn [ip-addresses]+ (into {}+ (for [ip ip-addresses]+ [ip (case ip+ "185.233.100.23" {:description "Paris, France"+ :timezone (t/zone-id "Europe/Paris")}+ "52.206.149.9" {:description "Virginia, United States"+ :timezone (t/zone-id "America/New_York")}+ {:description "Unknown location"+ :timezone nil})])))]+ (is (malli= [:cat+ ;; localhost ipv6+ [:map+ [:timestamp [:= "2021-03-18T20:55:50.955232Z"]]+ [:device_description [:= "Mobile Browser (Mobile Safari/iOS)"]]+ [:ip_address [:= "0:0:0:0:0:0:0:1"]]+ [:active [:= false]]+ [:location [:= "Unknown location"]]+ [:timezone nil?]]+ ;; localhost ipv4+ [:map+ [:timestamp [:= "2021-03-18T20:04:24.7273Z"]]+ [:device_description [:= "Library (Apache-HttpClient/JVM (Java))"]]+ [:ip_address [:= "127.0.0.1"]]+ [:active [:= false]]+ [:location [:= "Unknown location"]]+ [:timezone nil?]]+ ;; France (Paris) - timestamp converted to Europe/Paris timezone+ [:map+ [:timestamp [:= "2021-03-18T20:52:41.808482+01:00"]]+ [:device_description [:= "Browser (Chrome/Windows)"]]+ [:ip_address [:= "185.233.100.23"]]+ [:active [:= true]]+ [:location [:= "Paris, France"]]+ [:timezone [:= "CET"]]]+ ;; Virginia - timestamp converted to America/New_York timezone+ [:map+ [:timestamp [:= "2021-03-18T15:52:20.172351-04:00"]]+ [:device_description [:= "Browser (Chrome/Windows)"]]+ [:ip_address [:= "52.206.149.9"]]+ [:active [:= false]]+ [:location [:= "Virginia, United States"]]+ [:timezone [:= "ET"]]]]+ (mt/client session-key :get 200 "login-history/current"))))))))diff --git a/test/metabase/request/util_test.clj b/test/metabase/request/util_test.cljindex dc42e06f46d..55a07418d28 100644--- a/test/metabase/request/util_test.clj+++ b/test/metabase/request/util_test.clj@@ -1,10 +1,13 @@(ns metabase.request.util-test(:require+ [clj-http.client :as http]+ [clojure.string :as str][clojure.test :refer :all][clojure.tools.reader.edn :as edn][java-time.api :as t][metabase.request.util :as req.util][metabase.test :as mt]+ [metabase.util.json :as json][ring.mock.request :as ring.mock]))(deftest ^:parallel cacheable?-test@@ -115,47 +118,77 @@(is (= "127.0.0.1"(req.util/ip-address mock-request))))))))-(deftest ^:parallel geocode-ip-addresses-test- (are [ip-addresses expected] (malli= [:maybe expected]- (req.util/geocode-ip-addresses ip-addresses))- ;; Google DNS- ["8.8.8.8"]- [:map- ["8.8.8.8" [:map- [:description [:re #"United States"]]- [:timezone [:= (t/zone-id "America/Chicago")]]]]]-- ;; this is from the MaxMind sample high-risk IP address list https://www.maxmind.com/en/high-risk-ip-sample-list- ["185.233.100.23"]- [:map- ["185.233.100.23" [:map- [:description [:re #"France"]]- [:timezone [:= (t/zone-id "Europe/Paris")]]]]]-- ["127.0.0.1"]- [:map- ["127.0.0.1" [:map- [:description [:= "Unknown location"]]- [:timezone :nil]]]]-- ["0:0:0:0:0:0:0:1"]- [:map- ["0:0:0:0:0:0:0:1" [:map- [:description [:= "Unknown location"]]- [:timezone :nil]]]]-- ;; multiple addresses at once- ;; store.metabase.com, Google DNS- ["52.206.149.9" "2001:4860:4860::8844"]- [:map- ["52.206.149.9" [:map- [:description [:re #"United States"]]- [:timezone [:= (t/zone-id "America/New_York")]]]]- ["2001:4860:4860::8844" [:map- [:description [:re #"United States"]]- [:timezone [:= (t/zone-id "America/Chicago")]]]]]-- ["wow"] :nil- [" "] :nil- [] :nil- nil :nil))+(def ^:private mock-geojs-responses+ "Canned GeoJS responses for test IPs. These mock what GeoJS would return."+ {"8.8.8.8" {:city "" :region "" :country "United States" :timezone "America/Chicago"}+ "185.233.100.23" {:city "Paris" :region "" :country "France" :timezone "Europe/Paris"}+ "127.0.0.1" {:city "" :region "" :country "" :timezone nil}+ "0:0:0:0:0:0:0:1" {:city "" :region "" :country "" :timezone nil}+ "52.206.149.9" {:city "" :region "Virginia" :country "United States" :timezone "America/New_York"}+ "2001:4860:4860::8844" {:city "" :region "" :country "United States" :timezone "America/Chicago"}})++(defn- mock-geojs-http-get+ "Mock HTTP GET that returns canned GeoJS responses for test IPs."+ [url _opts]+ (let [ips (-> url (str/split #"\?ip=") second (str/split #","))]+ {:body (json/encode (mapv #(assoc (get mock-geojs-responses % {}) :ip %) ips))}))++(deftest geocode-ip-addresses-test+ ;; Not ^:parallel because with-redefs mutates global state+ (with-redefs [http/get mock-geojs-http-get]+ (are [ip-addresses expected] (malli= expected+ (req.util/geocode-ip-addresses ip-addresses))+ ;; Google DNS+ ["8.8.8.8"]+ [:map+ ["8.8.8.8" [:map+ [:description [:= "United States"]]+ [:timezone [:= (t/zone-id "America/Chicago")]]]]]++ ;; this is from the MaxMind sample high-risk IP address list https://www.maxmind.com/en/high-risk-ip-sample-list+ ["185.233.100.23"]+ [:map+ ["185.233.100.23" [:map+ [:description [:= "Paris, France"]]+ [:timezone [:= (t/zone-id "Europe/Paris")]]]]]++ ["127.0.0.1"]+ [:map+ ["127.0.0.1" [:map+ [:description [:= "Unknown location"]]+ [:timezone :nil]]]]++ ["0:0:0:0:0:0:0:1"]+ [:map+ ["0:0:0:0:0:0:0:1" [:map+ [:description [:= "Unknown location"]]+ [:timezone :nil]]]]++ ;; multiple addresses at once+ ;; store.metabase.com, Google DNS+ ["52.206.149.9" "2001:4860:4860::8844"]+ [:map+ ["52.206.149.9" [:map+ [:description [:= "Virginia, United States"]]+ [:timezone [:= (t/zone-id "America/New_York")]]]]+ ["2001:4860:4860::8844" [:map+ [:description [:= "United States"]]+ [:timezone [:= (t/zone-id "America/Chicago")]]]]]++ ;; invalid inputs - these don't make HTTP calls, filtered before request+ ["wow"] :nil+ [" "] :nil+ [] :nil+ nil :nil)))++(deftest geocode-ip-addresses-metrics-test+ (testing "increments :metabase-geocoding/requests on successful geocoding"+ (mt/with-prometheus-system! [_ system]+ (with-redefs [http/get mock-geojs-http-get]+ (req.util/geocode-ip-addresses ["8.8.8.8"])+ (is (= 1.0 (mt/metric-value system :metabase-geocoding/requests))))))+ (testing "increments :metabase-geocoding/errors on failed geocoding"+ (mt/with-prometheus-system! [_ system]+ (with-redefs [http/get (fn [_ _] (throw (Exception. "Network error")))]+ (req.util/geocode-ip-addresses ["8.8.8.8"])+ (is (= 1.0 (mt/metric-value system :metabase-geocoding/errors)))))))