The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
fix: update harbormaster API use, and support public openapi schema (#65549)
Filipe Silvad4bbf59cc033
diff --git a/enterprise/backend/src/metabase_enterprise/database_replication/api.clj b/enterprise/backend/src/metabase_enterprise/database_replication/api.cljindex d7b735de749..2039a659e65 100644--- a/enterprise/backend/src/metabase_enterprise/database_replication/api.clj+++ b/enterprise/backend/src/metabase_enterprise/database_replication/api.clj@@ -70,7 +70,7 @@;; apply filter over the memoized result for snappy UIhm-preview-memo:tables- (filter (comp (schema-filters->fn replication-schema-filters) :table_schema))))+ (filter (comp (schema-filters->fn replication-schema-filters) :table-schema))))(catch PatternSyntaxException _{:error "Invalid schema pattern"})))@@ -91,14 +91,14 @@tables (if tables-error [] tables)free-quota (get-free-quota quotas)replicated-tables (->> tables- (filter :has_pkey)- (filter :has_ownership))- tables-without-pk (filter (comp not :has_pkey) tables)- tables-without-owner-match (filter (comp not :has_ownership) tables)+ (filter :has-pkey)+ (filter :has-ownership))+ tables-without-pk (filter (comp not :has-pkey) tables)+ tables-without-owner-match (filter (comp not :has-ownership) tables)total-estimated-row-count (or(some->>replicated-tables- (map :estimated_row_count)+ (map :estimated-row-count)(remove nil?)(reduce +))0)diff --git a/enterprise/backend/src/metabase_enterprise/harbormaster/client.clj b/enterprise/backend/src/metabase_enterprise/harbormaster/client.cljindex 5816aa0a3c4..33c1a603bff 100644--- a/enterprise/backend/src/metabase_enterprise/harbormaster/client.clj+++ b/enterprise/backend/src/metabase_enterprise/harbormaster/client.clj@@ -104,6 +104,9 @@(mu/defn make-request :- :hm-client/http-reply"Makes a request to the store-api-url with the given method, path, and body.+ The Harbormaster API uses snake_keys, and this fn automatically converts kebab-keys to snake_keys on request,+ and back to kebab-keys on response.+Returns a tuple of [:ok response] if the request was successful, or [:error response] if it failed."[method :- [:enum :get :head :post :put :delete :options :copy :move :patch]url :- :string@@ -112,10 +115,10 @@api-key]} (->config)request (cond-> {:headers {"Authorization" (str "Bearer " api-key)"Content-Type" "application/json"}}- body (assoc :body (json/encode body)))+ body (assoc :body (json/encode (m.util/deep-snake-keys body))))request-method-fn (->requestor method)unparsed-response (send-request request-method-fn store-api-url url request)- response (decode-response unparsed-response url request)+ response (m.util/deep-kebab-keys (decode-response unparsed-response url request))success? (calculate-success response url request)][(if success? :ok :error) response]))@@ -130,10 +133,10 @@(martian-http/bootstrap-openapi(str store-api-url "/openapi.json");; martian options for calling operations- {:server-url store-api-url- :interceptors (into [(bearer-auth api-key)] martian-http/default-interceptors)}+ (merge {:server-url store-api-url}+ (when api-key {:interceptors (into [(bearer-auth api-key)] martian-http/default-interceptors)}));; clj-http options for loading the openapi.json itself- {:headers {"Authorization" (str "Bearer " api-key)}}))+ {:headers (merge {} (when api-key {"Authorization" (str "Bearer " api-key)}))}))(def ^:private create-client-memo(memoize/ttl create-client@@ -145,9 +148,6 @@(when (str/blank? store-api-url)(log/error "Missing store-api-url. Cannot create hm client config.")(throw (ex-info (tru "Missing store-api-url.") {:store-api-url store-api-url})))- (when (str/blank? api-key)- (log/error "Missing api-key. Cannot create hm client config.")- (throw (ex-info (tru "Missing api-key.") {:api-key api-key})))(create-client-memo store-api-url api-key)))(defn explore@@ -166,15 +166,17 @@(defn call"Call the API, using Martian. Will throw on non 2xx, and you can get the failure body (if any) using ex-data.+ The Harbormaster API uses snake_keys, and this fn automatically converts kebab-keys to snake_keys on request,+ and back to kebab-keys on response.e.g.;; call the :foo endpoint with {:some-id id};; use (explore :list-connections) for params, if any, and pass them in a map(call :list-connections)"[operation-id & {:as args}](try- (:body (martian/response-for (client) operation-id args))+ (m.util/deep-kebab-keys (:body (martian/response-for (client) operation-id (m.util/deep-snake-keys args))))(catch Exception e- (let [resp-body (some-> e ex-data :body maybe-decode)+ (let [resp-body (some-> e ex-data :body maybe-decode m.util/deep-kebab-keys)msg (format "Error on Harbormaster operation call %s" operation-id)](log/error msg (or resp-body e))(throw (ex-info msg (if (map? resp-body) resp-body {})))))))diff --git a/enterprise/backend/test/metabase_enterprise/database_replication/api_test.clj b/enterprise/backend/test/metabase_enterprise/database_replication/api_test.cljindex cf397e33498..41e27e6e5ca 100644--- a/enterprise/backend/test/metabase_enterprise/database_replication/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/database_replication/api_test.clj@@ -65,26 +65,26 @@(deftest lifecycle-test(let [hm-state (atom [])db-details {:dbname "dbname", :host "host", :user "user", :password "password"}- tables [{:table_schema "public",- :table_name "t1",- :estimated_row_count 9- :has_pkey true,- :has_ownership true}- {:table_schema "not_public",- :table_name "no_schema",- :estimated_row_count 11- :has_pkey false,- :has_ownership true}- {:table_schema "public",- :table_name "no_pkey",- :estimated_row_count 11- :has_pkey false,- :has_ownership true}- {:table_schema "public",- :table_name "no_ownership",- :estimated_row_count 11- :has_pkey true,- :has_ownership false}]]+ tables [{:table-schema "public",+ :table-name "t1",+ :estimated-row-count 9+ :has-pkey true,+ :has-ownership true}+ {:table-schema "not_public",+ :table-name "no_schema",+ :estimated-row-count 11+ :has-pkey false,+ :has-ownership true}+ {:table-schema "public",+ :table-name "no_pkey",+ :estimated-row-count 11+ :has-pkey false,+ :has-ownership true}+ {:table-schema "public",+ :table-name "no_ownership",+ :estimated-row-count 11+ :has-pkey true,+ :has-ownership false}]](with-redefs [token-check/quotas(constantly [{:usage 499990, :locked false, :updated-at "2025-08-05T08:48:11Z", :quota-type "rows", :hosting-feature "clickhouse-dwh", :soft-limit 500000}])@@ -142,9 +142,9 @@(deftest preview-replication-test(let [quotas [{:usage 100000, :locked false, :updated-at "2025-08-05T08:48:11Z", :quota-type "rows", :hosting-feature "clickhouse-dwh", :soft-limit 500000}]- valid-table {:table_schema "public", :table_name "valid_table", :estimated_row_count 1000, :has_pkey true, :has_ownership true}- no-pk-table {:table_schema "public", :table_name "no_pk_table", :estimated_row_count 2000, :has_pkey false, :has_ownership true}- no-ownership-table {:table_schema "public", :table_name "no_ownership_table", :estimated_row_count 3000, :has_pkey true, :has_ownership false}+ valid-table {:table-schema "public", :table-name "valid_table", :estimated-row-count 1000, :has-pkey true, :has-ownership true}+ no-pk-table {:table-schema "public", :table-name "no_pk_table", :estimated-row-count 2000, :has-pkey false, :has-ownership true}+ no-ownership-table {:table-schema "public", :table-name "no_ownership_table", :estimated-row-count 3000, :has-pkey true, :has-ownership false}base-response {:free-quota 400000:total-estimated-row-count 0:errors {:no-tables false, :no-quota false, :invalid-schema-filters-pattern false}@@ -168,7 +168,7 @@(api/preview-replication quotas [no-pk-table no-ownership-table]))))(testing "no-quota error condition"- (let [high-row-table {:table_schema "public", :table_name "high_row_table", :estimated_row_count 500000, :has_pkey true, :has_ownership true}]+ (let [high-row-table {:table-schema "public", :table-name "high_row_table", :estimated-row-count 500000, :has-pkey true, :has-ownership true}](is (= (merge base-response{:total-estimated-row-count 500000:errors {:no-tables false, :no-quota true, :invalid-schema-filters-pattern false}diff --git a/src/metabase/cloud_migration/models/cloud_migration.clj b/src/metabase/cloud_migration/models/cloud_migration.cljindex 679bc26a0d0..0b7cfebe825 100644--- a/src/metabase/cloud_migration/models/cloud_migration.clj+++ b/src/metabase/cloud_migration/models/cloud_migration.clj@@ -181,7 +181,12 @@{:form-params {:part_count (count parts)}:content-type :json}):body- json/decode+kw)+ json/decode+kw+ ;; This endpoint, and only this one in this ns, needs a backwards and forward compatible+ ;; key conversion. This can be removed when Harbormaster does only underscores in the API,+ ;; and the keys above (multipart-upload-id multipart-urls) renamed to use underscores.+ ;; But it can also stay here indefinitly and will be correct.+ u/deep-kebab-keys)etags(->> partsdiff --git a/src/metabase/util.cljc b/src/metabase/util.cljcindex 4803f448c91..8c7d681cd45 100644--- a/src/metabase/util.cljc+++ b/src/metabase/util.cljc@@ -316,6 +316,11 @@[m](recursive-map-keys ->snake_case_en m))+(defn deep-kebab-keys+ "Recursively convert the keys in a map to `kebab_case`."+ [m]+ (recursive-map-keys ->kebab-case-en m))+(defn deep-kebab->snake-keys"Recursively convert kebab-case keys in a map to snake_case by replacing hyphens.Preserves camelCase and other formatting."