Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

fix: update harbormaster API use, and support public openapi schema (#65549)

Filipe Silvad4bbf59cc033
diff --git a/enterprise/backend/src/metabase_enterprise/database_replication/api.clj b/enterprise/backend/src/metabase_enterprise/database_replication/api.clj
index d7b735de749..2039a659e65 100644
--- a/enterprise/backend/src/metabase_enterprise/database_replication/api.clj
+++ b/enterprise/backend/src/metabase_enterprise/database_replication/api.clj
@@ -70,7 +70,7 @@
;; apply filter over the memoized result for snappy UI
hm-preview-memo
:tables
- (filter (comp (schema-filters->fn replication-schema-filters) :table_schema))))
+ (filter (comp (schema-filters->fn replication-schema-filters) :table-schema))))
(catch PatternSyntaxException _
{:error "Invalid schema pattern"})))
@@ -91,14 +91,14 @@
tables (if tables-error [] tables)
free-quota (get-free-quota quotas)
replicated-tables (->> tables
- (filter :has_pkey)
- (filter :has_ownership))
- tables-without-pk (filter (comp not :has_pkey) tables)
- tables-without-owner-match (filter (comp not :has_ownership) tables)
+ (filter :has-pkey)
+ (filter :has-ownership))
+ tables-without-pk (filter (comp not :has-pkey) tables)
+ tables-without-owner-match (filter (comp not :has-ownership) tables)
total-estimated-row-count (or
(some->>
replicated-tables
- (map :estimated_row_count)
+ (map :estimated-row-count)
(remove nil?)
(reduce +))
0)
diff --git a/enterprise/backend/src/metabase_enterprise/harbormaster/client.clj b/enterprise/backend/src/metabase_enterprise/harbormaster/client.clj
index 5816aa0a3c4..33c1a603bff 100644
--- a/enterprise/backend/src/metabase_enterprise/harbormaster/client.clj
+++ b/enterprise/backend/src/metabase_enterprise/harbormaster/client.clj
@@ -104,6 +104,9 @@
(mu/defn make-request :- :hm-client/http-reply
"Makes a request to the store-api-url with the given method, path, and body.
+ The Harbormaster API uses snake_keys, and this fn automatically converts kebab-keys to snake_keys on request,
+ and back to kebab-keys on response.
+
Returns a tuple of [:ok response] if the request was successful, or [:error response] if it failed."
[method :- [:enum :get :head :post :put :delete :options :copy :move :patch]
url :- :string
@@ -112,10 +115,10 @@
api-key]} (->config)
request (cond-> {:headers {"Authorization" (str "Bearer " api-key)
"Content-Type" "application/json"}}
- body (assoc :body (json/encode body)))
+ body (assoc :body (json/encode (m.util/deep-snake-keys body))))
request-method-fn (->requestor method)
unparsed-response (send-request request-method-fn store-api-url url request)
- response (decode-response unparsed-response url request)
+ response (m.util/deep-kebab-keys (decode-response unparsed-response url request))
success? (calculate-success response url request)]
[(if success? :ok :error) response]))
@@ -130,10 +133,10 @@
(martian-http/bootstrap-openapi
(str store-api-url "/openapi.json")
;; martian options for calling operations
- {:server-url store-api-url
- :interceptors (into [(bearer-auth api-key)] martian-http/default-interceptors)}
+ (merge {:server-url store-api-url}
+ (when api-key {:interceptors (into [(bearer-auth api-key)] martian-http/default-interceptors)}))
;; clj-http options for loading the openapi.json itself
- {:headers {"Authorization" (str "Bearer " api-key)}}))
+ {:headers (merge {} (when api-key {"Authorization" (str "Bearer " api-key)}))}))
(def ^:private create-client-memo
(memoize/ttl create-client
@@ -145,9 +148,6 @@
(when (str/blank? store-api-url)
(log/error "Missing store-api-url. Cannot create hm client config.")
(throw (ex-info (tru "Missing store-api-url.") {:store-api-url store-api-url})))
- (when (str/blank? api-key)
- (log/error "Missing api-key. Cannot create hm client config.")
- (throw (ex-info (tru "Missing api-key.") {:api-key api-key})))
(create-client-memo store-api-url api-key)))
(defn explore
@@ -166,15 +166,17 @@
(defn call
"Call the API, using Martian. Will throw on non 2xx, and you can get the failure body (if any) using ex-data.
+ The Harbormaster API uses snake_keys, and this fn automatically converts kebab-keys to snake_keys on request,
+ and back to kebab-keys on response.
e.g.
;; call the :foo endpoint with {:some-id id}
;; use (explore :list-connections) for params, if any, and pass them in a map
(call :list-connections)"
[operation-id & {:as args}]
(try
- (:body (martian/response-for (client) operation-id args))
+ (m.util/deep-kebab-keys (:body (martian/response-for (client) operation-id (m.util/deep-snake-keys args))))
(catch Exception e
- (let [resp-body (some-> e ex-data :body maybe-decode)
+ (let [resp-body (some-> e ex-data :body maybe-decode m.util/deep-kebab-keys)
msg (format "Error on Harbormaster operation call %s" operation-id)]
(log/error msg (or resp-body e))
(throw (ex-info msg (if (map? resp-body) resp-body {})))))))
diff --git a/enterprise/backend/test/metabase_enterprise/database_replication/api_test.clj b/enterprise/backend/test/metabase_enterprise/database_replication/api_test.clj
index cf397e33498..41e27e6e5ca 100644
--- a/enterprise/backend/test/metabase_enterprise/database_replication/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/database_replication/api_test.clj
@@ -65,26 +65,26 @@
(deftest lifecycle-test
(let [hm-state (atom [])
db-details {:dbname "dbname", :host "host", :user "user", :password "password"}
- tables [{:table_schema "public",
- :table_name "t1",
- :estimated_row_count 9
- :has_pkey true,
- :has_ownership true}
- {:table_schema "not_public",
- :table_name "no_schema",
- :estimated_row_count 11
- :has_pkey false,
- :has_ownership true}
- {:table_schema "public",
- :table_name "no_pkey",
- :estimated_row_count 11
- :has_pkey false,
- :has_ownership true}
- {:table_schema "public",
- :table_name "no_ownership",
- :estimated_row_count 11
- :has_pkey true,
- :has_ownership false}]]
+ tables [{:table-schema "public",
+ :table-name "t1",
+ :estimated-row-count 9
+ :has-pkey true,
+ :has-ownership true}
+ {:table-schema "not_public",
+ :table-name "no_schema",
+ :estimated-row-count 11
+ :has-pkey false,
+ :has-ownership true}
+ {:table-schema "public",
+ :table-name "no_pkey",
+ :estimated-row-count 11
+ :has-pkey false,
+ :has-ownership true}
+ {:table-schema "public",
+ :table-name "no_ownership",
+ :estimated-row-count 11
+ :has-pkey true,
+ :has-ownership false}]]
(with-redefs [token-check/quotas
(constantly [{:usage 499990, :locked false, :updated-at "2025-08-05T08:48:11Z", :quota-type "rows", :hosting-feature "clickhouse-dwh", :soft-limit 500000}])
@@ -142,9 +142,9 @@
(deftest preview-replication-test
(let [quotas [{:usage 100000, :locked false, :updated-at "2025-08-05T08:48:11Z", :quota-type "rows", :hosting-feature "clickhouse-dwh", :soft-limit 500000}]
- valid-table {:table_schema "public", :table_name "valid_table", :estimated_row_count 1000, :has_pkey true, :has_ownership true}
- no-pk-table {:table_schema "public", :table_name "no_pk_table", :estimated_row_count 2000, :has_pkey false, :has_ownership true}
- no-ownership-table {:table_schema "public", :table_name "no_ownership_table", :estimated_row_count 3000, :has_pkey true, :has_ownership false}
+ valid-table {:table-schema "public", :table-name "valid_table", :estimated-row-count 1000, :has-pkey true, :has-ownership true}
+ no-pk-table {:table-schema "public", :table-name "no_pk_table", :estimated-row-count 2000, :has-pkey false, :has-ownership true}
+ no-ownership-table {:table-schema "public", :table-name "no_ownership_table", :estimated-row-count 3000, :has-pkey true, :has-ownership false}
base-response {:free-quota 400000
:total-estimated-row-count 0
:errors {:no-tables false, :no-quota false, :invalid-schema-filters-pattern false}
@@ -168,7 +168,7 @@
(api/preview-replication quotas [no-pk-table no-ownership-table]))))
(testing "no-quota error condition"
- (let [high-row-table {:table_schema "public", :table_name "high_row_table", :estimated_row_count 500000, :has_pkey true, :has_ownership true}]
+ (let [high-row-table {:table-schema "public", :table-name "high_row_table", :estimated-row-count 500000, :has-pkey true, :has-ownership true}]
(is (= (merge base-response
{:total-estimated-row-count 500000
:errors {:no-tables false, :no-quota true, :invalid-schema-filters-pattern false}
diff --git a/src/metabase/cloud_migration/models/cloud_migration.clj b/src/metabase/cloud_migration/models/cloud_migration.clj
index 679bc26a0d0..0b7cfebe825 100644
--- a/src/metabase/cloud_migration/models/cloud_migration.clj
+++ b/src/metabase/cloud_migration/models/cloud_migration.clj
@@ -181,7 +181,12 @@
{:form-params {:part_count (count parts)}
:content-type :json})
:body
- json/decode+kw)
+ json/decode+kw
+ ;; This endpoint, and only this one in this ns, needs a backwards and forward compatible
+ ;; key conversion. This can be removed when Harbormaster does only underscores in the API,
+ ;; and the keys above (multipart-upload-id multipart-urls) renamed to use underscores.
+ ;; But it can also stay here indefinitly and will be correct.
+ u/deep-kebab-keys)
etags
(->> parts
diff --git a/src/metabase/util.cljc b/src/metabase/util.cljc
index 4803f448c91..8c7d681cd45 100644
--- a/src/metabase/util.cljc
+++ b/src/metabase/util.cljc
@@ -316,6 +316,11 @@
[m]
(recursive-map-keys ->snake_case_en m))
+(defn deep-kebab-keys
+ "Recursively convert the keys in a map to `kebab_case`."
+ [m]
+ (recursive-map-keys ->kebab-case-en m))
+
(defn deep-kebab->snake-keys
"Recursively convert kebab-case keys in a map to snake_case by replacing hyphens.
Preserves camelCase and other formatting."