Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Fix caching of postgres domains in arrays (#66059)

Riley Thompson719ebd0a7bd8
diff --git a/src/metabase/driver/postgres.clj b/src/metabase/driver/postgres.clj
index bfffe074f01..3a89def0666 100644
--- a/src/metabase/driver/postgres.clj
+++ b/src/metabase/driver/postgres.clj
@@ -34,9 +34,10 @@
[metabase.util.i18n :refer [trs tru]]
[metabase.util.log :as log]
[metabase.util.malli :as mu]
- [metabase.util.performance :as perf :refer [some select-keys mapv not-empty]])
+ [metabase.util.performance :as perf :refer [some select-keys mapv not-empty]]
+ [taoensso.nippy :as nippy])
(:import
- (java.io StringReader)
+ (java.io DataInput DataOutput StringReader)
(java.sql
Connection
ResultSet
@@ -45,7 +46,8 @@
(java.time LocalDateTime OffsetDateTime OffsetTime)
(org.apache.commons.codec.binary Hex)
(org.postgresql.copy CopyManager)
- (org.postgresql.jdbc PgConnection)))
+ (org.postgresql.jdbc PgConnection)
+ (org.postgresql.util PGobject)))
(set! *warn-on-reflection* true)
@@ -1010,8 +1012,8 @@
[_ ^ResultSet rs _ ^Integer i]
(fn []
(let [obj (.getObject rs i)]
- (cond (instance? org.postgresql.util.PGobject obj)
- (.getValue ^org.postgresql.util.PGobject obj)
+ (cond (instance? PGobject obj)
+ (.getValue ^PGobject obj)
:else
obj))))
@@ -1275,3 +1277,16 @@
{:name "Scaleway" :pattern "\\.scw\\.cloud$"}
{:name "Supabase" :pattern "(pooler\\.supabase\\.com|\\.supabase\\.co)$"}
{:name "Timescale" :pattern "(\\.tsdb\\.cloud|\\.timescale\\.com)$"}]})
+
+;; Custom nippy handling for PGobject to enable proper caching of postgres domains in arrays (#55301)
+
+(nippy/extend-freeze PGobject :postgres/PGobject
+ [^PGobject obj ^DataOutput data-output]
+ (nippy/freeze-to-out! data-output (.getType obj))
+ (nippy/freeze-to-out! data-output (.getValue obj)))
+
+(nippy/extend-thaw :postgres/PGobject
+ [^DataInput data-input]
+ (let [type (nippy/thaw-from-in! data-input)
+ value (nippy/thaw-from-in! data-input)]
+ (doto (PGobject.) (.setType type) (.setValue value))))
diff --git a/test/metabase/driver/postgres_test.clj b/test/metabase/driver/postgres_test.clj
index aac847bcd63..6e50d2b8c53 100644
--- a/test/metabase/driver/postgres_test.clj
+++ b/test/metabase/driver/postgres_test.clj
@@ -47,9 +47,11 @@
[metabase.util.log :as log]
[metabase.warehouses.provider-detection :as provider-detection]
[next.jdbc :as next.jdbc]
+ [taoensso.nippy :as nippy]
[toucan2.core :as t2])
(:import
- (java.sql Connection)))
+ (java.sql Connection)
+ (org.postgresql.util PGobject)))
(set! *warn-on-reflection* true)
@@ -1960,3 +1962,20 @@
(lib/query mp)
(qp/process-query)
(mt/rows)))))))))
+
+(deftest pgobject-freeze-thaw-test
+ (letfn [(make-pgobject [type value]
+ (doto (PGobject.) (.setType type) (.setValue value)))
+ (test-pgobject-caching [obj]
+ (is (= obj (-> obj nippy/freeze nippy/thaw))))]
+ (testing "Simple PGobject instances can be frozen and thawed"
+ (test-pgobject-caching (make-pgobject "foo_type" "abc_val")))
+ (testing "PGobjects in an array can be frozen and thawed"
+ (let [pg-obj-1 (make-pgobject "foo_type" "abc_val")
+ pg-obj-2 (make-pgobject "foo_type" "xyz_val")
+ pg-obj-arr [pg-obj-1 pg-obj-2]]
+ (test-pgobject-caching pg-obj-arr)))
+ (testing "PGobjects in a map can be frozen and thawed"
+ (let [pg-obj (make-pgobject "foo_type" "abc_val")
+ pg-obj-map {:data [pg-obj] :metadata {:type "test"}}]
+ (test-pgobject-caching pg-obj-map)))))
diff --git a/test/metabase/query_processor/middleware/cache_test.clj b/test/metabase/query_processor/middleware/cache_test.clj
index 3edb8143b79..f4b565b40ca 100644
--- a/test/metabase/query_processor/middleware/cache_test.clj
+++ b/test/metabase/query_processor/middleware/cache_test.clj
@@ -4,12 +4,14 @@
[buddy.core.codecs :as codecs]
[clojure.core.async :as a]
[clojure.data.csv :as csv]
+ [clojure.java.jdbc :as jdbc]
[clojure.test :refer :all]
[java-time.api :as t]
[medley.core :as m]
[metabase.cache.core]
[metabase.driver :as driver]
[metabase.driver.settings :as driver.settings]
+ [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
[metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]
[metabase.lib.metadata :as lib.metadata]
[metabase.lib.test-util :as lib.tu]
@@ -333,18 +335,59 @@
:hash some?}
:row_count 1
:status :completed}
- (dissoc original-result :data)))
+ original-result))
(is (=? {:cache/details {:cached true
:updated_at #t "2025-02-06T00:00:00.000Z[UTC]"
:hash some?}
:row_count 1
:status :completed}
- (dissoc cached-result :data)))
+ cached-result))
(is (= (seq (-> original-result :cache/details :hash))
(seq (-> cached-result :cache/details :hash))))
(is (= (dissoc original-result :cache/details)
(dissoc cached-result :cache/details)))))))))
+(deftest postgres-domain-can-be-cached-test
+ #_{:clj-kondo/ignore [:metabase/disallow-hardcoded-driver-names-in-tests]}
+ (mt/test-driver :postgres
+ (mt/dataset (mt/dataset-definition
+ "domain_dataset"
+ [["placeholder"
+ [{:field-name "foo", :base-type :type/Integer}]
+ [[1]]]])
+ (let [spec (sql-jdbc.conn/connection-details->spec :postgres (:details (mt/db)))
+ dom-name (str "dom_" (mt/random-name))]
+ (jdbc/execute! spec [(format "CREATE DOMAIN %s AS text CHECK (VALUE <> '')" dom-name)])
+ (with-mock-cache! [save-chan]
+ (mt/with-temporary-setting-values [enable-query-caching true]
+ (mt/with-clock #t "2025-02-06T00:00:00.000Z[UTC]"
+ (let [query (mt/native-query {:query (format "SELECT 'foo'::%s;" dom-name)})
+ query (assoc query :cache-strategy (ttl-strategy))
+ original-result (qp/process-query query)
+ ;; clear any existing values in the `save-chan`
+ _ (while (a/poll! save-chan))
+ _ (mt/wait-for-result save-chan)
+ cached-result (qp/process-query query)]
+ (is (= [["foo"]]
+ (mt/rows original-result)))
+ (is (= [["foo"]]
+ (mt/rows cached-result)))
+ (is (=? {:cache/details {:stored true
+ :hash some?}
+ :row_count 1
+ :status :completed}
+ original-result))
+ (is (=? {:cache/details {:cached true
+ :updated_at #t "2025-02-06T00:00:00.000Z[UTC]"
+ :hash some?}
+ :row_count 1
+ :status :completed}
+ cached-result))
+ (is (= (seq (-> original-result :cache/details :hash))
+ (seq (-> cached-result :cache/details :hash))))
+ (is (= (dissoc original-result :cache/details)
+ (dissoc cached-result :cache/details)))))))))))
+
(deftest e2e-test
(testing "Test that the caching middleware actually working in the context of the entire QP"
(doseq [query [(mt/mbql-query venues {:order-by [[:asc $id]], :limit 5})