Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Move query cache DB access into the cache backend (#83609)

Chris Truter962a17b05c8b
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index 67fc46b2107..2e5a211ec2f 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -2376,7 +2376,6 @@
:model-imports #{:model/Card
:model/DashboardCardSeries
:model/Database
- :model/QueryCache
:model/QueryExecution
:model/Table}}
@@ -2385,7 +2384,8 @@
:ns-prefix "metabase.query-processor.middleware.cache-backend"
:api #{metabase.query-processor.middleware.cache-backend.db
metabase.query-processor.middleware.cache-backend.interface}
- :uses #{premium-features query-processor util}}
+ :uses #{app-db premium-features util}
+ :model-imports #{:model/QueryCache}}
query-processor.driver-api
{:team "Querying Platform"
diff --git a/.clj-kondo/ratchets.edn b/.clj-kondo/ratchets.edn
index 854b7734ea9..0a7caf5df9b 100644
--- a/.clj-kondo/ratchets.edn
+++ b/.clj-kondo/ratchets.edn
@@ -54,7 +54,7 @@
:legacy-mbql.schema 14
:legacy-mbql.util 2
:query-processor.store 8
- :toucan2.core 3
+ :toucan2.core 4
:util.malli.schema 2}
:discouraged-var-counts {:clojure.core/eval 6
:clojure.core/print 1
diff --git a/src/metabase/query_processor/db.clj b/src/metabase/query_processor/db.clj
index 14ea0fffdca..f5fd07b7455 100644
--- a/src/metabase/query_processor/db.clj
+++ b/src/metabase/query_processor/db.clj
@@ -3,7 +3,6 @@
additional logic, so the rest of the module never talks to `toucan2.core` itself."
(:require
[java-time.api :as t]
- [metabase.app-db.core :as app-db]
[metabase.lib.schema.id :as lib.schema.id]
[metabase.lib.schema.metadata :as lib.schema.metadata]
[metabase.queries.schema :as queries.schema]
@@ -26,34 +25,6 @@
dashcard-id :- [:maybe ::lib.schema.id/dashcard]]
(t2/exists? :model/DashboardCardSeries :card_id card-id :dashboardcard_id dashcard-id))
-(mu/defn cache-entry
- "`row-fn` applied to the results and updated-at of the QueryCache entry for `query-hash`, or nil."
- [row-fn :- fn?
- query-hash :- bytes?]
- (t2/select-one-fn row-fn [:model/QueryCache :results :updated_at] :query_hash query-hash))
-
-(mu/defn claim-cache-refresh-lease!
- "Set `refresh_started_at` of the QueryCache entry for `query-hash` to `started-at` if its lease (defaulting to
- `lease-free-sentinel`) is older than `lease-cutoff`, returning the number of rows updated."
- [query-hash :- bytes?
- lease-free-sentinel :- Temporal
- lease-cutoff :- Temporal
- started-at :- Temporal]
- (t2/update! (t2/table-name :model/QueryCache)
- {:query_hash query-hash
- [:coalesce :refresh_started_at lease-free-sentinel] [:< lease-cutoff]}
- {:refresh_started_at started-at}))
-
-(mu/defn delete-cache-entry!
- "Delete the QueryCache entry for `query-hash`."
- [query-hash :- bytes?]
- (t2/delete! (t2/table-name :model/QueryCache) :query_hash query-hash))
-
-(mu/defn delete-cache-entries-updated-before!
- "Delete the QueryCache entries last updated at or before `updated-before`."
- [updated-before :- Temporal]
- (t2/delete! (t2/table-name :model/QueryCache) :updated_at [:<= updated-before]))
-
(mu/defn insert-query-executions!
"Insert the QueryExecution `rows`. Runs in its own (possibly nested) transaction so that if the app DB rejects a
row, the failure is rolled back to a savepoint instead of aborting a caller's enclosing transaction."
@@ -84,14 +55,3 @@
"The `:id` and `:database_id` of the Cards with `card-ids`."
[card-ids :- [:set ::lib.schema.id/card]]
(t2/select [:model/Card :id :database_id] :id [:in card-ids]))
-
-(mu/defn upsert-cache-entry!
- "Insert or update the QueryCache entry for `query-hash`, setting `:results` to `results` and `:updated_at` to
- `timestamp`, and clearing `:refresh_started_at`. Returns the entry's primary key, which is the hash bytes."
- [query-hash :- bytes?
- timestamp :- Temporal
- results :- bytes?]
- (app-db/update-or-insert! :model/QueryCache {:query_hash query-hash}
- (constantly {:updated_at timestamp
- :results results
- :refresh_started_at nil})))
diff --git a/src/metabase/query_processor/middleware/cache_backend/db.clj b/src/metabase/query_processor/middleware/cache_backend/db.clj
index 8ec65cf916b..f3129e3c017 100644
--- a/src/metabase/query_processor/middleware/cache_backend/db.clj
+++ b/src/metabase/query_processor/middleware/cache_backend/db.clj
@@ -1,17 +1,64 @@
(ns metabase.query-processor.middleware.cache-backend.db
(:require
[java-time.api :as t]
+ [metabase.app-db.core :as app-db]
[metabase.premium-features.core :refer [defenterprise]]
- [metabase.query-processor.db :as query-processor.db]
[metabase.query-processor.middleware.cache-backend.interface :as i]
[metabase.util.date-2 :as u.date]
[metabase.util.encryption :as encryption]
- [metabase.util.log :as log])
+ [metabase.util.log :as log]
+ [metabase.util.malli :as mu]
+ ;; the QueryCache table belongs to this backend; the QP rule is about reading metadata
+ ^{:clj-kondo/ignore [:discouraged-namespace]}
+ [toucan2.core :as t2])
(:import
(java.io ByteArrayInputStream)))
(set! *warn-on-reflection* true)
+(def ^:private Temporal
+ "A `java.time` instant, date, or date-time."
+ [:fn #(instance? java.time.temporal.Temporal %)])
+
+(mu/defn- cache-entry
+ "`row-fn` applied to the results and updated-at of the QueryCache entry for `query-hash`, or nil."
+ [row-fn :- fn?
+ query-hash :- bytes?]
+ (t2/select-one-fn row-fn [:model/QueryCache :results :updated_at] :query_hash query-hash))
+
+(mu/defn- claim-cache-refresh-lease!
+ "Set `refresh_started_at` of the QueryCache entry for `query-hash` to `started-at` if its lease (defaulting to
+ `lease-free-sentinel`) is older than `lease-cutoff`, returning the number of rows updated."
+ [query-hash :- bytes?
+ lease-free-sentinel :- Temporal
+ lease-cutoff :- Temporal
+ started-at :- Temporal]
+ (t2/update! (t2/table-name :model/QueryCache)
+ {:query_hash query-hash
+ [:coalesce :refresh_started_at lease-free-sentinel] [:< lease-cutoff]}
+ {:refresh_started_at started-at}))
+
+(mu/defn- delete-cache-entry!
+ "Delete the QueryCache entry for `query-hash`."
+ [query-hash :- bytes?]
+ (t2/delete! (t2/table-name :model/QueryCache) :query_hash query-hash))
+
+(mu/defn- delete-cache-entries-updated-before!
+ "Delete the QueryCache entries last updated at or before `updated-before`."
+ [updated-before :- Temporal]
+ (t2/delete! (t2/table-name :model/QueryCache) :updated_at [:<= updated-before]))
+
+(mu/defn- upsert-cache-entry!
+ "Insert or update the QueryCache entry for `query-hash`, setting `:results` to `results` and `:updated_at` to
+ `timestamp`, and clearing `:refresh_started_at`. Returns the entry's primary key, which is the hash bytes."
+ [query-hash :- bytes?
+ timestamp :- Temporal
+ results :- bytes?]
+ (app-db/update-or-insert! :model/QueryCache {:query_hash query-hash}
+ (constantly {:updated_at timestamp
+ :results results
+ :refresh_started_at nil})))
+
(defn- ms-ago [n]
(u.date/add (t/offset-date-time) :millisecond (- n)))
@@ -42,10 +89,10 @@
its result set is open, so materializing the row first and reading the blob afterwards throws \"object is already
closed\"."
[query-hash]
- (query-processor.db/cache-entry (fn [row]
- {:results (results-as-bytes row)
- :updated-at (:updated_at row)})
- query-hash))
+ (cache-entry (fn [row]
+ {:results (results-as-bytes row)
+ :updated-at (:updated_at row)})
+ query-hash))
(defn invalidated-at-ttl
"Freshness boundary for a `:ttl` strategy: cache entries with `updated_at` older than this are stale. Returns nil when
@@ -85,7 +132,7 @@
blob was last written\", read that way by [[cache-fresh?]], [[purge-old-cache-entries!]], and the EE refresh
scheduler; bumping it here would let a crashed refresh silently extend the row's freshness (#76856)."
[query-hash lease-ms]
- (pos? (query-processor.db/claim-cache-refresh-lease! query-hash lease-free-sentinel (ms-ago lease-ms) (t/offset-date-time))))
+ (pos? (claim-cache-refresh-lease! query-hash lease-free-sentinel (ms-ago lease-ms) (t/offset-date-time))))
(defn delete-entry!
"Delete the cache entry for `query-hash`, if one exists. Deleting the row also releases any held refresh lease, so
@@ -94,7 +141,7 @@
shouldn't fail a query that already ran successfully."
[^bytes query-hash]
(try
- (query-processor.db/delete-cache-entry! query-hash)
+ (delete-cache-entry! query-hash)
(catch Throwable e
(log/errorf "Error deleting outdated cache entry: %s" (ex-message e))))
nil)
@@ -105,7 +152,7 @@
{:pre [(number? max-age-seconds)]}
(log/trace "Purging old cache entries.")
(try
- (query-processor.db/delete-cache-entries-updated-before! (seconds-ago max-age-seconds))
+ (delete-cache-entries-updated-before! (seconds-ago max-age-seconds))
(catch Throwable e
(log/errorf "Error purging old cache entries: %s" (ex-message e))))
nil)
@@ -118,7 +165,7 @@
(let [final-results (encryption/maybe-encrypt-for-stream results)
timestamp (t/offset-date-time)]
(try
- (query-processor.db/upsert-cache-entry! query-hash timestamp final-results)
+ (upsert-cache-entry! query-hash timestamp final-results)
(catch Throwable e
(log/errorf "Error saving query results to cache: %s" (ex-message e))))
nil))