The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Move query cache DB access into the cache backend (#83609)
Chris Truter962a17b05c8b
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex 67fc46b2107..2e5a211ec2f 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -2376,7 +2376,6 @@:model-imports #{:model/Card:model/DashboardCardSeries:model/Database- :model/QueryCache:model/QueryExecution:model/Table}}@@ -2385,7 +2384,8 @@:ns-prefix "metabase.query-processor.middleware.cache-backend":api #{metabase.query-processor.middleware.cache-backend.dbmetabase.query-processor.middleware.cache-backend.interface}- :uses #{premium-features query-processor util}}+ :uses #{app-db premium-features util}+ :model-imports #{:model/QueryCache}}query-processor.driver-api{:team "Querying Platform"diff --git a/.clj-kondo/ratchets.edn b/.clj-kondo/ratchets.ednindex 854b7734ea9..0a7caf5df9b 100644--- a/.clj-kondo/ratchets.edn+++ b/.clj-kondo/ratchets.edn@@ -54,7 +54,7 @@:legacy-mbql.schema 14:legacy-mbql.util 2:query-processor.store 8- :toucan2.core 3+ :toucan2.core 4:util.malli.schema 2}:discouraged-var-counts {:clojure.core/eval 6:clojure.core/print 1diff --git a/src/metabase/query_processor/db.clj b/src/metabase/query_processor/db.cljindex 14ea0fffdca..f5fd07b7455 100644--- a/src/metabase/query_processor/db.clj+++ b/src/metabase/query_processor/db.clj@@ -3,7 +3,6 @@additional logic, so the rest of the module never talks to `toucan2.core` itself."(:require[java-time.api :as t]- [metabase.app-db.core :as app-db][metabase.lib.schema.id :as lib.schema.id][metabase.lib.schema.metadata :as lib.schema.metadata][metabase.queries.schema :as queries.schema]@@ -26,34 +25,6 @@dashcard-id :- [:maybe ::lib.schema.id/dashcard]](t2/exists? :model/DashboardCardSeries :card_id card-id :dashboardcard_id dashcard-id))-(mu/defn cache-entry- "`row-fn` applied to the results and updated-at of the QueryCache entry for `query-hash`, or nil."- [row-fn :- fn?- query-hash :- bytes?]- (t2/select-one-fn row-fn [:model/QueryCache :results :updated_at] :query_hash query-hash))--(mu/defn claim-cache-refresh-lease!- "Set `refresh_started_at` of the QueryCache entry for `query-hash` to `started-at` if its lease (defaulting to- `lease-free-sentinel`) is older than `lease-cutoff`, returning the number of rows updated."- [query-hash :- bytes?- lease-free-sentinel :- Temporal- lease-cutoff :- Temporal- started-at :- Temporal]- (t2/update! (t2/table-name :model/QueryCache)- {:query_hash query-hash- [:coalesce :refresh_started_at lease-free-sentinel] [:< lease-cutoff]}- {:refresh_started_at started-at}))--(mu/defn delete-cache-entry!- "Delete the QueryCache entry for `query-hash`."- [query-hash :- bytes?]- (t2/delete! (t2/table-name :model/QueryCache) :query_hash query-hash))--(mu/defn delete-cache-entries-updated-before!- "Delete the QueryCache entries last updated at or before `updated-before`."- [updated-before :- Temporal]- (t2/delete! (t2/table-name :model/QueryCache) :updated_at [:<= updated-before]))-(mu/defn insert-query-executions!"Insert the QueryExecution `rows`. Runs in its own (possibly nested) transaction so that if the app DB rejects arow, the failure is rolled back to a savepoint instead of aborting a caller's enclosing transaction."@@ -84,14 +55,3 @@"The `:id` and `:database_id` of the Cards with `card-ids`."[card-ids :- [:set ::lib.schema.id/card]](t2/select [:model/Card :id :database_id] :id [:in card-ids]))--(mu/defn upsert-cache-entry!- "Insert or update the QueryCache entry for `query-hash`, setting `:results` to `results` and `:updated_at` to- `timestamp`, and clearing `:refresh_started_at`. Returns the entry's primary key, which is the hash bytes."- [query-hash :- bytes?- timestamp :- Temporal- results :- bytes?]- (app-db/update-or-insert! :model/QueryCache {:query_hash query-hash}- (constantly {:updated_at timestamp- :results results- :refresh_started_at nil})))diff --git a/src/metabase/query_processor/middleware/cache_backend/db.clj b/src/metabase/query_processor/middleware/cache_backend/db.cljindex 8ec65cf916b..f3129e3c017 100644--- a/src/metabase/query_processor/middleware/cache_backend/db.clj+++ b/src/metabase/query_processor/middleware/cache_backend/db.clj@@ -1,17 +1,64 @@(ns metabase.query-processor.middleware.cache-backend.db(:require[java-time.api :as t]+ [metabase.app-db.core :as app-db][metabase.premium-features.core :refer [defenterprise]]- [metabase.query-processor.db :as query-processor.db][metabase.query-processor.middleware.cache-backend.interface :as i][metabase.util.date-2 :as u.date][metabase.util.encryption :as encryption]- [metabase.util.log :as log])+ [metabase.util.log :as log]+ [metabase.util.malli :as mu]+ ;; the QueryCache table belongs to this backend; the QP rule is about reading metadata+ ^{:clj-kondo/ignore [:discouraged-namespace]}+ [toucan2.core :as t2])(:import(java.io ByteArrayInputStream)))(set! *warn-on-reflection* true)+(def ^:private Temporal+ "A `java.time` instant, date, or date-time."+ [:fn #(instance? java.time.temporal.Temporal %)])++(mu/defn- cache-entry+ "`row-fn` applied to the results and updated-at of the QueryCache entry for `query-hash`, or nil."+ [row-fn :- fn?+ query-hash :- bytes?]+ (t2/select-one-fn row-fn [:model/QueryCache :results :updated_at] :query_hash query-hash))++(mu/defn- claim-cache-refresh-lease!+ "Set `refresh_started_at` of the QueryCache entry for `query-hash` to `started-at` if its lease (defaulting to+ `lease-free-sentinel`) is older than `lease-cutoff`, returning the number of rows updated."+ [query-hash :- bytes?+ lease-free-sentinel :- Temporal+ lease-cutoff :- Temporal+ started-at :- Temporal]+ (t2/update! (t2/table-name :model/QueryCache)+ {:query_hash query-hash+ [:coalesce :refresh_started_at lease-free-sentinel] [:< lease-cutoff]}+ {:refresh_started_at started-at}))++(mu/defn- delete-cache-entry!+ "Delete the QueryCache entry for `query-hash`."+ [query-hash :- bytes?]+ (t2/delete! (t2/table-name :model/QueryCache) :query_hash query-hash))++(mu/defn- delete-cache-entries-updated-before!+ "Delete the QueryCache entries last updated at or before `updated-before`."+ [updated-before :- Temporal]+ (t2/delete! (t2/table-name :model/QueryCache) :updated_at [:<= updated-before]))++(mu/defn- upsert-cache-entry!+ "Insert or update the QueryCache entry for `query-hash`, setting `:results` to `results` and `:updated_at` to+ `timestamp`, and clearing `:refresh_started_at`. Returns the entry's primary key, which is the hash bytes."+ [query-hash :- bytes?+ timestamp :- Temporal+ results :- bytes?]+ (app-db/update-or-insert! :model/QueryCache {:query_hash query-hash}+ (constantly {:updated_at timestamp+ :results results+ :refresh_started_at nil})))+(defn- ms-ago [n](u.date/add (t/offset-date-time) :millisecond (- n)))@@ -42,10 +89,10 @@its result set is open, so materializing the row first and reading the blob afterwards throws \"object is alreadyclosed\"."[query-hash]- (query-processor.db/cache-entry (fn [row]- {:results (results-as-bytes row)- :updated-at (:updated_at row)})- query-hash))+ (cache-entry (fn [row]+ {:results (results-as-bytes row)+ :updated-at (:updated_at row)})+ query-hash))(defn invalidated-at-ttl"Freshness boundary for a `:ttl` strategy: cache entries with `updated_at` older than this are stale. Returns nil when@@ -85,7 +132,7 @@blob was last written\", read that way by [[cache-fresh?]], [[purge-old-cache-entries!]], and the EE refreshscheduler; bumping it here would let a crashed refresh silently extend the row's freshness (#76856)."[query-hash lease-ms]- (pos? (query-processor.db/claim-cache-refresh-lease! query-hash lease-free-sentinel (ms-ago lease-ms) (t/offset-date-time))))+ (pos? (claim-cache-refresh-lease! query-hash lease-free-sentinel (ms-ago lease-ms) (t/offset-date-time))))(defn delete-entry!"Delete the cache entry for `query-hash`, if one exists. Deleting the row also releases any held refresh lease, so@@ -94,7 +141,7 @@shouldn't fail a query that already ran successfully."[^bytes query-hash](try- (query-processor.db/delete-cache-entry! query-hash)+ (delete-cache-entry! query-hash)(catch Throwable e(log/errorf "Error deleting outdated cache entry: %s" (ex-message e))))nil)@@ -105,7 +152,7 @@{:pre [(number? max-age-seconds)]}(log/trace "Purging old cache entries.")(try- (query-processor.db/delete-cache-entries-updated-before! (seconds-ago max-age-seconds))+ (delete-cache-entries-updated-before! (seconds-ago max-age-seconds))(catch Throwable e(log/errorf "Error purging old cache entries: %s" (ex-message e))))nil)@@ -118,7 +165,7 @@(let [final-results (encryption/maybe-encrypt-for-stream results)timestamp (t/offset-date-time)](try- (query-processor.db/upsert-cache-entry! query-hash timestamp final-results)+ (upsert-cache-entry! query-hash timestamp final-results)(catch Throwable e(log/errorf "Error saving query results to cache: %s" (ex-message e))))nil))