Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Add guards to metabot and semantic search job implementations (#64031)

lbrdnkc28f66c96165
diff --git a/enterprise/backend/src/metabase_enterprise/metabot_v3/task/suggested_prompts_generator.clj b/enterprise/backend/src/metabase_enterprise/metabot_v3/task/suggested_prompts_generator.clj
index bd91d9860bc..5d87dccb5ae 100644
--- a/enterprise/backend/src/metabase_enterprise/metabot_v3/task/suggested_prompts_generator.clj
+++ b/enterprise/backend/src/metabase_enterprise/metabot_v3/task/suggested_prompts_generator.clj
@@ -20,16 +20,17 @@
(def ^:private generator-trigger-key (triggers/key "metabase.task.metabot-v3.suggested-prompts-generator.trigger"))
(defn- maybe-generate-suggested-prompts! []
- (let [metabot-eid (get-in metabot-v3.config/metabot-config
- [metabot-v3.config/internal-metabot-id :entity-id])
- metabot-id (t2/select-one-pk :model/Metabot :entity_id metabot-eid)
- suggested-prompts-cnt (t2/count :model/MetabotPrompt :metabot_id metabot-id)]
- (if (zero? suggested-prompts-cnt)
- (do
- (log/info "No suggested prompts found. Generating suggested prompts.")
- (metabot-v3.suggested-prompts/generate-sample-prompts metabot-id)
- (log/info "Suggested prompts generated successfully."))
- (log/info "Suggested prompts are present. Not generating."))))
+ (when (premium-features/has-feature? :metabot-v3)
+ (let [metabot-eid (get-in metabot-v3.config/metabot-config
+ [metabot-v3.config/internal-metabot-id :entity-id])
+ metabot-id (t2/select-one-pk :model/Metabot :entity_id metabot-eid)
+ suggested-prompts-cnt (t2/count :model/MetabotPrompt :metabot_id metabot-id)]
+ (if (zero? suggested-prompts-cnt)
+ (do
+ (log/info "No suggested prompts found. Generating suggested prompts.")
+ (metabot-v3.suggested-prompts/generate-sample-prompts metabot-id)
+ (log/info "Suggested prompts generated successfully."))
+ (log/info "Suggested prompts are present. Not generating.")))))
(task/defjob ^{DisallowConcurrentExecution true
:doc "Initial _suggested prompts_ generation for internal Metabot."}
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/task/index_cleanup.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/task/index_cleanup.clj
index 191fed820a9..ef8a8446052 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/task/index_cleanup.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/task/index_cleanup.clj
@@ -8,10 +8,9 @@
[honey.sql :as sql]
[honey.sql.helpers :as sql.helpers]
[java-time.api :as t]
- [metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource]
[metabase-enterprise.semantic-search.env :as semantic.env]
[metabase-enterprise.semantic-search.settings :as semantic.settings]
- [metabase.premium-features.core :as premium-features]
+ [metabase-enterprise.semantic-search.util :as semantic.u]
[metabase.task.core :as task]
[metabase.util.log :as log]
[next.jdbc :as jdbc]
@@ -165,11 +164,12 @@
(defn- cleanup-stale-indexes-and-gate-tombstones!
[]
- (let [pgvector (semantic.env/get-pgvector-datasource!)
- index-metadata (semantic.env/get-index-metadata)]
- (cleanup-stale-indexes! pgvector index-metadata)
- (cleanup-old-gate-tombstones! pgvector index-metadata)
- (cleanup-orphan-repair-tables! pgvector)))
+ (when (semantic.u/semantic-search-available?)
+ (let [pgvector (semantic.env/get-pgvector-datasource!)
+ index-metadata (semantic.env/get-index-metadata)]
+ (cleanup-stale-indexes! pgvector index-metadata)
+ (cleanup-old-gate-tombstones! pgvector index-metadata)
+ (cleanup-orphan-repair-tables! pgvector))))
(def ^:private cleanup-job-key (jobs/key "metabase.task.semantic-index-cleanup.job"))
(def ^:private cleanup-trigger-key (triggers/key "metabase.task.semantic-index-cleanup.trigger"))
@@ -180,8 +180,7 @@
(cleanup-stale-indexes-and-gate-tombstones!))
(defmethod task/init! ::SemanticIndexCleanup [_]
- (when (and (string? (not-empty semantic.db.datasource/db-url))
- (premium-features/has-feature? :semantic-search))
+ (when (semantic.u/semantic-search-available?)
(let [job (jobs/build
(jobs/of-type SemanticIndexCleanup)
(jobs/with-identity cleanup-job-key))
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/task/index_repair.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/task/index_repair.clj
index 05488c24fe2..6adea36cf33 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/task/index_repair.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/task/index_repair.clj
@@ -5,8 +5,7 @@
[clojurewerkz.quartzite.schedule.cron :as cron]
[clojurewerkz.quartzite.triggers :as triggers]
[metabase-enterprise.semantic-search.core :as semantic-search.core]
- [metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource]
- [metabase.premium-features.core :as premium-features]
+ [metabase-enterprise.semantic-search.util :as semantic.u]
[metabase.search.ingestion :as search.ingestion]
[metabase.task.core :as task]
[metabase.util.log :as log])
@@ -21,17 +20,17 @@
(task/defjob ^{DisallowConcurrentExecution true
:doc "Runs repair-index! to maintain semantic search consistency"}
SemanticIndexRepair [_ctx]
- (log/with-context {:quartz-job-type 'SemanticIndexRepair}
- (try
- (log/info "Starting semantic search index repair")
- (semantic-search.core/repair-index! (search.ingestion/searchable-documents))
- (log/info "Completed semantic search index repair")
- (catch Exception e
- (log/error e "Failed to complete semantic search index repair")))))
+ (when (semantic.u/semantic-search-available?)
+ (log/with-context {:quartz-job-type 'SemanticIndexRepair}
+ (try
+ (log/info "Starting semantic search index repair")
+ (semantic-search.core/repair-index! (search.ingestion/searchable-documents))
+ (log/info "Completed semantic search index repair")
+ (catch Exception e
+ (log/error e "Failed to complete semantic search index repair"))))))
(defmethod task/init! ::SemanticIndexRepair [_]
- (when (and (string? (not-empty semantic.db.datasource/db-url))
- (premium-features/has-feature? :semantic-search))
+ (when (semantic.u/semantic-search-available?)
(let [job (jobs/build
(jobs/of-type SemanticIndexRepair)
(jobs/with-identity repair-job-key))
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/task/indexer.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/task/indexer.clj
index a0195dc5b59..0048807f607 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/task/indexer.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/task/indexer.clj
@@ -3,10 +3,9 @@
[clojurewerkz.quartzite.jobs :as jobs]
[clojurewerkz.quartzite.schedule.simple :as simple]
[clojurewerkz.quartzite.triggers :as triggers]
- [metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource]
[metabase-enterprise.semantic-search.env :as semantic.env]
[metabase-enterprise.semantic-search.indexer :as semantic-search.indexer]
- [metabase.premium-features.core :as premium-features]
+ [metabase-enterprise.semantic-search.util :as semantic.u]
[metabase.search.engine :as search.engine]
[metabase.task.core :as task]
[metabase.util.log :as log])
@@ -32,14 +31,15 @@
SemanticSearchIndexer []
org.quartz.Job
(execute [_ _]
- (log/with-context {:quartz-job-type 'SemanticSearchIndexer}
- (when (search.engine/supported-engine? :search.engine/semantic)
- (try
- (vreset! execution-thread-ref (Thread/currentThread))
- (semantic-search.indexer/quartz-job-run! (semantic.env/get-pgvector-datasource!) (semantic.env/get-index-metadata))
- (finally
- (locking execution-thread-ref
- (vreset! execution-thread-ref nil)))))))
+ (when (semantic.u/semantic-search-available?)
+ (log/with-context {:quartz-job-type 'SemanticSearchIndexer}
+ (when (search.engine/supported-engine? :search.engine/semantic)
+ (try
+ (vreset! execution-thread-ref (Thread/currentThread))
+ (semantic-search.indexer/quartz-job-run! (semantic.env/get-pgvector-datasource!) (semantic.env/get-index-metadata))
+ (finally
+ (locking execution-thread-ref
+ (vreset! execution-thread-ref nil))))))))
org.quartz.InterruptableJob
(interrupt [_]
;; locking required here to avoid racing with the unset in the finally
@@ -52,8 +52,7 @@
(def ^:private ^Duration run-frequency (Duration/parse "PT20S"))
(defmethod task/init! ::SemanticSearchIndexer [_]
- (when (and (string? (not-empty semantic.db.datasource/db-url))
- (premium-features/has-feature? :semantic-search))
+ (when (semantic.u/semantic-search-available?)
(let [job (jobs/build
(jobs/of-type SemanticSearchIndexer)
(jobs/store-durably)
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/task/metric_collector.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/task/metric_collector.clj
index 2edc5667c21..f15d1ab54ac 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/task/metric_collector.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/task/metric_collector.clj
@@ -4,12 +4,10 @@
[clojurewerkz.quartzite.schedule.simple :as simple]
[clojurewerkz.quartzite.triggers :as triggers]
[honey.sql :as sql]
- [metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource]
[metabase-enterprise.semantic-search.dlq :as semantic.dlq]
[metabase-enterprise.semantic-search.env :as semantic.env]
- [metabase-enterprise.semantic-search.util :as semantic.util]
+ [metabase-enterprise.semantic-search.util :as semantic.u]
[metabase.analytics.core :as analytics]
- [metabase.premium-features.core :as premium-features]
[metabase.task.core :as task]
[metabase.util.log :as log]
[next.jdbc :as jdbc]
@@ -37,7 +35,7 @@
(let [{:keys [gate-table-name]} (semantic.env/get-index-metadata)]
(assert (string? gate-table-name))
(log/debugf "Checking size of gate table %s" gate-table-name)
- (if (semantic.util/table-exists? pgvector gate-table-name)
+ (if (semantic.u/table-exists? pgvector gate-table-name)
(let [table-size (row-count pgvector gate-table-name)]
(log/debugf "Setting `semantic-gate-size` metric to %d" table-size)
(analytics/set! :metabase-search/semantic-gate-size table-size)
@@ -46,7 +44,7 @@
(defn- active-index-id
[pgvector index-metadata]
- (if (semantic.util/table-exists? pgvector (:control-table-name index-metadata))
+ (if (semantic.u/table-exists? pgvector (:control-table-name index-metadata))
(:id (jdbc/execute-one!
pgvector
(sql/format {:select [[:active_id :id]]
@@ -59,7 +57,7 @@
(if-some [active-index-id (active-index-id pgvector index-metadata)]
(let [dlq-table-name (name (semantic.dlq/dlq-table-name-kw index-metadata active-index-id))]
(log/debugf "Checking size of DLQ table %s" dlq-table-name)
- (when (semantic.util/table-exists? pgvector dlq-table-name)
+ (when (semantic.u/table-exists? pgvector dlq-table-name)
(let [table-size (row-count pgvector dlq-table-name)]
(log/debugf "Setting `semantic-dlq-size` metric to %d" table-size)
(analytics/set! :metabase-search/semantic-dlq-size table-size)
@@ -67,10 +65,11 @@
(log/warn "DLQ table does not exist. Index may not have been initialized.")))
(defn- collect-metrics! []
- (let [pgvector (semantic.env/get-pgvector-datasource!)
- index-metadata (semantic.env/get-index-metadata)]
- (collect-gate-size! pgvector)
- (collect-dlq-size! pgvector index-metadata)))
+ (when (semantic.u/semantic-search-available?)
+ (let [pgvector (semantic.env/get-pgvector-datasource!)
+ index-metadata (semantic.env/get-index-metadata)]
+ (collect-gate-size! pgvector)
+ (collect-dlq-size! pgvector index-metadata))))
(task/defjob ^{DisallowConcurrentExecution true
:doc "Collect expensive semantic search metrics"}
@@ -81,8 +80,7 @@
(defmethod task/init! ::SemanticMetricCollector
[_]
- (when (and (string? (not-empty semantic.db.datasource/db-url))
- (premium-features/has-feature? :semantic-search))
+ (when (semantic.u/semantic-search-available?)
(let [job (jobs/build
(jobs/of-type SemanticMetricCollector)
(jobs/with-identity collector-job-key))
@@ -95,6 +93,3 @@
(simple/repeat-forever)))
(triggers/start-now))]
(task/schedule-task! job trigger))))
-
-(comment
- (task/trigger-now! collector-job-key))
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/task/usage_trimmer.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/task/usage_trimmer.clj
index 62f8086ee5b..049004ede02 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/task/usage_trimmer.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/task/usage_trimmer.clj
@@ -3,7 +3,7 @@
[clojurewerkz.quartzite.jobs :as jobs]
[clojurewerkz.quartzite.schedule.cron :as cron]
[clojurewerkz.quartzite.triggers :as triggers]
- [metabase.premium-features.core :as premium-features]
+ [metabase-enterprise.semantic-search.util :as semantic.u]
[metabase.task.core :as task]
[metabase.util.log :as log]
[toucan2.core :as t2])
@@ -20,10 +20,11 @@
(defn- trim-old-token-data!
[]
- (log/info "Attempting to delete old semantic search usage data.")
- (let [t (Timestamp/valueOf (.minusMonths (java.time.LocalDateTime/now) storage-months))]
- (t2/delete! :model/SemanticSearchTokenTracking {:where [:< :created_at t]}))
- (log/info "Semantic search old data cleanup successful."))
+ (when (semantic.u/semantic-search-available?)
+ (log/info "Attempting to delete old semantic search usage data.")
+ (let [t (Timestamp/valueOf (.minusMonths (java.time.LocalDateTime/now) storage-months))]
+ (t2/delete! :model/SemanticSearchTokenTracking {:where [:< :created_at t]}))
+ (log/info "Semantic search old data cleanup successful.")))
(task/defjob ^{DisallowConcurrentExecution true
:doc "Clean up inactive semantic search index tables"}
@@ -31,8 +32,8 @@
(trim-old-token-data!))
(defmethod task/init! ::SemanticSearchUsageTrimmer
- []
- (when (premium-features/has-feature? :semantic-search)
+ [_]
+ (when (semantic.u/semantic-search-available?)
(let [job (jobs/build
(jobs/of-type SemanticSearchUsageTrimmer)
(jobs/with-identity trimmer-job-key))
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/util.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/util.clj
index 11e563b0685..6615f590be5 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/util.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/util.clj
@@ -1,5 +1,7 @@
(ns metabase-enterprise.semantic-search.util
(:require
+ [metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource]
+ [metabase.premium-features.core :as premium-features]
[next.jdbc :as jdbc]
[next.jdbc.result-set :as jdbc.rs]))
@@ -11,3 +13,9 @@
table-name]
{:builder-fn jdbc.rs/as-unqualified-lower-maps})
(:table_exists false)))
+
+(defn semantic-search-available?
+ "Predicate to check whether semantic search is available on the instance."
+ []
+ (and (string? (not-empty semantic.db.datasource/db-url))
+ (premium-features/has-feature? :semantic-search)))
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/task/usage_trimmer_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/task/usage_trimmer_test.clj
index 74fe86e89f0..b2cf2fa2150 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/task/usage_trimmer_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/task/usage_trimmer_test.clj
@@ -2,7 +2,7 @@
(:require
[clojure.test :refer :all]
[metabase-enterprise.semantic-search.task.usage-trimmer :as semantic.task.trimmer]
- [metabase.test :as mt]
+ [metabase-enterprise.semantic-search.util :as semantic.u]
[toucan2.core :as t2])
(:import
(java.sql Timestamp)
@@ -11,7 +11,7 @@
(set! *warn-on-reflection* true)
(deftest trim-test
- (mt/with-premium-features #{:semantic-search}
+ (when (semantic.u/semantic-search-available?)
(t2/delete! :model/SemanticSearchTokenTracking)
(let [now (LocalDateTime/now)
;; semantic.task.trimmer/storage-months is set to 2