The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Add guards to metabot and semantic search job implementations (#64031)
lbrdnkc28f66c96165
diff --git a/enterprise/backend/src/metabase_enterprise/metabot_v3/task/suggested_prompts_generator.clj b/enterprise/backend/src/metabase_enterprise/metabot_v3/task/suggested_prompts_generator.cljindex bd91d9860bc..5d87dccb5ae 100644--- a/enterprise/backend/src/metabase_enterprise/metabot_v3/task/suggested_prompts_generator.clj+++ b/enterprise/backend/src/metabase_enterprise/metabot_v3/task/suggested_prompts_generator.clj@@ -20,16 +20,17 @@(def ^:private generator-trigger-key (triggers/key "metabase.task.metabot-v3.suggested-prompts-generator.trigger"))(defn- maybe-generate-suggested-prompts! []- (let [metabot-eid (get-in metabot-v3.config/metabot-config- [metabot-v3.config/internal-metabot-id :entity-id])- metabot-id (t2/select-one-pk :model/Metabot :entity_id metabot-eid)- suggested-prompts-cnt (t2/count :model/MetabotPrompt :metabot_id metabot-id)]- (if (zero? suggested-prompts-cnt)- (do- (log/info "No suggested prompts found. Generating suggested prompts.")- (metabot-v3.suggested-prompts/generate-sample-prompts metabot-id)- (log/info "Suggested prompts generated successfully."))- (log/info "Suggested prompts are present. Not generating."))))+ (when (premium-features/has-feature? :metabot-v3)+ (let [metabot-eid (get-in metabot-v3.config/metabot-config+ [metabot-v3.config/internal-metabot-id :entity-id])+ metabot-id (t2/select-one-pk :model/Metabot :entity_id metabot-eid)+ suggested-prompts-cnt (t2/count :model/MetabotPrompt :metabot_id metabot-id)]+ (if (zero? suggested-prompts-cnt)+ (do+ (log/info "No suggested prompts found. Generating suggested prompts.")+ (metabot-v3.suggested-prompts/generate-sample-prompts metabot-id)+ (log/info "Suggested prompts generated successfully."))+ (log/info "Suggested prompts are present. Not generating.")))))(task/defjob ^{DisallowConcurrentExecution true:doc "Initial _suggested prompts_ generation for internal Metabot."}diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/task/index_cleanup.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/task/index_cleanup.cljindex 191fed820a9..ef8a8446052 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/task/index_cleanup.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/task/index_cleanup.clj@@ -8,10 +8,9 @@[honey.sql :as sql][honey.sql.helpers :as sql.helpers][java-time.api :as t]- [metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource][metabase-enterprise.semantic-search.env :as semantic.env][metabase-enterprise.semantic-search.settings :as semantic.settings]- [metabase.premium-features.core :as premium-features]+ [metabase-enterprise.semantic-search.util :as semantic.u][metabase.task.core :as task][metabase.util.log :as log][next.jdbc :as jdbc]@@ -165,11 +164,12 @@(defn- cleanup-stale-indexes-and-gate-tombstones![]- (let [pgvector (semantic.env/get-pgvector-datasource!)- index-metadata (semantic.env/get-index-metadata)]- (cleanup-stale-indexes! pgvector index-metadata)- (cleanup-old-gate-tombstones! pgvector index-metadata)- (cleanup-orphan-repair-tables! pgvector)))+ (when (semantic.u/semantic-search-available?)+ (let [pgvector (semantic.env/get-pgvector-datasource!)+ index-metadata (semantic.env/get-index-metadata)]+ (cleanup-stale-indexes! pgvector index-metadata)+ (cleanup-old-gate-tombstones! pgvector index-metadata)+ (cleanup-orphan-repair-tables! pgvector))))(def ^:private cleanup-job-key (jobs/key "metabase.task.semantic-index-cleanup.job"))(def ^:private cleanup-trigger-key (triggers/key "metabase.task.semantic-index-cleanup.trigger"))@@ -180,8 +180,7 @@(cleanup-stale-indexes-and-gate-tombstones!))(defmethod task/init! ::SemanticIndexCleanup [_]- (when (and (string? (not-empty semantic.db.datasource/db-url))- (premium-features/has-feature? :semantic-search))+ (when (semantic.u/semantic-search-available?)(let [job (jobs/build(jobs/of-type SemanticIndexCleanup)(jobs/with-identity cleanup-job-key))diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/task/index_repair.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/task/index_repair.cljindex 05488c24fe2..6adea36cf33 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/task/index_repair.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/task/index_repair.clj@@ -5,8 +5,7 @@[clojurewerkz.quartzite.schedule.cron :as cron][clojurewerkz.quartzite.triggers :as triggers][metabase-enterprise.semantic-search.core :as semantic-search.core]- [metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource]- [metabase.premium-features.core :as premium-features]+ [metabase-enterprise.semantic-search.util :as semantic.u][metabase.search.ingestion :as search.ingestion][metabase.task.core :as task][metabase.util.log :as log])@@ -21,17 +20,17 @@(task/defjob ^{DisallowConcurrentExecution true:doc "Runs repair-index! to maintain semantic search consistency"}SemanticIndexRepair [_ctx]- (log/with-context {:quartz-job-type 'SemanticIndexRepair}- (try- (log/info "Starting semantic search index repair")- (semantic-search.core/repair-index! (search.ingestion/searchable-documents))- (log/info "Completed semantic search index repair")- (catch Exception e- (log/error e "Failed to complete semantic search index repair")))))+ (when (semantic.u/semantic-search-available?)+ (log/with-context {:quartz-job-type 'SemanticIndexRepair}+ (try+ (log/info "Starting semantic search index repair")+ (semantic-search.core/repair-index! (search.ingestion/searchable-documents))+ (log/info "Completed semantic search index repair")+ (catch Exception e+ (log/error e "Failed to complete semantic search index repair"))))))(defmethod task/init! ::SemanticIndexRepair [_]- (when (and (string? (not-empty semantic.db.datasource/db-url))- (premium-features/has-feature? :semantic-search))+ (when (semantic.u/semantic-search-available?)(let [job (jobs/build(jobs/of-type SemanticIndexRepair)(jobs/with-identity repair-job-key))diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/task/indexer.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/task/indexer.cljindex a0195dc5b59..0048807f607 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/task/indexer.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/task/indexer.clj@@ -3,10 +3,9 @@[clojurewerkz.quartzite.jobs :as jobs][clojurewerkz.quartzite.schedule.simple :as simple][clojurewerkz.quartzite.triggers :as triggers]- [metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource][metabase-enterprise.semantic-search.env :as semantic.env][metabase-enterprise.semantic-search.indexer :as semantic-search.indexer]- [metabase.premium-features.core :as premium-features]+ [metabase-enterprise.semantic-search.util :as semantic.u][metabase.search.engine :as search.engine][metabase.task.core :as task][metabase.util.log :as log])@@ -32,14 +31,15 @@SemanticSearchIndexer []org.quartz.Job(execute [_ _]- (log/with-context {:quartz-job-type 'SemanticSearchIndexer}- (when (search.engine/supported-engine? :search.engine/semantic)- (try- (vreset! execution-thread-ref (Thread/currentThread))- (semantic-search.indexer/quartz-job-run! (semantic.env/get-pgvector-datasource!) (semantic.env/get-index-metadata))- (finally- (locking execution-thread-ref- (vreset! execution-thread-ref nil)))))))+ (when (semantic.u/semantic-search-available?)+ (log/with-context {:quartz-job-type 'SemanticSearchIndexer}+ (when (search.engine/supported-engine? :search.engine/semantic)+ (try+ (vreset! execution-thread-ref (Thread/currentThread))+ (semantic-search.indexer/quartz-job-run! (semantic.env/get-pgvector-datasource!) (semantic.env/get-index-metadata))+ (finally+ (locking execution-thread-ref+ (vreset! execution-thread-ref nil))))))))org.quartz.InterruptableJob(interrupt [_];; locking required here to avoid racing with the unset in the finally@@ -52,8 +52,7 @@(def ^:private ^Duration run-frequency (Duration/parse "PT20S"))(defmethod task/init! ::SemanticSearchIndexer [_]- (when (and (string? (not-empty semantic.db.datasource/db-url))- (premium-features/has-feature? :semantic-search))+ (when (semantic.u/semantic-search-available?)(let [job (jobs/build(jobs/of-type SemanticSearchIndexer)(jobs/store-durably)diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/task/metric_collector.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/task/metric_collector.cljindex 2edc5667c21..f15d1ab54ac 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/task/metric_collector.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/task/metric_collector.clj@@ -4,12 +4,10 @@[clojurewerkz.quartzite.schedule.simple :as simple][clojurewerkz.quartzite.triggers :as triggers][honey.sql :as sql]- [metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource][metabase-enterprise.semantic-search.dlq :as semantic.dlq][metabase-enterprise.semantic-search.env :as semantic.env]- [metabase-enterprise.semantic-search.util :as semantic.util]+ [metabase-enterprise.semantic-search.util :as semantic.u][metabase.analytics.core :as analytics]- [metabase.premium-features.core :as premium-features][metabase.task.core :as task][metabase.util.log :as log][next.jdbc :as jdbc]@@ -37,7 +35,7 @@(let [{:keys [gate-table-name]} (semantic.env/get-index-metadata)](assert (string? gate-table-name))(log/debugf "Checking size of gate table %s" gate-table-name)- (if (semantic.util/table-exists? pgvector gate-table-name)+ (if (semantic.u/table-exists? pgvector gate-table-name)(let [table-size (row-count pgvector gate-table-name)](log/debugf "Setting `semantic-gate-size` metric to %d" table-size)(analytics/set! :metabase-search/semantic-gate-size table-size)@@ -46,7 +44,7 @@(defn- active-index-id[pgvector index-metadata]- (if (semantic.util/table-exists? pgvector (:control-table-name index-metadata))+ (if (semantic.u/table-exists? pgvector (:control-table-name index-metadata))(:id (jdbc/execute-one!pgvector(sql/format {:select [[:active_id :id]]@@ -59,7 +57,7 @@(if-some [active-index-id (active-index-id pgvector index-metadata)](let [dlq-table-name (name (semantic.dlq/dlq-table-name-kw index-metadata active-index-id))](log/debugf "Checking size of DLQ table %s" dlq-table-name)- (when (semantic.util/table-exists? pgvector dlq-table-name)+ (when (semantic.u/table-exists? pgvector dlq-table-name)(let [table-size (row-count pgvector dlq-table-name)](log/debugf "Setting `semantic-dlq-size` metric to %d" table-size)(analytics/set! :metabase-search/semantic-dlq-size table-size)@@ -67,10 +65,11 @@(log/warn "DLQ table does not exist. Index may not have been initialized.")))(defn- collect-metrics! []- (let [pgvector (semantic.env/get-pgvector-datasource!)- index-metadata (semantic.env/get-index-metadata)]- (collect-gate-size! pgvector)- (collect-dlq-size! pgvector index-metadata)))+ (when (semantic.u/semantic-search-available?)+ (let [pgvector (semantic.env/get-pgvector-datasource!)+ index-metadata (semantic.env/get-index-metadata)]+ (collect-gate-size! pgvector)+ (collect-dlq-size! pgvector index-metadata))))(task/defjob ^{DisallowConcurrentExecution true:doc "Collect expensive semantic search metrics"}@@ -81,8 +80,7 @@(defmethod task/init! ::SemanticMetricCollector[_]- (when (and (string? (not-empty semantic.db.datasource/db-url))- (premium-features/has-feature? :semantic-search))+ (when (semantic.u/semantic-search-available?)(let [job (jobs/build(jobs/of-type SemanticMetricCollector)(jobs/with-identity collector-job-key))@@ -95,6 +93,3 @@(simple/repeat-forever)))(triggers/start-now))](task/schedule-task! job trigger))))--(comment- (task/trigger-now! collector-job-key))diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/task/usage_trimmer.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/task/usage_trimmer.cljindex 62f8086ee5b..049004ede02 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/task/usage_trimmer.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/task/usage_trimmer.clj@@ -3,7 +3,7 @@[clojurewerkz.quartzite.jobs :as jobs][clojurewerkz.quartzite.schedule.cron :as cron][clojurewerkz.quartzite.triggers :as triggers]- [metabase.premium-features.core :as premium-features]+ [metabase-enterprise.semantic-search.util :as semantic.u][metabase.task.core :as task][metabase.util.log :as log][toucan2.core :as t2])@@ -20,10 +20,11 @@(defn- trim-old-token-data![]- (log/info "Attempting to delete old semantic search usage data.")- (let [t (Timestamp/valueOf (.minusMonths (java.time.LocalDateTime/now) storage-months))]- (t2/delete! :model/SemanticSearchTokenTracking {:where [:< :created_at t]}))- (log/info "Semantic search old data cleanup successful."))+ (when (semantic.u/semantic-search-available?)+ (log/info "Attempting to delete old semantic search usage data.")+ (let [t (Timestamp/valueOf (.minusMonths (java.time.LocalDateTime/now) storage-months))]+ (t2/delete! :model/SemanticSearchTokenTracking {:where [:< :created_at t]}))+ (log/info "Semantic search old data cleanup successful.")))(task/defjob ^{DisallowConcurrentExecution true:doc "Clean up inactive semantic search index tables"}@@ -31,8 +32,8 @@(trim-old-token-data!))(defmethod task/init! ::SemanticSearchUsageTrimmer- []- (when (premium-features/has-feature? :semantic-search)+ [_]+ (when (semantic.u/semantic-search-available?)(let [job (jobs/build(jobs/of-type SemanticSearchUsageTrimmer)(jobs/with-identity trimmer-job-key))diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/util.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/util.cljindex 11e563b0685..6615f590be5 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/util.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/util.clj@@ -1,5 +1,7 @@(ns metabase-enterprise.semantic-search.util(:require+ [metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource]+ [metabase.premium-features.core :as premium-features][next.jdbc :as jdbc][next.jdbc.result-set :as jdbc.rs]))@@ -11,3 +13,9 @@table-name]{:builder-fn jdbc.rs/as-unqualified-lower-maps})(:table_exists false)))++(defn semantic-search-available?+ "Predicate to check whether semantic search is available on the instance."+ []+ (and (string? (not-empty semantic.db.datasource/db-url))+ (premium-features/has-feature? :semantic-search)))diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/task/usage_trimmer_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/task/usage_trimmer_test.cljindex 74fe86e89f0..b2cf2fa2150 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/task/usage_trimmer_test.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/task/usage_trimmer_test.clj@@ -2,7 +2,7 @@(:require[clojure.test :refer :all][metabase-enterprise.semantic-search.task.usage-trimmer :as semantic.task.trimmer]- [metabase.test :as mt]+ [metabase-enterprise.semantic-search.util :as semantic.u][toucan2.core :as t2])(:import(java.sql Timestamp)@@ -11,7 +11,7 @@(set! *warn-on-reflection* true)(deftest trim-test- (mt/with-premium-features #{:semantic-search}+ (when (semantic.u/semantic-search-available?)(t2/delete! :model/SemanticSearchTokenTracking)(let [now (LocalDateTime/now);; semantic.task.trimmer/storage-months is set to 2