Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Make data-complexity-score CLI work against the live appdb (#74239)

Chris Truter899e541e2a26
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index 4b929bb0bf1..e0586f5a6bc 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -2884,6 +2884,7 @@
audit-app
collections
config
+ driver
metabot
models
premium-features
diff --git a/enterprise/backend/src/metabase_enterprise/data_complexity_score/cli.clj b/enterprise/backend/src/metabase_enterprise/data_complexity_score/cli.clj
index 741a59e2f41..b3f4ac6fc73 100644
--- a/enterprise/backend/src/metabase_enterprise/data_complexity_score/cli.clj
+++ b/enterprise/backend/src/metabase_enterprise/data_complexity_score/cli.clj
@@ -29,7 +29,6 @@
and the same bootstrap-dispatched [[entrypoint]] hook."
(:require
[clojure.java.io :as io]
- [clojure.pprint :as pprint]
[clojure.tools.cli :as cli]
[metabase-enterprise.data-complexity-score.complexity :as complexity]
[metabase-enterprise.data-complexity-score.metabot-scope :as metabot-scope]
@@ -37,7 +36,11 @@
[metabase-enterprise.data-complexity-score.representation :as representation]
[metabase-enterprise.data-complexity-score.synonym-source :as synonym-source]
[metabase-enterprise.data-complexity-score.task.complexity-score :as task.complexity-score]
- [metabase.app-db.core :as mdb]))
+ [metabase.app-db.core :as mdb]
+ ;; Loaded for side-effect: derives setting :on-change event topics from :metabase/event.
+ ;; metabase.core.core/entrypoint normally does this, but the standalone CLI bypasses it.
+ [metabase.driver.init]
+ [metabase.util.json :as json]))
(set! *warn-on-reflection* true)
@@ -84,7 +87,7 @@
:default unset
:parse-fn #(case % "true" true "false" false ::invalid)
:validate [#(not= % ::invalid) "--write-to-appdb must be 'true' or 'false'."]]
- ["-o" "--output PATH" "Write EDN result to this file instead of stdout."]
+ ["-o" "--output PATH" "Write pretty JSON to a file instead of single-line JSON on stdout."]
;; Consumed by `metabase.core.bootstrap` when invoked as `java -jar metabase.jar --mode …`. Declared
;; here so tools.cli accepts it and `parse-opts` doesn't report it as an unknown flag.
[nil "--mode MODE" "(JAR invocation only; handled by bootstrap before reaching this CLI.)"]
@@ -96,17 +99,11 @@
" java -jar metabase.jar --mode complexity-score [options]\n\n"
"Options:\n" summary))
-(defn- pretty [result]
- (with-out-str
- #_{:clj-kondo/ignore [:discouraged-var]}
- (pprint/pprint result)))
-
(defn- write-result! [result output-path]
(if output-path
- (spit output-path (pretty result))
+ (spit output-path (json/encode result {:pretty true}))
(do
- #_{:clj-kondo/ignore [:discouraged-var]}
- (print (pretty result))
+ (output! (json/encode result))
(flush))))
(defn- resolve-write?
@@ -131,16 +128,16 @@
(validate-dir! representation-dir))))
(defn- run-appdb-mode!
- "Score against the live appdb the same way the cron does; optionally persist."
+ "Score against the live appdb; optionally persist the row.
+ Snowplow is off here, so we don't advance `data-complexity-scoring-last-fingerprint` — leave that to the cron."
[write?]
(mdb/setup-db-without-migrations!)
(let [result (complexity/complexity-scores
(assoc (synonym-source/complexity-scores-opts)
- :metabot-scope (metabot-scope/internal-metabot-scope)))]
+ :metabot-scope (metabot-scope/internal-metabot-scope)
+ :emit-snowplow? false))]
(when write?
- (let [fp (task.complexity-score/current-fingerprint)]
- (data-complexity-score/record-score! fp "appdb" result)
- (task.complexity-score/maybe-advance-last-fingerprint! fp result)))
+ (data-complexity-score/record-score! (task.complexity-score/current-fingerprint) "appdb" result))
result))
(defn- run-representation-mode!
diff --git a/enterprise/backend/src/metabase_enterprise/data_complexity_score/complexity.clj b/enterprise/backend/src/metabase_enterprise/data_complexity_score/complexity.clj
index 8bac77affbe..72d4d9492df 100644
--- a/enterprise/backend/src/metabase_enterprise/data_complexity_score/complexity.clj
+++ b/enterprise/backend/src/metabase_enterprise/data_complexity_score/complexity.clj
@@ -4,7 +4,6 @@
:universe — everything (library entities + all active physical tables)
:metabot — what the internal Metabot can surface, narrowed by a caller-supplied scope."
(:require
- [clojure.pprint :as pprint]
[clojure.string :as str]
[metabase-enterprise.data-complexity-score.complexity-embedders :as embedders]
[metabase.analytics-interface.core :as analytics.interface]
@@ -56,30 +55,36 @@
;;; ----------------------------------- enumeration -----------------------------------
;;;
+(def ^:private ^:const in-clause-chunk-size
+ "Cap on the number of table-ids we put into a single `IN (...)` query. PostgreSQL prepared
+ statements top out at 65,535 parameters and we leave headroom for other clauses."
+ 50000)
+
(defn- table-field-counts
- "Return `{table-id field-count}` for active fields on the given `table-ids`. Single group-by query."
+ "Return `{table-id field-count}` for active fields on the given `table-ids`."
[table-ids]
- (if (empty? table-ids)
- {}
- (into {}
- (map (juxt :table_id :field_count))
- (t2/query {:select [:table_id [:%count.* :field_count]]
- :from [:metabase_field]
- :where [:and
- [:= :active true]
- [:in :table_id table-ids]]
- :group-by [:table_id]}))))
+ (into {}
+ (mapcat (fn [chunk]
+ (map (juxt :table_id :field_count)
+ (t2/query {:select [:table_id [:%count.* :field_count]]
+ :from [:metabase_field]
+ :where [:and
+ [:= :active true]
+ [:in :table_id chunk]]
+ :group-by [:table_id]}))))
+ (partition-all in-clause-chunk-size table-ids)))
(defn- table-measure-names
- "Return `{table-id [measure-name ...]}` for non-archived Measures on the given `table-ids`. A
- measure is a named MBQL aggregation attached to a Table — see [[metabase.measures.models.measure]]."
+ "Return `{table-id [measure-name ...]}` for non-archived Measures on the given `table-ids`.
+ A measure is a named MBQL aggregation attached to a Table — see [[metabase.measures.models.measure]]."
[table-ids]
- (if (empty? table-ids)
- {}
- (u/group-by :table_id :name
- (t2/select [:model/Measure :table_id :name]
- :archived false
- :table_id [:in table-ids]))))
+ (into {}
+ (mapcat (fn [chunk]
+ (u/group-by :table_id :name
+ (t2/select [:model/Measure :table_id :name]
+ :archived false
+ :table_id [:in chunk]))))
+ (partition-all in-clause-chunk-size table-ids)))
(defn- ->card-entity
"Shape a Card row into an entity map for scoring. Cards don't contribute to `:field-count` in
@@ -334,14 +339,6 @@
;;; ----------------------------------- public API ------------------------------------
-(defn- log-scores!
- "Log the result at :info so operators see the score in app logs even when Snowplow is off."
- [result]
- (log/info (str "Semantic complexity score:\n"
- ;; `pprint` goes through `with-out-str`, not `*out*`, so the "use metabase.util.log" lint is n/a.
- #_{:clj-kondo/ignore [:discouraged-var]}
- (with-out-str (pprint/pprint result)))))
-
(defn- snake ^String [x]
(str/replace (name x) "-" "_"))
@@ -485,8 +482,10 @@
to omit (the search-index path passes nil because its preprocessing
isn't a single named variant).
`:metabot-scope` — `{:verified-only? <bool> :collection-id <nil|Long>}` describing how
- the internal Metabot filters Cards."
- [& {:keys [embedder embedding-model-meta text-variant metabot-scope]}]
+ the internal Metabot filters Cards.
+ `:emit-snowplow?` — whether to publish per-score Snowplow events. Defaults true."
+ [& {:keys [embedder embedding-model-meta text-variant metabot-scope emit-snowplow?]
+ :or {emit-snowplow? true}}]
;;; NOTE: we fully materialize vectors of the relevant entities.
;;; For very large instances that means holding large lists in memory, but each catalog is consumed
;;; by many sub-score functions that each walk the collection, so making this reducible would
@@ -509,20 +508,20 @@
:meta (cond-> {:formula-version formula-version
:synonym-threshold synonym-similarity-threshold}
embedding-model-meta (assoc :embedding-model embedding-model-meta)
- text-variant (assoc :text-variant text-variant))}]
- (log-scores! result)
- (let [published? (time-phase! "publish" "all"
- (fn []
- (try
- (emit-snowplow! result)
- (catch Throwable t
- (log/warn t "Failed to publish complexity score to Snowplow")
- false))))]
- ;; `emit-snowplow!` returns true only when every event reached the tracker (false when
- ;; Snowplow is disabled or any emission failed) — scheduler/boot callers gate
- ;; `data-complexity-scoring-last-fingerprint` on this so a disabled collector or any
- ;; partial failure doesn't silently mark the fingerprint as published.
- (with-meta result {::snowplow-published? published?})))
+ text-variant (assoc :text-variant text-variant))}
+ ;; `emit-snowplow!` returns true only when every event reached the tracker (false when
+ ;; Snowplow is disabled or any emission failed) — scheduler/boot callers gate
+ ;; `data-complexity-scoring-last-fingerprint` on this so a disabled collector or any
+ ;; partial failure doesn't silently mark the fingerprint as published.
+ published? (time-phase! "publish" "all"
+ (fn []
+ (boolean
+ (when emit-snowplow?
+ (try
+ (emit-snowplow! result)
+ (catch Throwable t
+ (log/warn t "Failed to publish complexity score to Snowplow")))))))]
+ (with-meta result {::snowplow-published? published?}))
(finally
(analytics.interface/observe! :metabase-data-complexity/scoring-duration-ms
(u/since-ms total-timer))))))
diff --git a/enterprise/backend/src/metabase_enterprise/data_complexity_score/complexity_embedders.clj b/enterprise/backend/src/metabase_enterprise/data_complexity_score/complexity_embedders.clj
index cdcaee2c4d1..cdd6b5f3804 100644
--- a/enterprise/backend/src/metabase_enterprise/data_complexity_score/complexity_embedders.clj
+++ b/enterprise/backend/src/metabase_enterprise/data_complexity_score/complexity_embedders.clj
@@ -106,6 +106,11 @@
single oversized HTTP request — `get-embeddings-batch` does no chunking of its own for the
ai-service / openai providers.
+ Passes `:record-tokens? false` so complexity-score runs don't write to
+ `semantic_search_token_tracking` — the score is its own analytics signal and the embedding
+ calls here aren't user-driven search traffic. Holds for the CLI, the Quartz cron, and the
+ API endpoint alike.
+
Errors from the provider bubble up — `score-synonym-pairs` converts them into `nil`
measurements + an `:error` field so a broken run is visible downstream."
[model-descriptor]
@@ -120,7 +125,8 @@
(when (seq names)
(let [texts (mapv #(split-for-embedding (get name->raw %)) names)
vectors (into []
- (mapcat #(embeddings/get-embeddings-batch model-descriptor %))
+ (mapcat #(embeddings/get-embeddings-batch
+ model-descriptor % :record-tokens? false))
(partition-all provider-batch-size texts))]
(into {}
(keep (fn [[n v]] (when v [n (ensure-floats v)])))
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/embedding.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/embedding.clj
index aee4a49d10c..719b2822cb9 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/embedding.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/embedding.clj
@@ -9,7 +9,8 @@
[metabase.tracing.core :as tracing]
[metabase.util :as u]
[metabase.util.json :as json]
- [metabase.util.log :as log])
+ [metabase.util.log :as log]
+ [metabase.util.malli :as mu])
(:import
[com.knuddels.jtokkit Encodings]
[com.knuddels.jtokkit.api Encoding EncodingType]
@@ -187,20 +188,29 @@
(when (string? model-name)
(str/starts-with? model-name "text-embedding-3"))))
-(defn- openai-compatible-get-embeddings-batch
+(mu/defn- openai-compatible-get-embeddings-batch
"Call an OpenAI-compatible /v1/embeddings endpoint. Shared implementation for both
the `ai-service` and `openai` providers.
- `provider` — label for analytics (e.g. \"ai-service\", \"openai\")
- `endpoint` — full URL including /v1/embeddings
- `api-key` — Bearer token
- `model-name` — model identifier sent in the request body
- `texts` — collection of input strings
- `opts` — keyword opts; `:type` is forwarded to token tracking,
- `:extra-body` is merged into the request body (e.g. {:dimensions 1024}),
- `:snowplow?` when true fires a Snowplow token_usage event"
- [provider endpoint api-key model-name texts
- & {:keys [extra-body snowplow?] :as opts}]
+ `:provider` — label for analytics (e.g. \"ai-service\", \"openai\")
+ `:endpoint` — full URL including /v1/embeddings
+ `:api-key` — Bearer token
+ `:model-name` — model identifier sent in the request body
+ `:texts` — collection of input strings
+ `:record-tokens?` — true writes a `semantic_search_token_tracking` row, false skips it.
+ `:snowplow?` — optional; when true fires a Snowplow `token_usage` event
+ `:extra-body` — optional; merged into the request body (e.g. `{:dimensions 1024}`)
+ `:type` — optional; forwarded to the token-tracking row"
+ [{:keys [provider endpoint api-key model-name texts record-tokens? extra-body snowplow?] :as opts}
+ :- [:map
+ [:provider :string]
+ [:endpoint :string]
+ [:api-key :string]
+ [:model-name :string]
+ [:texts [:sequential :string]]
+ [:record-tokens? :boolean]
+ [:snowplow? {:optional true} [:maybe :boolean]]
+ [:extra-body {:optional true} [:maybe :map]]]]
(try
(log/debug (str "Calling " provider " embeddings API")
{:endpoint endpoint :documents (count texts) :tokens (count-tokens-batch texts)})
@@ -231,7 +241,8 @@
:estimated-costs-usd 0.0
:duration-ms (long (u/since-ms start-ms))
:tag "embedding_generation"}))
- (semantic.models.token-tracking/record-tokens model-name (:type opts) total-tokens)
+ (when record-tokens?
+ (semantic.models.token-tracking/record-tokens model-name (:type opts) total-tokens))
(decode-embeddings data))
(catch ConnectException e
(log/error e (str "Failed to connect to " provider) {:endpoint endpoint})
@@ -258,17 +269,31 @@
{:setting "ee-embedding-service-api-key"})))
[(str base-url "/v1/embeddings") api-key]))
-(defmethod get-embedding "ai-service" [{:keys [model-name]} text & {:as opts}]
+(defmethod get-embedding "ai-service"
+ [{:keys [model-name]} text & {:keys [record-tokens? type]}]
(let [[endpoint api-key] (embedding-service-resolve-config!)]
(first (openai-compatible-get-embeddings-batch
- "ai-service" endpoint api-key model-name [text]
- (assoc opts :snowplow? true)))))
-
-(defmethod get-embeddings-batch "ai-service" [{:keys [model-name]} texts & {:as opts}]
+ {:provider "ai-service"
+ :endpoint endpoint
+ :api-key api-key
+ :model-name model-name
+ :texts [text]
+ :snowplow? true
+ :record-tokens? record-tokens?
+ :type type}))))
+
+(defmethod get-embeddings-batch "ai-service"
+ [{:keys [model-name]} texts & {:keys [record-tokens? type]}]
(let [[endpoint api-key] (embedding-service-resolve-config!)]
(openai-compatible-get-embeddings-batch
- "ai-service" endpoint api-key model-name texts
- (assoc opts :snowplow? true))))
+ {:provider "ai-service"
+ :endpoint endpoint
+ :api-key api-key
+ :model-name model-name
+ :texts texts
+ :snowplow? true
+ :record-tokens? record-tokens?
+ :type type})))
(defmethod pull-model "ai-service" [_]
(log/debug "ai-service provider does not require pulling a model"))
@@ -283,19 +308,33 @@
(throw (ex-info "OpenAI API key not configured" {:setting "llm-openai-api-key"})))
[(str (semantic-settings/openai-api-base-url) "/v1/embeddings") api-key]))
-(defmethod get-embedding "openai" [embedding-model text & {:as opts}]
+(defmethod get-embedding "openai"
+ [embedding-model text & {:keys [record-tokens? type]}]
(let [[endpoint api-key] (openai-resolve-config!)]
(first (openai-compatible-get-embeddings-batch
- "openai" endpoint api-key (:model-name embedding-model) [text]
- (assoc opts :extra-body (when (supports-dimensions? embedding-model)
- {:dimensions (:vector-dimensions embedding-model)}))))))
-
-(defmethod get-embeddings-batch "openai" [embedding-model texts & {:as opts}]
+ {:provider "openai"
+ :endpoint endpoint
+ :api-key api-key
+ :model-name (:model-name embedding-model)
+ :texts [text]
+ :record-tokens? record-tokens?
+ :extra-body (when (supports-dimensions? embedding-model)
+ {:dimensions (:vector-dimensions embedding-model)})
+ :type type}))))
+
+(defmethod get-embeddings-batch "openai"
+ [embedding-model texts & {:keys [record-tokens? type]}]
(let [[endpoint api-key] (openai-resolve-config!)]
(openai-compatible-get-embeddings-batch
- "openai" endpoint api-key (:model-name embedding-model) texts
- (assoc opts :extra-body (when (supports-dimensions? embedding-model)
- {:dimensions (:vector-dimensions embedding-model)})))))
+ {:provider "openai"
+ :endpoint endpoint
+ :api-key api-key
+ :model-name (:model-name embedding-model)
+ :texts texts
+ :record-tokens? record-tokens?
+ :extra-body (when (supports-dimensions? embedding-model)
+ {:dimensions (:vector-dimensions embedding-model)})
+ :type type})))
(defmethod pull-model "openai" [_]
(log/debug "OpenAI provider does not require pulling a model"))
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj
index 7810e1892a5..24195268cb6 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj
@@ -397,8 +397,8 @@
results (transduce
(comp (partition-all *batch-size*)
(map (if serial?
- #(upsert-index-batch! connectable index % {:type :index})
- #(upsert-index-pooled! pool connectable index % {:type :index}))))
+ #(upsert-index-batch! connectable index % {:type :index :record-tokens? true})
+ #(upsert-index-pooled! pool connectable index % {:type :index :record-tokens? true}))))
conj
documents-reducible)]
(reduce (fn [update-counts result]
@@ -818,7 +818,8 @@
embedding (tracing/with-span :search "search.semantic.embedding"
{:search.semantic/provider (:provider embedding-model)
:search.semantic/model-name (:model-name embedding-model)}
- (embedding/get-embedding embedding-model search-string {:type :query}))
+ (embedding/get-embedding embedding-model search-string
+ {:type :query :record-tokens? true}))
embedding-time-ms (u/since-ms timer)
db-timer (u/start-timer)
@@ -881,7 +882,7 @@
(def embedding-model (embedding/get-configured-model))
(def index (default-index embedding-model))
(def search-ctx {:search-string "pasta"})
- (def embed (embedding/get-embedding embedding-model (:search-string search-ctx)))
+ (def embed (embedding/get-embedding embedding-model (:search-string search-ctx) {:record-tokens? true}))
(def scorers (scoring/semantic-scorers (:table-name index) search-ctx))
(keyword-search-query index search-ctx)
@@ -965,7 +966,7 @@
:verified nil})
(def search-context {:search-string search-string})
- (def embedding (embedding/get-embedding (:embedding-model index) search-string))
+ (def embedding (embedding/get-embedding (:embedding-model index) search-string {:record-tokens? true}))
(def scorers (scoring/semantic-scorers (:table-name index) search-context))
;; Format queries for execution
diff --git a/enterprise/backend/test/metabase_enterprise/data_complexity_score/cli_test.clj b/enterprise/backend/test/metabase_enterprise/data_complexity_score/cli_test.clj
index 51bc6ffeeff..a99e45a40f1 100644
--- a/enterprise/backend/test/metabase_enterprise/data_complexity_score/cli_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/data_complexity_score/cli_test.clj
@@ -1,6 +1,5 @@
(ns metabase-enterprise.data-complexity-score.cli-test
(:require
- [clojure.edn :as edn]
[clojure.java.io :as io]
[clojure.test :refer :all]
[metabase-enterprise.data-complexity-score.cli :as cli]
@@ -13,7 +12,8 @@
[metabase-enterprise.data-complexity-score.task.complexity-score :as task.complexity-score]
[metabase.app-db.core :as mdb]
[metabase.test :as mt]
- [metabase.util :as u]))
+ [metabase.util :as u]
+ [metabase.util.json :as json]))
(set! *warn-on-reflection* true)
@@ -89,16 +89,21 @@
(is (= 2 (:field-count events))
"events Table should have exactly 2 fields — the side-car must not be counted"))))
-(deftest ^:sequential run-cli-writes-readable-edn-to-output-file-test
+(deftest ^:sequential run-cli-writes-readable-json-test
;; Not ^:parallel: calls `cli/write-result!`, which kondo flags as a destructive function in
;; parallel tests. The temp file we hand it is unique-per-call so the write is safe in
;; principle, but the lint flag is the right default — drop it instead of whitelisting.
- (testing "--output path gets a readable EDN dump of the same result"
- (let [tmp (doto (java.io.File/createTempFile "complexity-cli-output-" ".edn") .deleteOnExit)]
- ;; Call internals instead of -main, which terminates the JVM via System/exit.
- (#'cli/write-result! (#'cli/run-cli {:representation-dir representation-fixture-dir})
- (.getAbsolutePath tmp))
- (is (= 215 (-> (slurp tmp) edn/read-string :library :total))))))
+ ;; Call internals instead of -main, which terminates the JVM via System/exit.
+ (let [result (#'cli/run-cli {:representation-dir representation-fixture-dir})]
+ (testing "without --output, stdout gets single-line JSON"
+ (let [stdout (with-out-str (#'cli/write-result! result nil))]
+ (is (= 215 (-> stdout (json/decode true) :library :total)))
+ (is (not (re-find #"\n.+" stdout)) "stdout JSON should be single-line")))
+ (testing "with --output, the file gets pretty JSON and stdout stays silent"
+ (let [tmp (doto (java.io.File/createTempFile "complexity-cli-output-" ".json") .deleteOnExit)
+ stdout (with-out-str (#'cli/write-result! result (.getAbsolutePath tmp)))]
+ (is (= "" stdout))
+ (is (= 215 (-> (slurp tmp) (json/decode true) :library :total)))))))
;;; ------------------------------------- pure embedder/scoring tests -------------------------------------
@@ -330,38 +335,38 @@
"representation-derived rows must never advance the cron's last-fingerprint setting")))))
(deftest ^:sequential run-cli-appdb-mode-defaults-to-writing-test
- (testing "appdb mode with no --write-to-appdb flag defaults to writing (true)"
+ (testing "appdb mode with no --write-to-appdb flag defaults to writing (true) but doesn't advance the cron fingerprint"
+ ;; CLI runs disable Snowplow, so they can't legitimately advance
+ ;; `data-complexity-scoring-last-fingerprint` — that setting is the cron's
+ ;; been-published-already gate and only a confirmed publish should move it. The CLI just
+ ;; persists the score row so operators can see the run.
(let [calls (atom [])
- advance-calls (atom [])]
+ advance-calls (atom 0)]
(mt/with-dynamic-fn-redefs [mdb/setup-db-without-migrations! (fn [])
complexity/complexity-scores (fn [& _] {:meta {}})
synonym-source/complexity-scores-opts (constantly {})
metabot-scope/internal-metabot-scope (constantly {})
task.complexity-score/current-fingerprint (constantly "appdb-fp")
- task.complexity-score/maybe-advance-last-fingerprint! (fn [fp _result]
- (swap! advance-calls conj fp))
+ task.complexity-score/maybe-advance-last-fingerprint! (fn [& _]
+ (swap! advance-calls inc))
data-complexity-score/record-score! (fn [fp source _result]
(swap! calls conj [fp source]))]
(#'cli/run-cli {:source "appdb"})
(is (= [["appdb-fp" "appdb"]] @calls)
"appdb-mode default must write one row stamped source=\"appdb\"")
- (is (= ["appdb-fp"] @advance-calls)
- "appdb-mode write path must call maybe-advance-last-fingerprint! the same way the cron does")))))
+ (is (zero? @advance-calls)
+ "CLI must not advance the cron's last-fingerprint setting")))))
(deftest ^:sequential run-cli-appdb-mode-respects-explicit-no-write-test
- (testing "appdb + --write-to-appdb false scores but never persists or advances the fingerprint"
- (let [persisted? (atom false)
- advance-calls (atom 0)]
+ (testing "appdb + --write-to-appdb false scores but never persists"
+ (let [persisted? (atom false)]
(mt/with-dynamic-fn-redefs [mdb/setup-db-without-migrations! (fn [])
complexity/complexity-scores (fn [& _] {:meta {}})
synonym-source/complexity-scores-opts (constantly {})
metabot-scope/internal-metabot-scope (constantly {})
- data-complexity-score/record-score! (fn [& _] (reset! persisted? true))
- task.complexity-score/maybe-advance-last-fingerprint! (fn [& _]
- (swap! advance-calls inc))]
+ data-complexity-score/record-score! (fn [& _] (reset! persisted? true))]
(#'cli/run-cli {:source "appdb" :write-to-appdb false})
- (is (false? @persisted?))
- (is (zero? @advance-calls))))))
+ (is (false? @persisted?))))))
(deftest ^:parallel dir-digest-is-stable-and-content-sensitive-test
(testing "dir-digest produces the same value for the same content"
diff --git a/enterprise/backend/test/metabase_enterprise/data_complexity_score/complexity_test.clj b/enterprise/backend/test/metabase_enterprise/data_complexity_score/complexity_test.clj
index b022e17711d..3a0d5d66cd3 100644
--- a/enterprise/backend/test/metabase_enterprise/data_complexity_score/complexity_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/data_complexity_score/complexity_test.clj
@@ -581,12 +581,28 @@
(is (fn? embedder)
"synonym-source returns a fresh provider-embedder for the descriptor")))))
+(deftest ^:sequential provider-embedder-suppresses-token-tracking-test
+ (testing "provider-embedder always passes :record-tokens? false to get-embeddings-batch"
+ ;; Complexity scoring isn't user-driven search traffic; the score itself is the analytics
+ ;; signal, so embedding calls here shouldn't write to `semantic_search_token_tracking`
+ ;; — regardless of caller (CLI, Quartz cron, API).
+ (let [captured (atom nil)
+ embedder (embedders/provider-embedder {:provider "ai-service" :model-name "fake" :model-dimensions 4})]
+ (mt/with-dynamic-fn-redefs [embeddings/get-embeddings-batch
+ (fn [_model texts & {:as opts}]
+ (reset! captured opts)
+ (repeat (count texts) [1.0]))]
+ (embedder [{:id 1 :name "orders" :kind :table}])
+ (is (false? (:record-tokens? @captured)))))))
+
(deftest ^:sequential provider-embedder-splits-names-before-calling-provider-test
(testing "provider-embedder splits names on _, -, ., and camelCase before sending to get-embeddings-batch"
(let [captured (atom nil)
embedder (embedders/provider-embedder {:provider "ai-service" :model-name "fake" :model-dimensions 4})]
(mt/with-dynamic-fn-redefs [embeddings/get-embeddings-batch
- (fn [_model texts] (reset! captured (vec texts)) (repeat (count texts) [1.0]))]
+ (fn [_model texts & _opts]
+ (reset! captured (vec texts))
+ (repeat (count texts) [1.0]))]
(embedder
[{:id 1 :name "monthly_active_users" :kind :table}
{:id 2 :name "dim-date" :kind :table}
@@ -602,7 +618,7 @@
(testing "provider errors bubble up so score-synonym-pairs can report nil measurements + :error"
(let [embedder (embedders/provider-embedder {:provider "ai-service" :model-name "fake" :model-dimensions 4})]
(mt/with-dynamic-fn-redefs [embeddings/get-embeddings-batch
- (fn [_ _] (throw (ex-info "ai-service down" {})))]
+ (fn [& _] (throw (ex-info "ai-service down" {})))]
(is (thrown-with-msg? Throwable #"ai-service down"
(embedder [{:id 1 :name "orders" :kind :table}])))))))
@@ -610,7 +626,7 @@
(testing "vectors come back keyed by the normalized name; nil slots are dropped"
(let [embedder (embedders/provider-embedder {:provider "ai-service" :model-name "fake" :model-dimensions 2})]
(mt/with-dynamic-fn-redefs [embeddings/get-embeddings-batch
- (fn [_ texts]
+ (fn [_ texts & _opts]
(for [t texts] (when-not (= t "drop me") [1.0 0.0])))]
(let [result (embedder
[{:id 1 :name "keep me" :kind :table}
@@ -778,7 +794,7 @@
(deftest ^:sequential emit-snowplow-includes-embedding-model-and-text-variant-meta-test
(testing "every event's parameters carry the nested embedding_model + text_variant from synonym-source"
(snowplow-test/with-fake-snowplow-collector
- (mt/with-dynamic-fn-redefs [embeddings/get-embeddings-batch (fn [_ _] [])
+ (mt/with-dynamic-fn-redefs [embeddings/get-embeddings-batch (fn [& _] [])
complexity/enumerate-catalogs
(constantly {:library [(entity :name "orders")]
:universe [(entity :name "orders")]
@@ -834,35 +850,24 @@
(messages))
"a warning about the publish failure was logged"))))))
-(deftest ^:sequential local-info-log-is-emitted-even-when-snowplow-fails-test
- (testing "the 'Semantic complexity score' info log fires independently of Snowplow emission"
- ;; Guards two regressions together: local logging being removed, and local logging being
- ;; gated on successful telemetry (so a broken collector would silence the operator-visible log).
- (mt/with-dynamic-fn-redefs [complexity/enumerate-catalogs
- (constantly {:library [(entity :name "orders")]
- :universe [(entity :name "orders")]
- :metabot []})
- analytics/track-event! (fn [& _] (throw (RuntimeException. "snowplow down")))]
- (mt/with-log-messages-for-level [messages [metabase-enterprise.data-complexity-score.complexity :info]]
- (complexity/complexity-scores :embedder nil)
- (is (some #(and (= :info (:level %))
- (re-find #"Semantic complexity score" (:message %)))
- (messages))
- "the score was logged locally at :info even though Snowplow emission threw")))))
-
-(deftest ^:sequential scheduled-task-logs-score-test
- (testing "the scheduled task body runs complexity-scores so operators see a score line in the logs"
- (mt/with-dynamic-fn-redefs [complexity/enumerate-catalogs
- (constantly {:library [(entity :name "orders")]
- :universe [(entity :name "orders")]
- :metabot []})]
- (mt/with-temporary-setting-values [data-complexity-scoring-enabled true]
- (mt/with-log-messages-for-level [messages [metabase-enterprise.data-complexity-score.complexity :info]]
- (#'task.complexity-score/run-scoring! "test-fp")
- (is (some #(and (= :info (:level %))
- (re-find #"Semantic complexity score" (:message %)))
- (messages))
- "the scheduled task produced the expected info log"))))))
+(deftest ^:sequential complexity-scores-emit-snowplow-false-skips-publish-test
+ (testing ":emit-snowplow? false bypasses Snowplow entirely and stamps ::snowplow-published? false"
+ ;; Pins the CLI/appdb path: the standalone scorer disables telemetry, so we must not call
+ ;; into the analytics tracker at all (catching failures upstream isn't enough — the cost is
+ ;; the syscall + the misleading WARN). The accompanying metadata flag lets callers tell
+ ;; intentional skip apart from a successful publish, so they can avoid advancing fingerprints
+ ;; or otherwise treating the run as observed.
+ (let [tracked? (atom false)]
+ (mt/with-dynamic-fn-redefs [complexity/enumerate-catalogs
+ (constantly {:library [(entity :name "orders")]
+ :universe [(entity :name "orders")]
+ :metabot []})
+ analytics/track-event! (fn [& _] (reset! tracked? true))]
+ (let [result (complexity/complexity-scores :embedder nil :emit-snowplow? false)]
+ (is (false? @tracked?)
+ "analytics/track-event! must not be called when :emit-snowplow? false")
+ (is (false? (::complexity/snowplow-published? (meta result)))
+ "result metadata reflects that nothing was published"))))))
(deftest ^:sequential complexity-score-library-hermetic-test
(testing "library score is computed over exactly the Library collection tree — known inputs produce known scores"
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/embedding_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/embedding_test.clj
index 1e7d3956b6d..9624292217e 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/embedding_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/embedding_test.clj
@@ -209,11 +209,13 @@
(is (= mock-embedding (vec (#'embedding/get-embedding {:provider provider
:model-name "some-model"
:vector-dimensions 4}
- "hello"))))
+ "hello"
+ {:record-tokens? true}))))
(is (= [mock-embedding] (mapv vec (#'embedding/get-embeddings-batch {:provider provider
:model-name "some-model"
:vector-dimensions 4}
- ["hello"]))))
+ ["hello"]
+ {:record-tokens? true}))))
(when counts-tokens?
(let [tokens-calls (filter #(= :metabase-search/semantic-embedding-tokens (first %)) @analytics-calls)]
(is (= 2 (count tokens-calls)))
@@ -264,7 +266,8 @@
(embedding/get-embeddings-batch {:provider "ai-service"
:model-name "test-model"
:vector-dimensions 3}
- ["hello world"]))
+ ["hello world"]
+ {:record-tokens? true}))
(let [events (->> (snowplow-test/pop-event-data-and-user-id!)
;; Filter out unrelated setup events (e.g. new_instance_created
;; triggered by instance-creation setting being read for the first time)