Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Semantic search token usage tracking (#63591)

lbrdnk63cc5c3137a7
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/embedding.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/embedding.clj
index e49597c485f..13f2631feb0 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/embedding.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/embedding.clj
@@ -3,6 +3,7 @@
[clj-http.client :as http]
[clojure.string :as str]
[metabase-enterprise.metabot-v3.client :as metabot-v3.client]
+ [metabase-enterprise.semantic-search.models.token-tracking :as semantic.models.token-tracking]
[metabase-enterprise.semantic-search.settings :as semantic-settings]
[metabase.analytics.core :as analytics]
[metabase.util :as u]
@@ -128,11 +129,11 @@
(defmulti get-embedding
"Returns a single embedding vector for the given text"
- {:arglists '([embedding-model text])} dispatch-provider)
+ {:arglists '([embedding-model text & opts])} dispatch-provider)
(defmulti get-embeddings-batch
"Returns a sequential collection of embedding vectors, in the same order as the input texts."
- {:arglists '([embedding-model texts])} dispatch-provider)
+ {:arglists '([embedding-model texts & opts])} dispatch-provider)
(defmulti pull-model
"If a model needs to be downloaded (which is the case for ollama), downloads it."
@@ -171,34 +172,37 @@
(log/error e "Failed to pull embedding model")
(throw e))))
-(defmethod get-embedding "ollama" [{:keys [model-name]} text] (ollama-get-embedding model-name text))
-(defmethod get-embeddings-batch "ollama" [{:keys [model-name]} texts] (ollama-get-embeddings-batch model-name texts))
+;; Ollama is not used in production. Token tracking is not implemented.
+(defmethod get-embedding "ollama" [{:keys [model-name]} text & {:as _opts}] (ollama-get-embedding model-name text))
+(defmethod get-embeddings-batch "ollama" [{:keys [model-name]} texts & {:as _opts}] (ollama-get-embeddings-batch model-name texts))
(defmethod pull-model "ollama" [{:keys [model-name]}] (ollama-pull-model model-name))
;;;; AI Service impl
-(defn- ai-service-get-embeddings-batch [model-name texts]
+(defn- ai-service-get-embeddings-batch [model-name texts & {:as opts}]
(try
(log/debug "Calling AI Service embeddings API" {:documents (count texts) :tokens (count-tokens-batch texts)})
- (let [response (metabot-v3.client/generate-embeddings model-name texts)]
+ (let [response (metabot-v3.client/generate-embeddings model-name texts)
+ total-tokens (->> response :usage :total_tokens)]
(analytics/inc! :metabase-search/semantic-embedding-tokens
{:provider "ai-service", :model model-name}
- (->> response :usage :total_tokens))
+ total-tokens)
+ (semantic.models.token-tracking/record-tokens model-name (:type opts) total-tokens)
(extract-base64-response-embeddings response))
(catch Exception e
(log/error e "AI Service embeddings API call failed" {:documents (count texts) :tokens (count-tokens-batch texts)})
(throw e))))
-(defn- ai-service-get-embedding [model-name text]
+(defn- ai-service-get-embedding [model-name text & {:as opts}]
(try
(log/debug "Generating AI Service embedding for text of length:" (count text))
- (first (ai-service-get-embeddings-batch model-name [text]))
+ (first (ai-service-get-embeddings-batch model-name [text] opts))
(catch Exception e
(log/error e "Failed to generate AI Service embedding for text of length:" (count text))
(throw e))))
-(defmethod get-embedding "ai-service" [{:keys [model-name]} text] (ai-service-get-embedding model-name text))
-(defmethod get-embeddings-batch "ai-service" [{:keys [model-name]} texts] (ai-service-get-embeddings-batch model-name texts))
+(defmethod get-embedding "ai-service" [{:keys [model-name]} text & {:as opts}] (ai-service-get-embedding model-name text opts))
+(defmethod get-embeddings-batch "ai-service" [{:keys [model-name]} texts & {:as opts}] (ai-service-get-embeddings-batch model-name texts opts))
;;;; OpenAI impl
@@ -211,7 +215,7 @@
(str/starts-with? model-name "text-embedding-3"))))
(defn- openai-get-embeddings-batch
- [{:keys [model-name vector-dimensions] :as embedding-model} texts]
+ [{:keys [model-name vector-dimensions] :as embedding-model} texts & {:as opts}]
(let [api-key (semantic-settings/openai-api-key)
endpoint (str (semantic-settings/openai-api-base-url) "/v1/embeddings")]
(when-not api-key
@@ -227,25 +231,27 @@
(when (supports-dimensions? embedding-model)
{:dimensions vector-dimensions})))})
:body
- (json/decode true))]
+ (json/decode true))
+ total-tokens (->> response :usage :total_tokens)]
(analytics/inc! :metabase-search/semantic-embedding-tokens
{:provider "openai", :model model-name}
- (->> response :usage :total_tokens))
+ total-tokens)
+ (semantic.models.token-tracking/record-tokens model-name (:type opts) total-tokens)
(extract-base64-response-embeddings response))
(catch Exception e
(log/error e "OpenAI embeddings API call failed" {:documents (count texts) :tokens (count-tokens-batch texts)})
(throw e)))))
-(defn- openai-get-embedding [embedding-model text]
+(defn- openai-get-embedding [embedding-model text & {:as opts}]
(try
(log/debug "Generating OpenAI embedding for text of length:" (count text))
- (first (openai-get-embeddings-batch embedding-model [text]))
+ (first (openai-get-embeddings-batch embedding-model [text] opts))
(catch Exception e
(log/error e "Failed to generate OpenAI embedding for text of length:" (count text))
(throw e))))
-(defmethod get-embedding "openai" [embedding-model text] (openai-get-embedding embedding-model text))
-(defmethod get-embeddings-batch "openai" [embedding-model texts] (openai-get-embeddings-batch embedding-model texts))
+(defmethod get-embedding "openai" [embedding-model text & {:as opts}] (openai-get-embedding embedding-model text opts))
+(defmethod get-embeddings-batch "openai" [embedding-model texts & {:as opts}] (openai-get-embeddings-batch embedding-model texts opts))
(defmethod pull-model "openai" [_] (log/debug "OpenAI provider does not require pulling a model"))
;;;; Global embedding model
@@ -270,7 +276,7 @@
(defn process-embeddings-streaming
"Process texts in provider-appropriate batches, calling process-fn for each batch. process-fn will be called with
a map from text to embedding for each batch."
- [embedding-model texts process-fn]
+ [embedding-model texts process-fn & {:as opts}]
(when (seq texts)
(let [{:keys [model-name provider vector-dimensions]} embedding-model]
(u/profile (str "Generating embeddings " {:model model-name
@@ -284,13 +290,13 @@
(fn [batch-idx batch-texts]
(let [embeddings (u/profile (format "Embedding batch %d/%d %s"
(inc batch-idx) (count batches) (str (calc-token-metrics batch-texts)))
- (openai-get-embeddings-batch embedding-model batch-texts))
+ (openai-get-embeddings-batch embedding-model batch-texts opts))
text-embedding-map (zipmap batch-texts embeddings)]
(process-fn text-embedding-map)))]
(transduce (map-indexed process-batch) (partial merge-with +) batches))
- (let [embeddings (get-embeddings-batch embedding-model texts)
+ (let [embeddings (get-embeddings-batch embedding-model texts opts)
text-embedding-map (zipmap texts embeddings)]
(process-fn text-embedding-map)))))))
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj
index 3313786ae29..fba232ac253 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj
@@ -297,7 +297,7 @@
[(remove found-embeddings texts) found-embeddings]))
(defn- upsert-index-batch!
- [connectable index documents]
+ [connectable index documents & {:as opts}]
(when (seq documents)
(let [text->docs (group-by :searchable_text documents)
searchable-texts (keys text->docs)
@@ -315,7 +315,8 @@
(embedding/process-embeddings-streaming
(:embedding-model index)
new-texts
- upsert-embedding!)))
+ upsert-embedding!
+ opts)))
(merge-with + stats)))))
(def ^:private ^:dynamic *retrying* false)
@@ -351,8 +352,8 @@
(defn upsert-index-pooled!
"Returns a future which upserts the provided documents into the index table, executed using the provided thread pool."
- [pool connectable index documents]
- (cp/future pool (upsert-index-batch! connectable index documents)))
+ [pool connectable index documents & {:as opts}]
+ (cp/future pool (upsert-index-batch! connectable index documents opts)))
(defn upsert-index!
"Inserts or updates documents in the index table. If a document with the same
@@ -364,8 +365,8 @@
results (transduce
(comp (partition-all *batch-size*)
(map (if serial?
- #(upsert-index-batch! connectable index %)
- #(upsert-index-pooled! pool connectable index %))))
+ #(upsert-index-batch! connectable index % {:type :index})
+ #(upsert-index-pooled! pool connectable index % {:type :index}))))
conj
documents-reducible)]
(reduce (fn [update-counts result]
@@ -770,6 +771,11 @@
(analytics/inc! :metabase-search/semantic-collection-filter-ms time-ms)
filtered-docs))))
+(defn- reducible-search-query
+ "Extracted so can be redefd in tests."
+ [db query]
+ (jdbc/plan db (sql-format-quoted query) {:builder-fn jdbc.rs/as-unqualified-lower-maps}))
+
(defn query-index
"Query the index for documents similar to the search string.
Returns a map with :results and :raw-count."
@@ -780,7 +786,7 @@
{:results [] :raw-count 0}
(let [timer (u/start-timer)
- embedding (embedding/get-embedding embedding-model search-string)
+ embedding (embedding/get-embedding embedding-model search-string {:type :query})
embedding-time-ms (u/since-ms timer)
db-timer (u/start-timer)
@@ -789,7 +795,7 @@
query (scored-search-query index embedding search-context scorers)
xform (comp (map decode-metadata)
(map (partial legacy-input-with-score weights (keys scorers))))
- reducible (jdbc/plan db (sql-format-quoted query) {:builder-fn jdbc.rs/as-unqualified-lower-maps})
+ reducible (reducible-search-query db query)
raw-results (into [] xform reducible)
db-query-time-ms (u/since-ms db-timer)
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/init.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/init.clj
index 5a2e6a24bae..30e92d534c0 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/init.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/init.clj
@@ -4,4 +4,5 @@
[metabase-enterprise.semantic-search.task.index-cleanup]
[metabase-enterprise.semantic-search.task.index-repair]
[metabase-enterprise.semantic-search.task.indexer]
- [metabase-enterprise.semantic-search.task.metric-collector]))
+ [metabase-enterprise.semantic-search.task.metric-collector]
+ [metabase-enterprise.semantic-search.task.usage-trimmer]))
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/models/token_tracking.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/models/token_tracking.clj
new file mode 100644
index 00000000000..c4d063f8066
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/models/token_tracking.clj
@@ -0,0 +1,19 @@
+(ns metabase-enterprise.semantic-search.models.token-tracking
+ (:require
+ [metabase.models.interface :as mi]
+ [methodical.core :as methodical]
+ [toucan2.core :as t2]))
+
+(derive :model/SemanticSearchTokenTracking :metabase/model)
+
+(methodical/defmethod t2/table-name :model/SemanticSearchTokenTracking [_model] :semantic_search_token_tracking)
+
+(t2/deftransforms :model/SemanticSearchTokenTracking
+ {:request_type mi/transform-keyword})
+
+(defn record-tokens
+ "Record semantic search token usage."
+ [model request-type total-tokens]
+ (t2/insert! :model/SemanticSearchTokenTracking {:model_name model
+ :request_type request-type
+ :total_tokens total-tokens}))
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/task/usage_trimmer.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/task/usage_trimmer.clj
new file mode 100644
index 00000000000..62f8086ee5b
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/task/usage_trimmer.clj
@@ -0,0 +1,45 @@
+(ns metabase-enterprise.semantic-search.task.usage-trimmer
+ (:require
+ [clojurewerkz.quartzite.jobs :as jobs]
+ [clojurewerkz.quartzite.schedule.cron :as cron]
+ [clojurewerkz.quartzite.triggers :as triggers]
+ [metabase.premium-features.core :as premium-features]
+ [metabase.task.core :as task]
+ [metabase.util.log :as log]
+ [toucan2.core :as t2])
+ (:import
+ (java.sql Timestamp)
+ (org.quartz DisallowConcurrentExecution)))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private trimmer-job-key (jobs/key "metabase.task.semantic-search.usage-trimmer.job"))
+(def ^:private trimmer-trigger-key (triggers/key "metabase.task.semantic-search.usage-trimmer.trigger"))
+
+(def ^:private storage-months 2)
+
+(defn- trim-old-token-data!
+ []
+ (log/info "Attempting to delete old semantic search usage data.")
+ (let [t (Timestamp/valueOf (.minusMonths (java.time.LocalDateTime/now) storage-months))]
+ (t2/delete! :model/SemanticSearchTokenTracking {:where [:< :created_at t]}))
+ (log/info "Semantic search old data cleanup successful."))
+
+(task/defjob ^{DisallowConcurrentExecution true
+ :doc "Clean up inactive semantic search index tables"}
+ SemanticSearchUsageTrimmer [_ctx]
+ (trim-old-token-data!))
+
+(defmethod task/init! ::SemanticSearchUsageTrimmer
+ []
+ (when (premium-features/has-feature? :semantic-search)
+ (let [job (jobs/build
+ (jobs/of-type SemanticSearchUsageTrimmer)
+ (jobs/with-identity trimmer-job-key))
+ trigger (triggers/build
+ (triggers/with-identity trimmer-trigger-key)
+ (triggers/start-now)
+ (triggers/with-schedule
+ ;; daily at 22:59:42
+ (cron/cron-schedule "42 59 22 * * ?")))]
+ (task/schedule-task! job trigger))))
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj
index f740399506f..0ca9bf0922c 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj
@@ -101,6 +101,7 @@
"Sandbox"
"SearchIndexMetadata"
"Secret"
+ "SemanticSearchTokenTracking"
"Session"
"TaskHistory"
"TransformJobRun"
diff --git a/enterprise/backend/test/metabase_enterprise/models/entity_id_test.clj b/enterprise/backend/test/metabase_enterprise/models/entity_id_test.clj
index 9e3758d7997..3ebaf238729 100644
--- a/enterprise/backend/test/metabase_enterprise/models/entity_id_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/models/entity_id_test.clj
@@ -79,6 +79,7 @@
:model/QueryTable
:model/RecentViews
:model/Revision
+ :model/SemanticSearchTokenTracking
:model/SearchIndexMetadata
:model/Secret
:model/Session
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/embedding_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/embedding_test.clj
index 26b2d7fcf9f..81c1d1d2dcc 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/embedding_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/embedding_test.clj
@@ -2,10 +2,20 @@
(:require
[clj-http.client :as http]
[clojure.test :refer :all]
+ [environ.core :as env]
[metabase-enterprise.semantic-search.embedding :as embedding]
+ [metabase-enterprise.semantic-search.env :as semantic.env]
+ [metabase-enterprise.semantic-search.gate :as semantic.gate]
+ [metabase-enterprise.semantic-search.index :as semantic.index]
+ [metabase-enterprise.semantic-search.index-metadata :as semantic.index-metadata]
+ [metabase-enterprise.semantic-search.indexer :as semantic.indexer]
+ [metabase-enterprise.semantic-search.pgvector-api :as semantic.pgvector-api]
+ [metabase-enterprise.semantic-search.settings :as semantic.settings]
+ [metabase-enterprise.semantic-search.test-util :as semantic.tu]
[metabase.analytics.core :as analytics]
[metabase.test :as mt]
- [metabase.util.json :as json])
+ [metabase.util.json :as json]
+ [toucan2.core :as t2])
(:import
[java.nio ByteBuffer ByteOrder]
[java.util Base64]))
@@ -206,6 +216,9 @@
{:provider "ollama"
:mock-response {:embedding mock-embedding}
:counts-tokens? false}]]
+
+ (t2/delete! :model/SemanticSearchTokenTracking)
+
(mt/with-dynamic-fn-redefs [analytics/inc! (fn [metric & args]
(swap! analytics-calls conj [metric args]))
http/post (fn post-mock [_url & _options]
@@ -229,4 +242,54 @@
:model "some-model"}
(-> tokens-calls first second first)))
(is (= (get-in mock-response [:usage :total_tokens])
- (-> tokens-calls first second second)))))))))))
+ (-> tokens-calls first second second)))
+ (is (= 2 (t2/count :model/SemanticSearchTokenTracking)))))))))))
+
+(deftest token-tracking-write-test
+ (mt/with-premium-features #{:semantic-search}
+ (when (string? (not-empty (:mb-pgvector-db-url env/env)))
+ (doseq [provider ["openai" "ai-service"]]
+ (semantic.tu/with-test-db! {:mode :blank}
+ (let [mock-embedding (repeat 1024 1.0)
+ mock-response {:data [{:object "embedding"
+ :embedding (encode-floats-to-base64 mock-embedding)
+ :index 0}]
+ :model "some-model"
+ :usage {:prompt_tokens 1
+ :total_tokens 13}}]
+ (with-redefs [semantic.settings/ee-embedding-provider (constantly provider)
+ semantic.settings/ee-embedding-model (constantly "mock-model")
+ semantic.settings/openai-api-key (constantly "xyz")
+ semantic.settings/openai-api-base-url (constantly "xyz")
+ http/post (fn post-mock [_url & _options]
+ {:status 200
+ :headers {"Content-Type" "application/json"}
+ :body (json/encode mock-response)})]
+ (let [pgvector (semantic.env/get-pgvector-datasource!)
+ index-metadata (semantic.env/get-index-metadata)
+ embedding-model (semantic.env/get-configured-embedding-model)
+ _ (semantic.pgvector-api/init-semantic-search! pgvector index-metadata embedding-model)
+ {:keys [index metadata-row]} (semantic.index-metadata/get-active-index-state pgvector index-metadata)
+ indexing-state (semantic.indexer/init-indexing-state metadata-row)
+ gate-docs (mapv #(semantic.gate/search-doc->gate-doc % (java.sql.Timestamp. 1000))
+ (semantic.tu/mock-documents))]
+
+ (semantic.gate/gate-documents! pgvector index-metadata gate-docs)
+ (t2/delete! :model/SemanticSearchTokenTracking)
+
+ (testing "Indexing tokens are tracked"
+ (semantic.indexer/indexing-step pgvector index-metadata index indexing-state)
+ (is (= 1 (t2/count :model/SemanticSearchTokenTracking)))
+ (let [{:keys [request_type total_tokens]}
+ (t2/select-one :model/SemanticSearchTokenTracking)]
+ (is (= :index request_type))
+ (is (= 13 total_tokens))))
+
+ (testing "Querying tokens are tracked"
+ (t2/delete! :model/SemanticSearchTokenTracking)
+ (semantic.index/query-index pgvector index {:search-string "elephant"})
+ (is (= 1 (t2/count :model/SemanticSearchTokenTracking)))
+ (let [{:keys [request_type total_tokens]}
+ (t2/select-one :model/SemanticSearchTokenTracking)]
+ (is (= :query request_type))
+ (is (= 13 total_tokens))))))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/task/usage_trimmer_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/task/usage_trimmer_test.clj
new file mode 100644
index 00000000000..74fe86e89f0
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/task/usage_trimmer_test.clj
@@ -0,0 +1,32 @@
+(ns metabase-enterprise.semantic-search.task.usage-trimmer-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase-enterprise.semantic-search.task.usage-trimmer :as semantic.task.trimmer]
+ [metabase.test :as mt]
+ [toucan2.core :as t2])
+ (:import
+ (java.sql Timestamp)
+ (java.time LocalDateTime)))
+
+(set! *warn-on-reflection* true)
+
+(deftest trim-test
+ (mt/with-premium-features #{:semantic-search}
+ (t2/delete! :model/SemanticSearchTokenTracking)
+ (let [now (LocalDateTime/now)
+ ;; semantic.task.trimmer/storage-months is set to 2
+ long-ago (.minusMonths now 6)]
+ (t2/insert! :model/SemanticSearchTokenTracking {:model_name "model now"
+ :request_type :index
+ :created_at (Timestamp/valueOf now)
+ :total_tokens 999})
+ (t2/insert! :model/SemanticSearchTokenTracking {:model_name "model long-ago"
+ :request_type :query
+ :created_at (Timestamp/valueOf long-ago)
+ :total_tokens 1000})
+ (is (=? [{:model_name "model now"}
+ {:model_name "model long-ago"}]
+ (t2/select :model/SemanticSearchTokenTracking {:order-by [[:total_tokens :asc]]})))
+ (@#'semantic.task.trimmer/trim-old-token-data!)
+ (is (=? [{:model_name "model now"}]
+ (t2/select :model/SemanticSearchTokenTracking {:order-by [[:total_tokens :asc]]}))))))
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/test_util.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/test_util.clj
index 056ce492781..f93a5f61484 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/test_util.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/test_util.clj
@@ -245,8 +245,9 @@
(-> (semantic.index/default-index mock-embedding-model)
(semantic.index-metadata/qualify-index mock-index-metadata))))
-(defmethod semantic.embedding/get-embedding "mock" [_ text] (get-mock-embedding text))
-(defmethod semantic.embedding/get-embeddings-batch "mock" [_ texts] (get-mock-embeddings-batch texts))
+;; NOTE: opts are currently unused in following mock implementations
+(defmethod semantic.embedding/get-embedding "mock" [_ text & {:as _opts}] (get-mock-embedding text))
+(defmethod semantic.embedding/get-embeddings-batch "mock" [_ texts & {:as _opts}] (get-mock-embeddings-batch texts))
(defmethod semantic.embedding/pull-model "mock" [_])
#_{:clj-kondo/ignore [:metabase/test-helpers-use-non-thread-safe-functions]}
diff --git a/resources/migrations/056_update_migrations.yaml b/resources/migrations/056_update_migrations.yaml
index 17447e091ce..856691def00 100644
--- a/resources/migrations/056_update_migrations.yaml
+++ b/resources/migrations/056_update_migrations.yaml
@@ -504,6 +504,60 @@ databaseChangeLog:
name: conversation_id
rollback: # will be dropped with table
+ - changeSet:
+ id: v56.2025-09-16T18:20:00
+ author: lbrdnk
+ comment: Semantic search token usage tracking
+ changes:
+ - createTable:
+ tableName: semantic_search_token_tracking
+ remarks: Token usage tracking info for semantic search
+ columns:
+ - column:
+ name: id
+ remarks: Unique ID of a request
+ type: int
+ autoIncrement: true
+ constraints:
+ primaryKey: true
+ nullable: false
+ - column:
+ name: model_name
+ remarks: Name of model used for embeddings generation
+ type: varchar(256)
+ constraints:
+ nullable: false
+ - column:
+ name: request_type
+ remarks: Type of request, possibly index or query
+ type: varchar(32)
+ - column:
+ name: created_at
+ remarks: Datetime of insertion
+ type: ${timestamp_type}
+ defaultValueComputed: current_timestamp
+ constraints:
+ nullable: false
+ - column:
+ name: total_tokens
+ remarks: Total tokens value as per OpenAI compatible API
+ type: int
+ constraints:
+ nullable: false
+
+ - changeSet:
+ id: v56.2025-09-19T15:33:18
+ author: lbrdnk
+ comment: Semantic search token usage tracking
+ changes:
+ - createIndex:
+ tableName: semantic_search_token_tracking
+ indexName: idx_semantic_search_token_tracking_created_at
+ columns:
+ column:
+ name: created_at
+
+
# >>>>>>>>>> DO NOT ADD NEW MIGRATIONS BELOW THIS LINE! ADD THEM ABOVE <<<<<<<<<<
########################################################################################################################
diff --git a/src/metabase/models/resolution.clj b/src/metabase/models/resolution.clj
index f2814a9e489..de6480b6022 100644
--- a/src/metabase/models/resolution.clj
+++ b/src/metabase/models/resolution.clj
@@ -85,6 +85,7 @@
:model/SearchIndexMetadata metabase.search.models.search-index-metadata
:model/Secret metabase.secrets.models.secret
:model/Segment metabase.segments.models.segment
+ :model/SemanticSearchTokenTracking metabase-enterprise.semantic-search.models.token-tracking
:model/Session metabase.session.models.session
:model/Setting metabase.settings.models.setting
:model/Table metabase.warehouse-schema.models.table
diff --git a/test/metabase/cmd/copy_test.clj b/test/metabase/cmd/copy_test.clj
index 2725594cf97..67862862661 100644
--- a/test/metabase/cmd/copy_test.clj
+++ b/test/metabase/cmd/copy_test.clj
@@ -39,6 +39,7 @@
:model/QueryField
:model/QueryTable
:model/SearchIndexMetadata
+ :model/SemanticSearchTokenTracking
:model/TaskHistory
;; TODO we should remove these models from here once serialization is supported
:model/Transform