The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Return field details without statistics when sampling fails (#84898)
Andrei Simionescucf7698c6a433
diff --git a/resources/metabot/prompts/llm_shape.selmer b/resources/metabot/prompts/llm_shape.selmerindex 49bc563924b..b33f9f12076 100644--- a/resources/metabot/prompts/llm_shape.selmer+++ b/resources/metabot/prompts/llm_shape.selmer@@ -325,6 +325,11 @@ They provide a rough idea of the data characteristics but may not reflect the co{{stats_table|safe}}{% endif %}+{% if stats_unavailable %}++**Field Statistics**+Metabase couldn't sample this field from the database, so there are no statistics to show. Continue without them.+{% endif %}{% endif %}{% if is_field_metadata %}{% if field_metadata_value_xml %}diff --git a/src/metabase/metabot/tools/field_stats.clj b/src/metabase/metabot/tools/field_stats.cljindex a54bbb750f9..8c35f11331c 100644--- a/src/metabase/metabot/tools/field_stats.clj+++ b/src/metabase/metabot/tools/field_stats.clj@@ -8,24 +8,41 @@[metabase.metabot.tools.util :as metabot.tools.u][metabase.parameters.field-values :as params.field-values][metabase.request.core :as request]- [metabase.sync.core :as sync]))+ [metabase.sync.core :as sync]+ [metabase.sync.util :as sync-util]+ [metabase.util.log :as log]))(defn- build-field-statistics [fvs fp limit](merge- (when fp- {:statistics (-> (or (:global fp) {})- (set/rename-keys {:nil% :percent-null})- (into (vals (:type fp))))})+ (cond+ (= ::unavailable fp) {:statistics_unavailable true}+ fp {:statistics (-> (or (:global fp) {})+ (set/rename-keys {:nil% :percent-null})+ (into (vals (:type fp))))})(when-let [values (-> fvs :values not-empty)]{:field_values (into [] (if limit (take limit) identity) values):field_values_total (count values):has_more_values (:has_more_values fvs)})))-(defn- get-or-create-fingerprint! [{:keys [id fingerprint] :as field}]+(defn- get-or-create-fingerprint!+ "Return the fingerprint of `field`, computing it if needed.++ Return nil if sampling saves no fingerprint, or `::unavailable` if the database can't be sampled."+ [{:keys [id fingerprint] :as field}](or fingerprint- ;; Run with admin perms to match behavior during normal sync.- (and (pos? (:updated-fingerprints (request/as-admin (sync/refingerprint-field! field))))- (metabot.db/field-fingerprint id))))+ (let [result (try+ ;; Run with admin perms to match behavior during normal sync.+ (request/as-admin (sync/refingerprint-field! field))+ (catch Exception e+ (if (sync-util/transient-exception? e)+ (do+ (log/warnf e "Could not fingerprint field %d" id)+ nil)+ (throw e))))]+ ;; Sync returns `:sync/no-connection` or a Throwable when it can't sample the database+ (cond+ (not (map? result)) ::unavailable+ (pos? (:updated-fingerprints result)) (metabot.db/field-fingerprint id)))))(defn- field-statistics"Fingerprint statistics are global (computed across every row of the table), so they're withhelddiff --git a/src/metabase/metabot/tools/shared/llm_shape.clj b/src/metabase/metabot/tools/shared/llm_shape.cljindex dde3f52e4f8..735f8daf150 100644--- a/src/metabase/metabot/tools/shared/llm_shape.clj+++ b/src/metabase/metabot/tools/shared/llm_shape.clj@@ -920,10 +920,10 @@(defn field-values-metadata->xml"Format field values metadata for LLM consumption.- Matches Python FieldValuesMetadata.llm_representation exactly, plus a note on how many values the- field has when only some of them are listed.- Note: Tables are used with |safe in the template, so values must be escaped."- [{:keys [field_values field_values_total has_more_values statistics]}]+ Matches Python FieldValuesMetadata.llm_representation, with notes when only some values are listed+ or when the database couldn't be sampled for statistics.+ Tables use |safe in the template, so values must be escaped."+ [{:keys [field_values field_values_total has_more_values statistics statistics_unavailable]}](let [escape-value (fn [_k v] (escape-pipes (escape-xml (str v))))sample-values-table (when (seq field_values)(te/markdown-table@@ -946,7 +946,8 @@:sample_values_shown shown:sample_values_total (when partial? field_values_total):sample_values_more has_more_values- :stats_table stats-table})))+ :stats_table stats-table+ :stats_unavailable statistics_unavailable})))(defn field-metadata->xml"Format field metadata for LLM consumption.diff --git a/test/metabase/metabot/tools/resources_test.clj b/test/metabase/metabot/tools/resources_test.cljindex ffee1a92dc6..702ef599cb5 100644--- a/test/metabase/metabot/tools/resources_test.clj+++ b/test/metabase/metabot/tools/resources_test.clj@@ -21,6 +21,7 @@[metabase.permissions.core :as perms][metabase.permissions.models.permissions-group :as perms-group][metabase.query-processor :as qp]+ [metabase.sync.core :as sync][metabase.test :as mt][metabase.transforms.core :as transforms.core][toucan2.core :as t2]))@@ -1538,6 +1539,30 @@(is (str/includes? output "| Widget |"))(is (not (str/includes? output "This list shows"))))))))+(deftest read-table-field-whose-database-cannot-be-sampled-test+ (mt/with-current-user (mt/user->id :crowberto)+ (let [field-id (mt/id :orders :total)+ uri (str "metabase://table/" (mt/id :orders) "/fields/" field-id)]+ (mt/with-temp-vals-in-db :model/Field field-id {:fingerprint nil}+ (testing "field details include a note and no statistics when sync can't compute a fingerprint"+ (doseq [[failure refingerprint] {"no connection"+ (constantly :sync/no-connection)+ "a sample query that failed"+ (constantly (ex-info "Table \"ORDERS\" not found" {}))+ "a transient connection error"+ (fn [_] (throw (java.net.ConnectException. "Connection refused")))}]+ (testing failure+ (mt/with-dynamic-fn-redefs [sync/refingerprint-field! refingerprint]+ (let [{:keys [resources output]} (read-uris {:uris [uri]})]+ (is (= {:statistics_unavailable true}+ (get-in resources [0 :content :structured-output :value_metadata])))+ (is (str/includes? output (str "Metabase couldn't sample this field from the database, so"+ " there are no statistics to show."))))))))+ (testing "any other exception fails the read as an internal error"+ (mt/with-dynamic-fn-redefs [sync/refingerprint-field! (fn [_] (throw (NullPointerException.)))]+ (is (= {:class :unrecoverable :code :internal}+ (test-util/tool-failure read-resource/read-resource-tool {:uris [uri]})))))))))+(deftest read-card-question-vs-model-test(mt/with-current-user (mt/user->id :crowberto)(mt/with-temp [:model/Database {db-id :id} {}