Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Tell Metabot when it only sees some of a column's values (#82961)

Andrei Simionescuf09e8fb24adb
diff --git a/enterprise/backend/test/metabase_enterprise/metabot/tools/field_stats_test.clj b/enterprise/backend/test/metabase_enterprise/metabot/tools/field_stats_test.clj
index 157cdafb873..c423c0e4644 100644
--- a/enterprise/backend/test/metabase_enterprise/metabot/tools/field_stats_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/metabot/tools/field_stats_test.clj
@@ -24,6 +24,11 @@
{:entity-type "table", :entity-id table-id, :field-id field-id, :limit 10})]
(testing "returns sandboxed field values"
(is (= ["African" "American"] (get-in result [:structured-output :value_metadata :field_values])))))
+ (testing "a truncated sample counts only the values the sandbox lets the user see"
+ (is (=? {:field_values ["African"] :field_values_total 2}
+ (get-in (metabot.tools.field-stats/field-values
+ {:entity-type "table", :entity-id table-id, :field-id field-id, :limit 1})
+ [:structured-output :value_metadata]))))
(finally
(t2/delete! :model/FieldValues :field_id field-id :type :advanced))))))
diff --git a/resources/metabot/prompts/llm_shape.selmer b/resources/metabot/prompts/llm_shape.selmer
index 020062b9dbf..88d33d201db 100644
--- a/resources/metabot/prompts/llm_shape.selmer
+++ b/resources/metabot/prompts/llm_shape.selmer
@@ -307,7 +307,9 @@ Here are the search results:
The following sample values show the data format in this field. The actual dataset may contain
other values following the same pattern:
-{{sample_values_table|safe}}
+{{sample_values_table|safe}}{% if sample_values_total %}
+
+This list shows {{sample_values_shown}} of the field's {{sample_values_total}}{% if sample_values_more %}+{% endif %} values. The rest can include the value the user asked for, so don't assume it's missing or swap in the closest listed one: when the user asks for a specific value, filter on that value, following the casing the listed values show.{% endif %}
{% else %}
**Sample Values**
This field hasn't been sampled yet. Continue without samples.
diff --git a/src/metabase/metabot/tools/field_stats.clj b/src/metabase/metabot/tools/field_stats.clj
index a4c0515ee7c..a54bbb750f9 100644
--- a/src/metabase/metabot/tools/field_stats.clj
+++ b/src/metabase/metabot/tools/field_stats.clj
@@ -16,8 +16,10 @@
{:statistics (-> (or (:global fp) {})
(set/rename-keys {:nil% :percent-null})
(into (vals (:type fp))))})
- (when-let [fvs (-> fvs :values not-empty)]
- {:field_values (into [] (if limit (take limit) identity) fvs)})))
+ (when-let [values (-> fvs :values not-empty)]
+ {:field_values (into [] (if limit (take limit) identity) values)
+ :field_values_total (count values)
+ :has_more_values (:has_more_values fvs)})))
(defn- get-or-create-fingerprint! [{:keys [id fingerprint] :as field}]
(or fingerprint
diff --git a/src/metabase/metabot/tools/shared/llm_shape.clj b/src/metabase/metabot/tools/shared/llm_shape.clj
index fcdc3582089..b0044f7fb79 100644
--- a/src/metabase/metabot/tools/shared/llm_shape.clj
+++ b/src/metabase/metabot/tools/shared/llm_shape.clj
@@ -892,15 +892,19 @@
(defn field-values-metadata->xml
"Format field values metadata for LLM consumption.
- Matches Python FieldValuesMetadata.llm_representation exactly.
+ Matches Python FieldValuesMetadata.llm_representation exactly, plus a note on how many values the
+ field has when only some of them are listed.
Note: Tables are used with |safe in the template, so values must be escaped."
- [{:keys [field_values statistics]}]
+ [{:keys [field_values field_values_total has_more_values statistics]}]
(let [escape-value (fn [_k v] (escape-pipes (escape-xml (str v))))
sample-values-table (when (seq field_values)
(te/markdown-table
(map vector field_values)
{:value "Value"}
{:value-fn escape-value}))
+ shown (count field_values)
+ partial? (and field_values_total
+ (or has_more_values (< shown field_values_total)))
stats-map (into {} (filter (fn [[_ v]] (some? v)) statistics))
stats-table (when (seq stats-map)
@@ -911,6 +915,9 @@
(render-llm-template
:field_values_metadata
{:sample_values_table sample-values-table
+ :sample_values_shown shown
+ :sample_values_total (when partial? field_values_total)
+ :sample_values_more has_more_values
:stats_table stats-table})))
(defn field-metadata->xml
diff --git a/test/metabase/metabot/tools/resources_test.clj b/test/metabase/metabot/tools/resources_test.clj
index 33f59823a17..2e28e0b4a8b 100644
--- a/test/metabase/metabot/tools/resources_test.clj
+++ b/test/metabase/metabot/tools/resources_test.clj
@@ -1320,6 +1320,20 @@
(#'read-resource/dispatch "metabase://table/3/fields/c75/17")
(is (= ["3" "c75/17"] @calls))))))
+(deftest read-table-field-partial-values-test
+ (mt/with-current-user (mt/user->id :crowberto)
+ (let [read-field (fn [table field]
+ (:output (read-resource/read-resource
+ {:uris [(str "metabase://table/" (mt/id table) "/fields/" (mt/id table field))]})))]
+ (testing "a field with more values than the sample says how many it has"
+ (let [output (read-field :people :state)]
+ (is (str/includes? output "This list shows 30 of the field's 49 values."))
+ (is (str/includes? output "when the user asks for a specific value, filter on that value"))))
+ (testing "a field whose values all fit in the sample doesn't"
+ (let [output (read-field :products :category)]
+ (is (str/includes? output "| Widget |"))
+ (is (not (str/includes? output "This list shows"))))))))
+
(deftest read-card-question-vs-model-test
(mt/with-current-user (mt/user->id :crowberto)
(mt/with-temp [:model/Database {db-id :id} {}
diff --git a/test/metabase/metabot/tools/shared/llm_shape_test.clj b/test/metabase/metabot/tools/shared/llm_shape_test.clj
index c0300298af9..41b4b4c6c2d 100644
--- a/test/metabase/metabot/tools/shared/llm_shape_test.clj
+++ b/test/metabase/metabot/tools/shared/llm_shape_test.clj
@@ -750,7 +750,13 @@
(is (str/includes? xml "This field hasn't been sampled yet"))))
(testing "a pipe in a sample value is escaped so it can't break the table"
(let [xml (llm-shape/field-values-metadata->xml {:field_values ["a|b"]})]
- (is (str/includes? xml "a\\|b")))))
+ (is (str/includes? xml "a\\|b"))))
+ (testing "a partial list says how many values the field has, and a complete one doesn't"
+ (are [value-metadata note] (= note (re-find #"This list shows [^.]+\."
+ (llm-shape/field-values-metadata->xml value-metadata)))
+ {:field_values ["Paris"] :field_values_total 37} "This list shows 1 of the field's 37 values."
+ {:field_values ["Paris"] :field_values_total 1000 :has_more_values true} "This list shows 1 of the field's 1000+ values."
+ {:field_values ["Paris"] :field_values_total 1} nil)))
(deftest ^:parallel field-metadata->xml-test
(testing "formats field metadata"