Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

feat: add remote-sync support for PythonLibrary (#68515)

Edward Paget965ce2a47bfe
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index f766004d44f..00bc4bfbf86 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -2354,8 +2354,10 @@
app-db
config
driver
+ events
lib
lib-be
+ models
query-processor
settings
util
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/events.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/events.clj
index 2bcf4ec3ae1..751f4d2aa90 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/events.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/events.clj
@@ -187,24 +187,6 @@
[id]
(t2/select-one [:model/Collection :name [:id :collection_id]] :id id))
-(defn- track-published-tables-in-collection!
- "When a collection becomes remote-synced, find all published tables in it
- and create 'create' entries for them in RemoteSyncObject (if not already tracked)."
- [collection-id]
- (let [published-tables (t2/select :model/Table
- :collection_id collection-id
- :is_published true)
- table-spec (spec/remote-sync-specs :model/Table)]
- (doseq [table published-tables]
- (let [existing (t2/select-one :model/RemoteSyncObject
- :model_type "Table"
- :model_id (:id table))]
- (when (or (nil? existing)
- (contains? #{"removed" "synced"} (:status existing)))
- (log/infof "Creating remote sync object entry for published table %s in newly remote-synced collection %s"
- (:id table) collection-id)
- (create-or-update-sync-object-from-spec! table-spec (:id table) "create"))))))
-
(defn- handle-library-sync-status-change!
"When the Library collection's is_remote_synced status changes, trigger snippet sync tracking.
This ensures all snippets are tracked/untracked when Library sync is enabled/disabled."
@@ -226,8 +208,6 @@
should-sync? (spec/should-sync-collection? object)
is-remote-synced? (collections/remote-synced-collection? object)
existing-entry (t2/select-one :model/RemoteSyncObject :model_type "Collection" :model_id (:id object))
- was-synced? (and existing-entry
- (not (contains? #{"removed"} (:status existing-entry))))
status (if (:archived object)
"delete"
(case topic
@@ -240,9 +220,7 @@
should-sync?
(do
(log/infof "Creating remote sync object entry for collection %s (status: %s)" (:id object) status)
- (create-or-update-remote-sync-object-entry! "Collection" (:id object) status hydrate-collection-details)
- (when (and is-remote-synced? (not was-synced?))
- (track-published-tables-in-collection! (:id object))))
+ (create-or-update-remote-sync-object-entry! "Collection" (:id object) status hydrate-collection-details))
(and existing-entry (not should-sync?))
(do
(log/infof "Collection %s no longer needs syncing, marking as removed" (:id object))
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
index 1a31ed83281..422587830b8 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
@@ -144,7 +144,7 @@
(log/infof (:message <>)))
(let [path-filters (cond-> [#"collections/.*" #"databases/.*" #"actions/.*"]
(settings/remote-sync-transforms)
- (conj #"transforms/.*")
+ (conj #"transforms/.*" #"python-libraries/.*")
(settings/library-is-remote-synced?)
(conj #"snippets/.*"))
ingestable-snapshot (->> (source.p/->ingestable snapshot {:path-filters path-filters})
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj
index 3000c5229b6..d4a1e8bdaea 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj
@@ -53,11 +53,6 @@
:statuses - Set of statuses to check for removal (e.g., #{\"removed\" \"delete\"})
:scope-key - Optional key for scoping deletions (e.g., :collection_id, :id).
If nil, deletions are global (by entity_id only).
- - :export-path - Export path construction:
- :type - Path type: :collection-entity, :table-path, :field-path, :segment-path,
- :transform-path, :transform-tag-path, :snippet-path, :measure-path
- - :side-effects - Optional side effects:
- :on-first-sync - Function to call when model first becomes synced
- :enabled? - true, or setting keyword (e.g., :remote-sync-transforms, :library-synced).
When :library-synced, uses the library-is-remote-synced? setting."
{:model/Card
@@ -76,7 +71,6 @@
:model_display [:display #(some-> % name)]}}
:removal {:statuses #{"removed"}
:scope-key :collection_id}
- :export-path {:type :collection-entity}
:enabled? true}
:model/Dashboard
@@ -94,7 +88,6 @@
:model_collection_id :collection_id}}
:removal {:statuses #{"removed"}
:scope-key :collection_id}
- :export-path {:type :collection-entity}
:enabled? true}
:model/Document
@@ -112,7 +105,6 @@
:model_collection_id :collection_id}}
:removal {:statuses #{"removed"}
:scope-key :collection_id}
- :export-path {:type :collection-entity}
:enabled? true}
:model/NativeQuerySnippet
@@ -128,7 +120,6 @@
:field-mappings {:model_name :name
:model_collection_id :collection_id}}
:removal {:statuses #{"removed" "delete"}} ; no scope-key = global deletion
- :export-path {:type :snippet-path}
:export-scope :all ; export all snippets
:enabled? :library-synced}
@@ -147,7 +138,6 @@
:model_collection_id :collection_id}}
:removal {:statuses #{"removed"}
:scope-key :collection_id}
- :export-path {:type :collection-entity}
:enabled? true}
:model/Collection
@@ -164,9 +154,7 @@
:model_collection_id :id}} ; collection_id is self
:removal {:statuses #{"removed" "delete"} ; delete is set when collection is archived
:scope-key :id} ; collections are scoped by their own id
- :export-path {:type :collection-entity}
:export-scope :root-collections ; query for root-level remote-synced + transforms-namespace collections
- :side-effects {:on-first-sync :track-published-tables}
:enabled? true}
:model/Table
@@ -184,7 +172,6 @@
:model_table_id :id
:model_table_name :name}} ; self-reference
:removal {:statuses #{"removed"}}
- :export-path {:type :table-path}
:enabled? true}
:model/Field
@@ -208,7 +195,6 @@
:model_table_id :table_id
:model_table_name :table_name}}
:removal {:statuses #{"removed"}}
- :export-path {:type :field-path}
:enabled? true}
:model/Segment
@@ -232,7 +218,6 @@
:model_table_id :table_id
:model_table_name :table_name}}
:removal {:statuses #{"removed" "delete"}} ; Segment has both
- :export-path {:type :segment-path}
:enabled? true}
:model/Measure
@@ -258,7 +243,6 @@
:model_table_id :table_id
:model_table_name :table_name}}
:removal {:statuses #{"removed" "delete"}}
- :export-path {:type :measure-path}
:enabled? true}
:model/Transform
@@ -275,7 +259,6 @@
:field-mappings {:model_name :name
:model_collection_id :collection_id}}
:removal {:statuses #{"removed" "delete"}} ; no scope-key = global deletion
- :export-path {:type :transform-path}
:export-scope :root-only ; query for root transforms (collection_id = nil)
:enabled? :remote-sync-transforms}
@@ -292,7 +275,21 @@
:field-mappings {:model_name :name}}
:removal {:statuses #{"removed" "delete"} ; no scope-key = global deletion
:conditions {:built_in_type nil}} ; protect built-in tags from removal
- :export-path {:type :transform-tag-path}
+ :export-scope :all ; query for all instances
+ :enabled? :remote-sync-transforms}
+
+ :model/PythonLibrary
+ {:model-type "PythonLibrary"
+ :model-key :model/PythonLibrary
+ :identity :entity-id
+ :events {:prefix :event/python-library
+ :types [:create :update]} ; no delete - upsert only
+ :eligibility {:type :setting
+ :setting :remote-sync-transforms}
+ :archived-key nil ; no archived field
+ :tracking {:select-fields [:path]
+ :field-mappings {:model_name :path}}
+ :removal {:statuses #{"removed" "delete"}} ; no scope-key = global deletion
:export-scope :all ; query for all instances
:enabled? :remote-sync-transforms}})
@@ -716,14 +713,12 @@
;; Hybrid models like Segment need a join query for table info
[{:keys [model-type tracking]} entity-ids timestamp]
(when (seq entity-ids)
- (if-let [query-template (:hydrate-query tracking)]
+ (when-let [query-template (:hydrate-query tracking)]
;; Use custom hydrate query adapted for batch lookup
;; Keep the existing from/join structure, just modify select and where
;; The query template uses alias :s for the main model (segment)
(let [base-query (-> query-template
- ;; Add :id to select for model_id field
(update :select (fn [cols] (vec (concat [:s.id] cols))))
- ;; Replace where clause to use batch entity_id lookup
(assoc :where [:in :s.entity_id entity-ids]))]
(->> (t2/query base-query)
(map (fn [entity]
@@ -735,9 +730,7 @@
:model_id (:id entity)
:status "synced"
:status_changed_at timestamp}
- mapped-fields))))))
- ;; Fallback - shouldn't happen for hybrid models with proper spec
- nil)))
+ mapped-fields)))))))))
(defn- build-path-where-clause
"Builds an :or where clause for path-based lookups."
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj
index dfa13110a32..9207a100e32 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj
@@ -35,7 +35,7 @@
(conj "Setting")
(not (:no-transforms opts))
- (conj "Transform" "TransformTag" "TransformJob")))
+ (conj "Transform" "TransformTag" "TransformJob" "PythonLibrary")))
(defn make-targets-of-type
"Returns a targets seq with model type and given ids"
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/ingest.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/ingest.clj
index 410385f7cc7..575ef9b62a8 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/ingest.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/ingest.clj
@@ -74,7 +74,7 @@
read-timestamps))
(def legal-top-level-paths "Known top-level paths for directory with serialization output"
- #{"actions" "collections" "databases" "snippets" "glossary" "transforms"}) ; But return the hierarchy without labels.
+ #{"actions" "collections" "databases" "glossary" "python-libraries" "snippets" "transforms"})
(defn- ingest-all [^File root-dir]
;; This returns a map {unlabeled-hierarchy [original-hierarchy File]}.
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj
index 8c388e88567..f215ea54f85 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj
@@ -27,8 +27,7 @@
"Document"
"Glossary"
"NativeQuerySnippet"
- "Timeline"
- "Transform"])
+ "Timeline"])
(def exported-models
"The list of all models exported by serialization by default. Used for production code and by tests."
@@ -36,7 +35,9 @@
content
["FieldValues"
"Metabot"
+ "PythonLibrary"
"Setting"
+ "Transform"
"TransformJob"
"TransformTag"]))
@@ -96,7 +97,6 @@
"PermissionsGroupMembership"
"PermissionsRevision"
"PersistedInfo"
- "PythonLibrary"
"Pulse"
"PulseCard"
"PulseChannel"
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/models/python_library.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/models/python_library.clj
index 6b3f7c63edb..703477e47df 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms_python/models/python_library.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/models/python_library.clj
@@ -1,6 +1,8 @@
(ns metabase-enterprise.transforms-python.models.python-library
(:require
[metabase.app-db.core :as app-db]
+ [metabase.events.core :as events]
+ [metabase.models.serialization :as serdes]
[metabase.util.i18n :refer [tru]]
[methodical.core :as methodical]
[toucan2.core :as t2]))
@@ -26,7 +28,7 @@
(update library :path normalize-path)
library))
-(doseq [trait [:metabase/model :hook/timestamped?]]
+(doseq [trait [:metabase/model :hook/timestamped? :hook/entity-id]]
(derive :model/PythonLibrary trait))
(def ^:private allowed-paths
@@ -59,3 +61,35 @@
{:path normalized-path}
(constantly {:path normalized-path :source source}))]
(t2/select-one :model/PythonLibrary id))))
+
+;;; ------------------------------------------------- Serialization --------------------------------------------------
+
+(defmethod serdes/make-spec "PythonLibrary"
+ [_model-name _opts]
+ {:copy [:path :source :entity_id]
+ :transform {:created_at (serdes/date)}})
+
+(defmethod serdes/hash-fields :model/PythonLibrary
+ [_model]
+ [:path])
+
+(defmethod serdes/storage-path "PythonLibrary" [entity _ctx]
+ (let [{:keys [id label]} (-> entity serdes/path last)]
+ ["python-libraries" (serdes/storage-leaf-file-name id label)]))
+
+;;; ------------------------------------------------ Event Hooks -----------------------------------------------------
+
+;; Event type hierarchy for remote-sync tracking
+(derive ::event :metabase/event)
+(doseq [e [:event/python-library-create :event/python-library-update :event/python-library-delete]]
+ (derive e ::event))
+
+(t2/define-after-insert :model/PythonLibrary
+ [library]
+ (events/publish-event! :event/python-library-create {:object library})
+ library)
+
+(t2/define-after-update :model/PythonLibrary
+ [library]
+ (events/publish-event! :event/python-library-update {:object library})
+ library)
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/events_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/events_test.clj
index b46a40eb9a7..0175ae0e006 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/events_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/events_test.clj
@@ -935,43 +935,6 @@
(is (isa? :event/table-update ::remote-sync.events/table-change-event))
(is (isa? :event/table-delete ::remote-sync.events/table-change-event))))
-(deftest collection-becomes-remote-synced-tracks-existing-tables-test
- (testing "when collection becomes remote-synced, existing published tables are tracked with 'create' status"
- (mt/with-temp [:model/Collection collection {:name "Soon Remote-Sync"}
- :model/Table table {:name "Test Table"
- :is_published true
- :collection_id (:id collection)}]
- (t2/delete! :model/RemoteSyncObject)
- ;; Collection is not yet remote-synced, fire update event
- (events/publish-event! :event/collection-update
- {:object collection :user-id (mt/user->id :rasta)})
- (is (= 0 (count (t2/select :model/RemoteSyncObject))))
- ;; Now make collection remote-synced
- (events/publish-event! :event/collection-update
- {:object (assoc collection :is_remote_synced true)
- :user-id (mt/user->id :rasta)})
- (let [table-entries (t2/select :model/RemoteSyncObject :model_type "Table")
- collection-entries (t2/select :model/RemoteSyncObject :model_type "Collection")]
- ;; Should have both collection and table entries
- (is (= 1 (count collection-entries)))
- (is (= 1 (count table-entries)))
- (is (=? {:model_type "Table"
- :model_id (:id table)
- :status "create"}
- (first table-entries))))))
- (testing "unpublished tables are not tracked when collection becomes remote-synced"
- (mt/with-temp [:model/Collection collection {:name "Soon Remote-Sync"}
- :model/Table _ {:name "Test Table"
- :is_published false
- :collection_id (:id collection)}]
- (t2/delete! :model/RemoteSyncObject)
- ;; Make collection remote-synced
- (events/publish-event! :event/collection-update
- {:object (assoc collection :is_remote_synced true)
- :user-id (mt/user->id :rasta)})
- (let [table-entries (t2/select :model/RemoteSyncObject :model_type "Table")]
- (is (= 0 (count table-entries)))))))
-
;;; Segment Event Tests
(deftest segment-create-event-creates-entry-test
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/snippets_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/snippets_test.clj
index 1dbe7033473..ff790f27884 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/snippets_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/snippets_test.clj
@@ -13,7 +13,7 @@
[metabase-enterprise.remote-sync.spec :as spec]
[metabase-enterprise.remote-sync.test-helpers :as test-helpers]
[metabase.collections.models.collection :as collection]
- [metabase.collections.test-utils :refer [with-library-not-synced with-library-synced]]
+ [metabase.collections.test-utils :as collections.tu]
[metabase.events.core :as events]
[metabase.search.core :as search]
[metabase.test :as mt]
@@ -41,7 +41,7 @@
(deftest snippet-event-creates-sync-object-when-library-synced-test
(testing "Creating a snippet creates a RemoteSyncObject entry when Library is remote-synced"
- (with-library-synced
+ (collections.tu/with-library-synced
(mt/with-temporary-setting-values [remote-sync-enabled true]
(mt/with-temp [:model/NativeQuerySnippet snippet {:name "Test Snippet" :content "SELECT 1"}]
(events/publish-event! :event/snippet-create {:object snippet :user-id (mt/user->id :rasta)})
@@ -52,7 +52,7 @@
(deftest snippet-event-ignored-when-library-not-synced-test
(testing "Snippet events are ignored when Library is not remote-synced"
- (with-library-not-synced
+ (collections.tu/with-library-not-synced
(mt/with-temporary-setting-values [remote-sync-enabled true]
(mt/with-temp [:model/NativeQuerySnippet snippet {:name "Test Snippet" :content "SELECT 1"}]
(events/publish-event! :event/snippet-create {:object snippet :user-id (mt/user->id :rasta)})
@@ -63,7 +63,7 @@
(deftest snippet-event-updates-sync-object-test
(testing "Updating a snippet updates the RemoteSyncObject entry when Library is synced"
- (with-library-synced
+ (collections.tu/with-library-synced
(mt/with-temporary-setting-values [remote-sync-enabled true]
(mt/with-temp [:model/NativeQuerySnippet snippet {:name "Test Snippet" :content "SELECT 1"}]
;; Delete auto-created entry from snippet creation event and set up test state
@@ -82,7 +82,7 @@
(deftest archived-snippet-marked-for-deletion-test
(testing "Archiving a snippet marks it for deletion when Library is synced"
- (with-library-synced
+ (collections.tu/with-library-synced
(mt/with-temporary-setting-values [remote-sync-enabled true]
(mt/with-temp [:model/NativeQuerySnippet snippet {:name "Test Snippet" :content "SELECT 1"}]
;; Delete auto-created entry from snippet creation event and set up test state
@@ -104,7 +104,7 @@
(deftest snippets-namespace-collection-tracked-when-library-synced-test
(testing "Snippets-namespace collection creates RemoteSyncObject entry when Library is synced"
- (with-library-synced
+ (collections.tu/with-library-synced
(mt/with-temporary-setting-values [remote-sync-enabled true]
(mt/with-temp [:model/Collection coll {:name "Snippets Collection" :namespace :snippets :location "/"}]
(events/publish-event! :event/collection-create {:object coll :user-id (mt/user->id :rasta)})
@@ -115,7 +115,7 @@
(deftest snippets-namespace-collection-ignored-when-library-not-synced-test
(testing "Snippets-namespace collection is NOT tracked when Library is not synced"
- (with-library-not-synced
+ (collections.tu/with-library-not-synced
(mt/with-temporary-setting-values [remote-sync-enabled true]
(mt/with-temp [:model/Collection coll {:name "Snippets Collection" :namespace :snippets :location "/"}]
(events/publish-event! :event/collection-create {:object coll :user-id (mt/user->id :rasta)})
@@ -128,7 +128,7 @@
(deftest enable-snippet-sync-marks-all-snippets-test
(testing "Enabling snippet sync (via Library becoming synced) marks all existing snippets for initial sync"
- (with-library-not-synced
+ (collections.tu/with-library-not-synced
(mt/with-temporary-setting-values [remote-sync-enabled true]
(mt/with-temp [:model/Collection {coll-id :id} {:name "Snippets Collection" :namespace :snippets}
:model/NativeQuerySnippet snippet {:name "Existing Snippet" :content "SELECT 1" :collection_id coll-id}]
@@ -150,7 +150,7 @@
(deftest disable-snippet-sync-removes-all-tracking-test
(testing "Disabling snippet sync removes all snippet-related tracking entries"
- (with-library-not-synced
+ (collections.tu/with-library-not-synced
(mt/with-temporary-setting-values [remote-sync-enabled true]
(mt/with-temp [:model/Collection {coll-id :id} {:name "Snippets Collection" :namespace :snippets}
:model/NativeQuerySnippet snippet {:name "Test Snippet" :content "SELECT 1" :collection_id coll-id}
@@ -180,7 +180,7 @@
(deftest snippets-included-in-dirty-check-when-library-synced-test
(testing "Snippets are included in dirty check when Library is synced"
- (with-library-synced
+ (collections.tu/with-library-synced
(mt/with-temporary-setting-values [remote-sync-enabled true]
(mt/with-temp [:model/NativeQuerySnippet snippet {:name "Dirty Snippet" :content "SELECT 1"}]
;; Delete auto-created entry and insert with specific status
@@ -195,7 +195,7 @@
(deftest snippets-excluded-from-dirty-check-when-library-not-synced-test
(testing "Snippets are excluded from dirty check when Library is not synced"
- (with-library-not-synced
+ (collections.tu/with-library-not-synced
(mt/with-temporary-setting-values [remote-sync-enabled true]
(mt/with-temp [:model/NativeQuerySnippet snippet {:name "Dirty Snippet" :content "SELECT 1"}
:model/RemoteSyncObject _rso {:model_type "NativeQuerySnippet"
@@ -242,10 +242,10 @@
(deftest library-is-remote-synced-helper-test
(testing "library-is-remote-synced? returns correct value"
- (with-library-synced
+ (collections.tu/with-library-synced
(is (true? (settings/library-is-remote-synced?))
"Should return true when Library is synced"))
- (with-library-not-synced
+ (collections.tu/with-library-not-synced
(is (false? (settings/library-is-remote-synced?))
"Should return false when Library is not synced"))))
@@ -298,7 +298,7 @@ serdes/meta:
(deftest export-includes-snippets-when-library-synced-test
(testing "Export includes snippets when Library is synced"
- (with-library-synced
+ (collections.tu/with-library-synced
(mt/with-model-cleanup [:model/RemoteSyncTask]
(mt/with-temporary-setting-values [remote-sync-type :read-write
remote-sync-enabled true]
@@ -338,7 +338,7 @@ serdes/meta:
(testing "Import brings in snippets from YAML files"
(mt/with-model-cleanup [:model/NativeQuerySnippet :model/Collection :model/RemoteSyncTask]
(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]
- (with-library-synced
+ (collections.tu/with-library-synced
(mt/with-temporary-setting-values [remote-sync-enabled true]
(let [coll-entity-id "snippets-coll-importx"
snippet-entity-id "test-snippet-importxx"
@@ -360,14 +360,14 @@ serdes/meta:
(deftest all-syncable-collection-ids-includes-all-namespace-types-test
(testing "Snippet collections are included when Library is synced"
- (with-library-synced
+ (collections.tu/with-library-synced
(mt/with-temporary-setting-values [remote-sync-enabled true]
(mt/with-temp [:model/Collection {snippet-coll-id :id} {:name "Snippets Collection"
:namespace :snippets
:location "/"}]
(is (contains? (set (spec/all-syncable-collection-ids)) snippet-coll-id))))))
(testing "Snippet collections are NOT included when Library is not synced"
- (with-library-not-synced
+ (collections.tu/with-library-not-synced
(mt/with-temporary-setting-values [remote-sync-enabled true]
(mt/with-temp [:model/Collection {snippet-coll-id :id} {:name "Snippets Collection"
:namespace :snippets
@@ -396,7 +396,7 @@ serdes/meta:
(deftest export-deletes-archived-snippet-files-test
(testing "export! deletes files from source for snippets with 'delete' status (archived snippets)"
- (with-library-synced
+ (collections.tu/with-library-synced
(mt/with-model-cleanup [:model/RemoteSyncTask]
(mt/with-temporary-setting-values [remote-sync-type :read-write
remote-sync-enabled true]
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/spec_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/spec_test.clj
index a21369d3e0a..5ed4a09ab0a 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/spec_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/spec_test.clj
@@ -12,7 +12,7 @@
(deftest all-specs-have-required-keys-test
(testing "Every spec has all required keys"
(let [required-keys #{:model-type :model-key :identity :events :eligibility
- :archived-key :tracking :removal :export-path :enabled?}]
+ :archived-key :tracking :removal :enabled?}]
(doseq [[model-key spec] spec/remote-sync-specs]
(testing (str "Spec for " model-key)
(let [missing-keys (set/difference required-keys (set (keys spec)))]
@@ -78,15 +78,6 @@
(is (keyword? scope-key)
"removal :scope-key should be a keyword when present"))))))
-(deftest all-specs-have-valid-export-path-test
- (testing "Every spec has valid export-path configuration"
- (let [valid-path-types #{:collection-entity :table-path :field-path
- :segment-path :measure-path :transform-path :transform-tag-path :snippet-path}]
- (doseq [[model-key spec] spec/remote-sync-specs]
- (testing (str "Spec for " model-key)
- (is (contains? valid-path-types (get-in spec [:export-path :type]))
- (str "Invalid export-path type: " (get-in spec [:export-path :type]))))))))
-
;;; ------------------------------------------------ Helper Function Tests ---------------------------------------------
(deftest spec-for-model-type-test
@@ -116,7 +107,7 @@
(is (contains? types "Measure"))
(is (contains? types "Transform"))
(is (contains? types "TransformTag"))
- (is (= 12 (count types))))))
+ (is (= 13 (count types))))))
(deftest specs-by-identity-type-test
(testing "specs-by-identity-type filters correctly"
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj
index d63dd63c754..b215946f852 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj
@@ -521,3 +521,166 @@ is_sample: false
"Local transforms collection should be deleted after import")
(is (t2/exists? :model/Collection :entity_id remote-coll-entity-id :namespace "transforms")
"Remote transforms collection should be imported")))))))))
+
+;;; ------------------------------------------- PythonLibrary Tests -------------------------------------------
+
+(defn- generate-python-library-yaml
+ "Generates YAML content for a PythonLibrary."
+ [entity-id path source]
+ (format "path: %s
+source: |
+%s
+entity_id: %s
+created_at: '2024-08-28T09:46:18.671622Z'
+serdes/meta:
+- id: %s
+ model: PythonLibrary
+"
+ path
+ (str/join "\n" (map #(str " " %) (str/split-lines source)))
+ entity-id
+ entity-id))
+
+(defn- with-clean-python-library
+ "Ensures PythonLibrary table is clean before and after running the test function.
+ Saves and restores any existing entries."
+ [f]
+ (let [existing (t2/select :model/PythonLibrary)]
+ (try
+ (t2/delete! :model/PythonLibrary)
+ (f)
+ (finally
+ (t2/delete! :model/PythonLibrary)
+ (when (seq existing)
+ (t2/insert! :model/PythonLibrary existing))))))
+
+(deftest python-library-event-creates-sync-object-when-setting-enabled-test
+ (testing "Creating a PythonLibrary creates a RemoteSyncObject entry when remote-sync-transforms is enabled"
+ (with-clean-python-library
+ (fn []
+ (mt/with-premium-features #{:transforms}
+ (mt/with-temporary-setting-values [remote-sync-transforms true
+ remote-sync-enabled true]
+ (let [library (t2/insert-returning-instance! :model/PythonLibrary {:path "common.py" :source "# test"})]
+ (is (t2/exists? :model/RemoteSyncObject
+ :model_type "PythonLibrary"
+ :model_id (:id library))
+ "PythonLibrary should be tracked when remote-sync-transforms is enabled"))))))))
+
+(deftest python-library-event-ignored-when-setting-disabled-test
+ (testing "PythonLibrary events are ignored when remote-sync-transforms is disabled"
+ (with-clean-python-library
+ (fn []
+ (mt/with-premium-features #{:transforms}
+ (mt/with-temporary-setting-values [remote-sync-transforms false
+ remote-sync-enabled true]
+ (let [library (t2/insert-returning-instance! :model/PythonLibrary {:path "common.py" :source "# test"})]
+ (is (not (t2/exists? :model/RemoteSyncObject
+ :model_type "PythonLibrary"
+ :model_id (:id library)))
+ "PythonLibrary should NOT be tracked when remote-sync-transforms is disabled"))))))))
+
+(deftest python-library-event-updates-sync-object-test
+ (testing "Updating a PythonLibrary updates the RemoteSyncObject entry when setting is enabled"
+ (with-clean-python-library
+ (fn []
+ (mt/with-premium-features #{:transforms}
+ (mt/with-temporary-setting-values [remote-sync-transforms true
+ remote-sync-enabled true]
+ (let [library (t2/insert-returning-instance! :model/PythonLibrary {:path "common.py" :source "# test"})]
+ (t2/update! :model/RemoteSyncObject {:model_type "PythonLibrary" :model_id (:id library)}
+ {:status "synced"})
+ (t2/update! :model/PythonLibrary (:id library) {:source "# updated"})
+ (let [entry (t2/select-one :model/RemoteSyncObject
+ :model_type "PythonLibrary"
+ :model_id (:id library))]
+ (is (= "update" (:status entry))
+ "PythonLibrary should have 'update' status after modification")))))))))
+
+(deftest export-includes-python-library-when-setting-enabled-test
+ (testing "Export includes PythonLibrary when remote-sync-transforms is enabled"
+ (with-clean-python-library
+ (fn []
+ (mt/with-premium-features #{:transforms}
+ (mt/with-temporary-setting-values [remote-sync-type :read-write
+ remote-sync-transforms true
+ remote-sync-enabled true]
+ (mt/with-model-cleanup [:model/RemoteSyncTask]
+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})
+ library (t2/insert-returning-instance! :model/PythonLibrary {:path "common.py" :source "# shared code"})]
+ (t2/update! :model/RemoteSyncObject {:model_type "PythonLibrary" :model_id (:id library)}
+ {:status "create"})
+ (let [mock-source (test-helpers/create-mock-source)
+ result (impl/export! (source.p/snapshot mock-source) task-id "Test export")]
+ (is (= :success (:status result))
+ (str "Export should succeed. Result: " result))
+ (let [files-after-export (get @(:files-atom mock-source) "main")]
+ (is (some #(str/includes? % "python-libraries/") (keys files-after-export))
+ "Export should include the PythonLibrary file")))))))))))
+
+(deftest import-python-library-from-yaml-test
+ (testing "Import brings in PythonLibrary from YAML files"
+ (with-clean-python-library
+ (fn []
+ (mt/with-premium-features #{:transforms}
+ (mt/with-temporary-setting-values [remote-sync-transforms true
+ remote-sync-enabled true]
+ (mt/with-model-cleanup [:model/RemoteSyncTask]
+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})
+ lib-entity-id (u/generate-nano-id)
+ test-files {"main" {(str "python-libraries/" lib-entity-id ".yaml")
+ (generate-python-library-yaml lib-entity-id "common.py" "def shared_func():\n return 42")}}
+ mock-source (test-helpers/create-mock-source :initial-files test-files)
+ result (impl/import! (source.p/snapshot mock-source) task-id)]
+ (is (= :success (:status result))
+ (str "Import should succeed. Result: " result))
+ (is (t2/exists? :model/PythonLibrary :path "common.py")
+ "PythonLibrary should be imported")
+ (when-let [library (t2/select-one :model/PythonLibrary :path "common.py")]
+ (is (str/includes? (:source library) "def shared_func()")
+ "PythonLibrary source should be imported correctly"))))))))))
+
+(deftest import-removes-python-library-not-on-remote-test
+ (testing "Import removes local PythonLibrary that doesn't exist on the remote"
+ (with-clean-python-library
+ (fn []
+ (mt/with-premium-features #{:transforms}
+ (mt/with-temporary-setting-values [remote-sync-transforms true
+ remote-sync-enabled true]
+ (mt/with-model-cleanup [:model/RemoteSyncTask]
+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})
+ local-library (t2/insert-returning-instance! :model/PythonLibrary {:path "common.py" :source "# local only"})]
+ (is (t2/exists? :model/PythonLibrary :id (:id local-library))
+ "Local PythonLibrary should exist before import")
+ (let [test-files {"main" {}}
+ mock-source (test-helpers/create-mock-source :initial-files test-files)
+ result (impl/import! (source.p/snapshot mock-source) task-id)]
+ (is (= :success (:status result))
+ (str "Import should succeed. Result: " result))
+ (is (not (t2/exists? :model/PythonLibrary :id (:id local-library)))
+ "Local PythonLibrary should be deleted after import since it wasn't on remote"))))))))))
+
+(deftest import-replaces-python-library-with-remote-version-test
+ (testing "Import updates local PythonLibrary when remote has same entity_id"
+ (with-clean-python-library
+ (fn []
+ (mt/with-premium-features #{:transforms}
+ (mt/with-temporary-setting-values [remote-sync-transforms true
+ remote-sync-enabled true]
+ (mt/with-model-cleanup [:model/RemoteSyncTask]
+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})
+ local-library (t2/insert-returning-instance! :model/PythonLibrary {:path "common.py" :source "# local version"})
+ local-entity-id (:entity_id local-library)]
+ (is (t2/exists? :model/PythonLibrary :path "common.py")
+ "Local PythonLibrary should exist before import")
+ ;; Use the same entity_id as the local library so it gets updated
+ (let [test-files {"main" {(str "python-libraries/" local-entity-id ".yaml")
+ (generate-python-library-yaml local-entity-id "common.py" "# remote version\ndef new_func():\n pass")}}
+ mock-source (test-helpers/create-mock-source :initial-files test-files)
+ result (impl/import! (source.p/snapshot mock-source) task-id)]
+ (is (= :success (:status result))
+ (str "Import should succeed. Result: " result))
+ (let [library (t2/select-one :model/PythonLibrary :path "common.py")]
+ (is (some? library) "PythonLibrary should still exist")
+ (is (str/includes? (:source library) "# remote version")
+ "PythonLibrary source should be updated with remote version")))))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj
index ef89aa6e201..3b996b0471e 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj
@@ -38,7 +38,8 @@
#"/snippets/(.*)\.yaml" :snippet
#"/databases/.*/schemas/(.*)" :schema
#"/databases/(.*)\.yaml" :database
- #"/transforms/(.*)\.yaml" :transform])
+ #"/transforms/(.*)\.yaml" :transform
+ #"/python-libraries/(.*)\.yaml" :python-library])
(defn- file-type
"Find out entity type by file path"
@@ -121,13 +122,13 @@
(testing "API respects parameters"
(let [f (mt/user-http-request :crowberto :post 200 "ee/serialization/export" {}
:all_collections false :data_model false :settings true)]
- (is (= #{:log :dir :settings :transform}
+ (is (= #{:log :dir :settings :transform :python-library}
(tar-file-types f)))))
(testing "We can export just a single collection"
(let [f (mt/user-http-request :crowberto :post 200 "ee/serialization/export" {}
:collection (:id coll) :data_model false :settings false)]
- (is (= #{:log :dir :dashboard :card :collection :transform}
+ (is (= #{:log :dir :dashboard :card :collection :transform :python-library}
(tar-file-types f)))))
(testing "We can export two collections"
@@ -143,18 +144,18 @@
(let [f (mt/user-http-request :crowberto :post 200 "ee/serialization/export" {}
;; eid:... syntax is kept for backward compat
:collection (str "eid:" (:entity_id coll)) :data_model false :settings false)]
- (is (= #{:log :dir :dashboard :card :collection :transform}
+ (is (= #{:log :dir :dashboard :card :collection :transform :python-library}
(tar-file-types f)))))
(testing "We can export that collection using entity id"
(let [f (mt/user-http-request :crowberto :post 200 "ee/serialization/export" {}
:collection (:entity_id coll) :data_model false :settings false)]
- (is (= #{:log :dir :dashboard :card :collection :transform}
+ (is (= #{:log :dir :dashboard :card :collection :transform :python-library}
(tar-file-types f)))))
(testing "Default export: all-collections, data-model, settings"
(let [f (mt/user-http-request :crowberto :post 200 "ee/serialization/export" {})]
- (is (= #{:transform :log :dir :dashboard :card :collection :settings :schema :database}
+ (is (= #{:transform :log :dir :dashboard :card :collection :settings :schema :database :python-library}
(tar-file-types f)))))
(testing "On exception API returns log"
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/cmd_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/cmd_test.clj
index aaf8a1ec741..807658a875f 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/cmd_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/cmd_test.clj
@@ -45,7 +45,7 @@
"settings" true
"field_values" false
"duration_ms" pos?
- "count" 11
+ "count" 12
"source" "cli"
"secrets" false
"success" true
@@ -60,8 +60,8 @@
"direction" "import"
"duration_ms" pos?
"source" "cli"
- "models" "Card,Collection,Setting,TransformJob,TransformTag"
- "count" 11
+ "models" "Card,Collection,PythonLibrary,Setting,TransformJob,TransformTag"
+ "count" 12
"success" true
"error_message" nil}
(-> (snowplow-test/pop-event-data-and-user-id!) first :data))))
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj
index c658b050f02..78877a5c9b1 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj
@@ -100,7 +100,6 @@
:model/TransformJobRun
:model/TransformJobTransformTag
:model/TransformTransformTag
- :model/PythonLibrary
:model/Undo
:model/User
:model/UserParameterValue
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj
index 783346bde1e..efa01bc64cf 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj
@@ -1553,7 +1553,7 @@
:dimension dimension}}}}})}}}]
(testing "selecting a collection includes settings metabot and data model by default"
- (is (= #{"Card" "Collection" "Dashboard" "Database" "Setting" "TransformTag" "TransformJob"}
+ (is (= #{"Card" "Collection" "Dashboard" "Database" "PythonLibrary" "Setting" "TransformTag" "TransformJob"}
(->> (extract/extract {:targets [["Collection" coll1-id]]})
(map (comp :model first serdes/path))
set))))
@@ -2566,3 +2566,23 @@
(is (contains? segment-ids segment-id)))
(testing "fields from non-published table are NOT exported"
(is (= 2 (count field-ids)))))))))
+
+(deftest python-library-test
+ (mt/with-empty-h2-app-db!
+ ;; Delete any pre-existing python_library entries from migrations
+ (t2/delete! :model/PythonLibrary)
+ (ts/with-temp-dpc [:model/PythonLibrary {lib-id :id lib-entity-id :entity_id} {:path "common"
+ :source "def helper():\n return 42"}]
+ (testing "python library extraction"
+ (let [ser (serdes/extract-one "PythonLibrary" {} (t2/select-one :model/PythonLibrary :id lib-id))]
+ (is (=? {:serdes/meta [{:model "PythonLibrary"
+ :id lib-entity-id}]
+ :path "common.py"
+ :source "def helper():\n return 42"
+ :entity_id lib-entity-id
+ :created_at string?}
+ ser))
+ (is (not (contains? ser :id)))
+
+ (testing "has no dependencies"
+ (is (empty? (serdes/dependencies ser)))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/storage_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/storage_test.clj
index b3a8ce02916..af96f87dfcf 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/storage_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/storage_test.clj
@@ -27,7 +27,7 @@
(mt/with-empty-h2-app-db!
(ts/with-temp-dpc [:model/Collection parent {:name "Some Collection"}
:model/Collection child {:name "Child Collection" :location (format "/%d/" (:id parent))}]
- (let [export (into [] (extract/extract nil))
+ (let [export (into [] (extract/extract {:no-transforms true}))
parent-filename (format "%s_some_collection" (:entity_id parent))
child-filename (format "%s_child_collection" (:entity_id child))]
(storage/store! export dump-dir)
@@ -72,7 +72,7 @@
:model/Card c3 {:name "parent card" :collection_id (:id parent)}
:model/Card c4 {:name "child card" :collection_id (:id child)}
:model/Dashboard d1 {:name "parent dash" :collection_id (:id parent)}]
- (let [export (into [] (extract/extract {}))]
+ (let [export (into [] (extract/extract {:no-transforms true}))]
(storage/store! export dump-dir)
(testing "the right files in the right places"
(let [gp-dir (str (:entity_id grandparent) "_grandparent_collection")
@@ -105,7 +105,8 @@
:model/NativeQuerySnippet c3 {:name "parent snippet" :collection_id (:id parent)}
:model/NativeQuerySnippet c4 {:name "child snippet" :collection_id (:id child)}]
(let [export (into [] (extract/extract {:no-settings true
- :no-data-model true}))]
+ :no-data-model true
+ :no-transforms true}))]
(storage/store! export dump-dir)
(let [gp-dir (str (:entity_id grandparent) "_grandparent_collection")
p-dir (str (:entity_id parent) "_parent_collection")
@@ -237,12 +238,24 @@
["parent.child.yaml"]}
(file-set (io/file dump-dir "databases" "mydb" "tables" "table" "fields")))))))))
+(deftest python-library-storage-test
+ (ts/with-random-dump-dir [dump-dir "serdesv2-"]
+ (mt/with-empty-h2-app-db!
+ (t2/delete! :model/PythonLibrary)
+ (ts/with-temp-dpc [:model/PythonLibrary lib {:path "common" :source "def test(): pass"}]
+ (let [export (into [] (extract/extract {:no-settings true :no-data-model true}))]
+ (storage/store! export dump-dir)
+ (testing "python library stored at top-level python-libraries/"
+ (is (= #{[(str (:entity_id lib) ".yaml")]}
+ (file-set (io/file dump-dir "python-libraries"))))))))))
+
(deftest name-too-long-test
(ts/with-random-dump-dir [dump-dir "serdesv2-"]
;; that's a char that takes 3 bytes in utf-8
(ts/with-temp-dpc [:model/Card card {:name (str/join (repeat 100 "ป"))}]
(let [export (into [] (extract/extract {:no-settings true
:no-data-model true
+ :no-transforms true
:targets [["Card" (:id card)]]}))
;; 66 is 'char-count * max-bytes / byte-count'
card-filename (format "%s_%s" (:entity_id card) (str/join (repeat 66 "ป")))]
diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-has-transform-dirty-changes.ts b/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-has-transform-dirty-changes.ts
index 8e3aaefa129..fe4ed106413 100644
--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-has-transform-dirty-changes.ts
+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-has-transform-dirty-changes.ts
@@ -35,10 +35,14 @@ export function useHasTransformDirtyChanges(): boolean {
);
// Check if any dirty entity is:
- // 1. A transform or transform tag
+ // 1. A transform, transform tag, or Python library
// 2. A collection in the transforms namespace
return dirty.some((entity) => {
- if (entity.model === "transform" || entity.model === "transformtag") {
+ if (
+ entity.model === "transform" ||
+ entity.model === "transformtag" ||
+ entity.model === "pythonlibrary"
+ ) {
return true;
}
if (
diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-has-transform-dirty-changes.unit.spec.ts b/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-has-transform-dirty-changes.unit.spec.ts
index 6cc2a10fc5f..8b861505749 100644
--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-has-transform-dirty-changes.unit.spec.ts
+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-has-transform-dirty-changes.unit.spec.ts
@@ -141,6 +141,22 @@ describe("useHasTransformDirtyChanges", () => {
});
});
+ it("returns true when a pythonlibrary entity is dirty", async () => {
+ const { result } = setup({
+ collections: [createMockTransformsCollection()],
+ dirty: [
+ createMockRemoteSyncEntity({
+ model: "pythonlibrary",
+ name: "common.py",
+ }),
+ ],
+ });
+
+ await waitFor(() => {
+ expect(result.current).toBe(true);
+ });
+ });
+
it("returns true when a transforms-namespace collection is dirty", async () => {
const transformsCollection = createMockTransformsCollection({ id: 100 });
const { result } = setup({
diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/middleware/model-configs.ts b/enterprise/frontend/src/metabase-enterprise/remote_sync/middleware/model-configs.ts
index c8bdc3ad1a2..559165e3390 100644
--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/middleware/model-configs.ts
+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/middleware/model-configs.ts
@@ -12,6 +12,7 @@ import { tableApi as coreTableApi } from "metabase/api/table";
import { timelineApi } from "metabase/api/timeline";
import { timelineEventApi } from "metabase/api/timeline-event";
import { getCollectionFromCollectionsTree } from "metabase/selectors/collection";
+import { pythonLibraryApi } from "metabase-enterprise/api/python-transform-library";
import { tableApi as enterpriseTableApi } from "metabase-enterprise/api/table";
import { transformApi } from "metabase-enterprise/api/transform";
import { transformTagApi } from "metabase-enterprise/api/transform-tag";
@@ -208,4 +209,12 @@ export const MODEL_MUTATION_CONFIGS: ModelMutationConfig[] = [
],
invalidation: { type: InvalidationType.Always },
},
+ {
+ modelType: "pythonLibrary",
+ // PythonLibrary only has an upsert endpoint (updatePythonLibrary creates or updates)
+ updateEndpoints: [
+ pythonLibraryApi.endpoints.updatePythonLibrary.matchFulfilled,
+ ],
+ invalidation: { type: InvalidationType.Always },
+ },
];
diff --git a/frontend/src/metabase-types/api/remote-sync.ts b/frontend/src/metabase-types/api/remote-sync.ts
index 1fe5eb0b0ce..bb6b0ed9f02 100644
--- a/frontend/src/metabase-types/api/remote-sync.ts
+++ b/frontend/src/metabase-types/api/remote-sync.ts
@@ -16,7 +16,8 @@ export type RemoteSyncEntityModel =
| "measure"
| "transform"
| "transformtag"
- | "transformjob";
+ | "transformjob"
+ | "pythonlibrary";
export type RemoteSyncEntityStatus =
| "create"
diff --git a/frontend/src/metabase/lib/icon.ts b/frontend/src/metabase/lib/icon.ts
index cf4dc1069f1..440b7cb0fd8 100644
--- a/frontend/src/metabase/lib/icon.ts
+++ b/frontend/src/metabase/lib/icon.ts
@@ -22,7 +22,8 @@ export type IconModel =
| "question"
| "transform"
| "user"
- | "nativequerysnippet";
+ | "nativequerysnippet"
+ | "pythonlibrary";
export type ObjectWithModel = {
id?: unknown;
@@ -59,6 +60,7 @@ export const modelIconMap: Record<IconModel, IconName> = {
timeline: "calendar",
transform: "transform",
user: "person",
+ pythonlibrary: "code_block",
};
export type IconData = {
diff --git a/frontend/src/metabase/lib/icon.unit.spec.ts b/frontend/src/metabase/lib/icon.unit.spec.ts
index 28cef8de5fe..3a3d8221ed2 100644
--- a/frontend/src/metabase/lib/icon.unit.spec.ts
+++ b/frontend/src/metabase/lib/icon.unit.spec.ts
@@ -36,6 +36,10 @@ describe("getIcon", () => {
expect(getIcon({ model: "indexed-entity" })).toEqual({ name: "index" });
});
+ it("should return the correct icon for a python library", () => {
+ expect(getIcon({ model: "pythonlibrary" })).toEqual({ name: "code_block" });
+ });
+
it("should return the correct icon for a dashboard", () => {
expect(getIcon({ model: "dashboard" })).toEqual({ name: "dashboard" });
});
diff --git a/resources/migrations/059_update_migrations.yaml b/resources/migrations/059_update_migrations.yaml
index f5deb014722..2858201e529 100644
--- a/resources/migrations/059_update_migrations.yaml
+++ b/resources/migrations/059_update_migrations.yaml
@@ -744,6 +744,66 @@ databaseChangeLog:
sql: INSERT INTO python_library (path, source) VALUES ('common.py', '')
rollback:
+ - changeSet:
+ id: v59.2026-01-27T12:00:00
+ author: epaget
+ comment: Add entity_id column to python_library table
+ changes:
+ - addColumn:
+ tableName: python_library
+ columns:
+ - column:
+ name: entity_id
+ type: char(21)
+ remarks: Random NanoID tag for unique identity
+ constraints:
+ nullable: true
+ rollback:
+ - dropColumn:
+ tableName: python_library
+ columnName: entity_id
+
+ - changeSet:
+ id: v59.2026-01-27T12:00:01
+ author: epaget
+ comment: Backfill entity_id for existing common.py python_library row
+ changes:
+ - sql:
+ sql: UPDATE python_library SET entity_id = 'cWWH9qJPvHNB3rP2vLZrK' WHERE path = 'common.py' AND entity_id IS NULL
+ rollback:
+ - sql:
+ sql: UPDATE python_library SET entity_id = NULL WHERE path = 'common.py'
+
+ - changeSet:
+ id: v59.2026-01-27T12:00:02
+ author: epaget
+ comment: Add index on entity_id column in python_library table
+ changes:
+ - createIndex:
+ tableName: python_library
+ indexName: idx_python_library_entity_id
+ columns:
+ - column:
+ name: entity_id
+ rollback:
+ - dropIndex:
+ tableName: python_library
+ indexName: idx_python_library_entity_id
+
+ - changeSet:
+ id: v59.2026-01-27T12:00:03
+ author: epaget
+ comment: Add unique constraint on entity_id column in python_library table
+ changes:
+ - addUniqueConstraint:
+ tableName: python_library
+ columnNames: entity_id
+ constraintName: unique_python_library_entity_id
+ rollback:
+ - dropUniqueConstraint:
+ tableName: python_library
+ constraintName: unique_python_library_entity_id
+
# >>>>>>>>>> DO NOT ADD NEW MIGRATIONS BELOW THIS LINE! ADD THEM ABOVE <<<<<<<<<<
########################################################################################################################
diff --git a/resources/serialization-order.edn b/resources/serialization-order.edn
index ad3f6f27759..54d755a63ef 100644
--- a/resources/serialization-order.edn
+++ b/resources/serialization-order.edn
@@ -165,6 +165,11 @@
:TransformJob []
[:TransformJob :job_tags] []
:TransformTag []
+ :PythonLibrary [:path
+ :source
+ :entity_id
+ :created_at
+ :serdes/meta]
:Metabot [:name
:description
:created_at
diff --git a/test_resources/serialization_baseline/python-libraries/cWWH9qJPvHNB3rP2vLZrK.yaml b/test_resources/serialization_baseline/python-libraries/cWWH9qJPvHNB3rP2vLZrK.yaml
new file mode 100644
index 00000000000..df7768a7956
--- /dev/null
+++ b/test_resources/serialization_baseline/python-libraries/cWWH9qJPvHNB3rP2vLZrK.yaml
@@ -0,0 +1,9 @@
+created_at: '2026-01-21T21:50:59.518133Z'
+entity_id: cWWH9qJPvHNB3rP2vLZrK
+path: common.py
+source: |-
+ def helper():
+ return 42
+serdes/meta:
+- id: cWWH9qJPvHNB3rP2vLZrK
+ model: PythonLibrary
diff --git a/test_resources/serialization_baseline/settings.yaml b/test_resources/serialization_baseline/settings.yaml
index f7340d39168..26c4c4cd92e 100644
--- a/test_resources/serialization_baseline/settings.yaml
+++ b/test_resources/serialization_baseline/settings.yaml
@@ -12,9 +12,9 @@ available-timezones: null
breakout-bins-num: null
custom-formatting: null
custom-geojson: null
+custom-geojson-enabled: null
custom-homepage: null
custom-homepage-dashboard: null
-custom-geojson-enabled: null
default-maps-enabled: null
disable-cors-on-localhost: null
email-max-recipients-per-second: null