Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Serdes file and folder human readable names (#71583)

Alexander Polyankin4bbdb37c66a1
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index f7229d463a0..3028cabb3bf 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -3133,6 +3133,7 @@
collections
config
events
+ lib
logger
models
plugins
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
index e4cb7a610b8..f3f3e1a1c3f 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
@@ -243,7 +243,8 @@
last-imported-version (remote-sync.task/last-version)
first-import? (nil? last-imported-version)
path-filters [#"collections/.*" #"databases/.*" #"actions/.*"
- #"transforms/.*" #"python-libraries/.*" #"snippets/.*"]
+ ;; python-libraries is the old name for python_libraries, support both for backwards compatibility
+ #"transforms/.*" #"python[_-]libraries/.*" #"snippets/.*"]
base-ingestable (source.p/->ingestable snapshot {:path-filters path-filters})
has-transforms? (snapshot-has-transforms? base-ingestable)
{:keys [conflicts summary]} (get-conflicts base-ingestable first-import?)
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/source.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/source.clj
index 6a832f9641b..db71503007a 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/source.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/source.clj
@@ -48,10 +48,10 @@
(defn- remote-sync-path
[opts entity]
- (let [base-path (serdes/storage-path entity opts)
- dirnames (drop-last base-path)
- basename (str (last base-path) ".yaml")]
- (str/join File/separator (map serialization/escape-segment (concat dirnames [basename])))))
+ (let [resolved (serialization/resolve-storage-path opts entity)
+ dirnames (drop-last resolved)
+ basename (str (last resolved) ".yaml")]
+ (str/join File/separator (concat dirnames [basename]))))
(defn- ->file-spec
"Converts entity from serdes stream into file spec for source write-files! "
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj
index 2078d9c4729..47bd61da729 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj
@@ -15,6 +15,7 @@
[clojure.string :as str]
[clojure.walk :as walk]
[metabase-enterprise.remote-sync.settings :as rs-settings]
+ [metabase-enterprise.serialization.core :as serialization]
[metabase-enterprise.transforms-python.core :as transforms-python]
[metabase.collections.core :as collections]
[metabase.collections.models.collection :as collection]
@@ -886,11 +887,11 @@
(let [;; Transform entity fields to serdes format (e.g., table_id -> [db schema table])
transformed-entity (transform-entity-for-serdes entity)
entity-with-meta (assoc transformed-entity :serdes/meta serdes-meta)
- storage-path (serdes/storage-path entity-with-meta ctx)]
+ resolved (serialization/resolve-storage-path ctx entity-with-meta)]
;; For Collections, drop the last segment (filename) to get directory path
(if (= model-type "Collection")
- (str/join "/" (butlast storage-path))
- (str/join "/" storage-path))))
+ (str/join "/" (butlast resolved))
+ (str/join "/" resolved))))
(catch Exception e
(log/warnf "Failed to build storage path for %s %s: %s"
model-type (:id entity) (ex-message e))
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/core.clj b/enterprise/backend/src/metabase_enterprise/serialization/core.clj
index ea0219ed869..73449d05e0b 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/core.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/core.clj
@@ -33,4 +33,4 @@
load-metabase!]
[metabase-enterprise.serialization.v2.storage
store!
- escape-segment])
+ resolve-storage-path])
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/ingest.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/ingest.clj
index 6c5eeea0e66..96064c8c294 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/ingest.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/ingest.clj
@@ -79,8 +79,10 @@
(yaml/from-file {:key-fn parse-key})
read-timestamps))
-(def legal-top-level-paths "Known top-level paths for directory with serialization output"
- #{"actions" "collections" "databases" "glossary" "python-libraries" "snippets" "transforms"})
+(def legal-top-level-paths
+ "Known top-level paths for directory with serialization output.
+ We support both \"python-libraries\" and \"python_libraries\" for backwards compatibility. The modern name is \"python_libraries\"."
+ #{"actions" "collections" "databases" "glossary" "python_libraries" "python-libraries" "snippets" "transforms"})
(defn- ingest-all
"Returns {:entities {unlabeled-hierarchy [hierarchy File]}, :errors [Exception...]}.
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/storage.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/storage.clj
index 86edda12e92..73d7890e1a0 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/storage.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/storage.clj
@@ -3,29 +3,65 @@
[clojure.java.io :as io]
[clojure.string :as str]
[metabase-enterprise.serialization.dump :refer [spit-yaml!]]
+ [metabase.lib.core :as lib]
[metabase.models.serialization :as serdes]
- [metabase.util.log :as log])
+ [metabase.util :as u]
+ [metabase.util.log :as log]
+ [metabase.util.string :as u.str])
(:import
[java.io File]
[java.nio.file Path]))
(set! *warn-on-reflection* true)
-(defn escape-segment
- "Given a path segment, which is supposed to be the name of a single file or directory, escape any slashes inside it.
- This occurs in practice, for example with a `Field.name` containing a slash like \"Company/organization website\"."
- [segment]
- (-> segment
- (str/replace "/" "__SLASH__")
- (str/replace "\\" "__BACKSLASH__")))
+(def ^:private max-label-length 100)
+(def ^:private max-label-bytes 200) ;; 255 is a limit in ext4
+
+(defn- slugify-name
+ "Slugify a name for use as a file or directory name: lowercase, replace special chars with underscores,
+ preserve dots and unicode, escape slashes. Truncated for filesystem safety."
+ [^String s]
+ (when (seq s)
+ (-> s
+ u/lower-case-en
+ (str/replace "\\" "__BACKSLASH__")
+ (str/replace "/" "__SLASH__")
+ (str/replace #"[^\p{L}\p{N}_.]" "_")
+ (u.str/limit-bytes max-label-bytes)
+ (u.str/limit-chars max-label-length))))
+
+(defn- resolve-path
+ "Given a storage path (vector of `{:label ... :key ...}` maps), resolves to a vector of strings
+ with deduplication per folder. Uses `unique-name-fns` atom `{parent-key -> unique-name-fn}` where
+ each fn is a `lib/non-truncating-unique-name-generator`."
+ [unique-name-fns path]
+ (loop [remaining path
+ parent-key nil
+ resolved []]
+ (if (empty? remaining)
+ resolved
+ (let [{:keys [label key]} (first remaining)
+ slug (slugify-name label)
+ gen (or (get @unique-name-fns parent-key)
+ (let [g (lib/non-truncating-unique-name-generator)]
+ (swap! unique-name-fns assoc parent-key g)
+ g))
+ unique-name (gen key slug)]
+ (recur (rest remaining)
+ key
+ (conj resolved unique-name))))))
+
+(defn resolve-storage-path
+ "Given ctx and entity, returns a vector of resolved (slugified, deduplicated) path strings.
+ The last element is the filename (without extension)."
+ [ctx entity]
+ (resolve-path (:unique-name-fns ctx) (serdes/storage-path entity ctx)))
(defn- file ^File [ctx entity]
- (let [;; Get the desired [[serdes/storage-path]].
- base-path (serdes/storage-path entity ctx)
- dirnames (drop-last base-path)
- ;; Attach the file extension to the last part.
- basename (str (last base-path) ".yaml")]
- (apply io/file (:root-dir ctx) (map escape-segment (concat dirnames [basename])))))
+ (let [resolved (resolve-storage-path ctx entity)
+ dirnames (drop-last resolved)
+ basename (str (last resolved) ".yaml")]
+ (apply io/file (:root-dir ctx) (concat dirnames [basename]))))
(defn- store-entity! [opts entity]
(let [f (file opts entity)]
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/models/python_library.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/models/python_library.clj
index 19d881c0a22..cfe501725e6 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms_python/models/python_library.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/models/python_library.clj
@@ -94,8 +94,7 @@
[:path])
(defmethod serdes/storage-path "PythonLibrary" [entity _ctx]
- (let [{:keys [id label]} (-> entity serdes/path last)]
- ["python-libraries" (serdes/storage-leaf-file-name id label)]))
+ [{:label "python-libraries"} {:label (:path entity) :key (:entity_id entity)}])
;;; ------------------------------------------------ Event Hooks -----------------------------------------------------
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj
index 49e069e262c..4c0117e570f 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj
@@ -44,7 +44,7 @@
(deftest import!-with-branch-parameter-test
(testing "import! with branch parameter uses provided branch"
(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})
- custom-files {"custom-branch" {"collections/custom-collection.yaml"
+ custom-files {"custom-branch" {"collections/main/custom_collection/custom_collection.yaml"
(test-helpers/generate-collection-yaml "custom-collection-idx" "Custom Collection")}}
result (impl/import! (source.p/snapshot (test-helpers/create-mock-source :initial-files custom-files)) task-id)]
(is (= :success (:status result))))))
@@ -83,9 +83,9 @@
(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})
;; A valid collection + a card with unparseable YAML (malformed syntax)
bad-yaml "name: Bad Card\nentity_id: bad-card-entity0001\ndataset_query: [invalid\n"
- files {"main" {"collections/coll01xxxxxxxxxxxxx_test/coll01xxxxxxxxxxxxx_test.yaml"
+ files {"main" {"collections/main/test/test.yaml"
(test-helpers/generate-collection-yaml "coll01xxxxxxxxxxxxx" "Test")
- "collections/coll01xxxxxxxxxxxxx_test/cards/bad-card-entity0001.yaml"
+ "collections/main/test/bad_card.yaml"
bad-yaml}}
result (impl/import! (source.p/snapshot (test-helpers/create-mock-source :initial-files files)) task-id)]
(is (= :error (:status result))
@@ -257,9 +257,9 @@
:model/Collection {coll2-id :id} {:name "Collection 2" :is_remote_synced true :entity_id "test-collection-2xxxx" :location "/"}
:model/Card {card1-id :id} {:name "Card 1" :collection_id coll1-id :entity_id "test-card-1xxxxxxxxxx"}
:model/Card {card2-id :id} {:name "Card 2" :collection_id coll2-id :entity_id "test-card-2xxxxxxxxxx"}]
- (let [test-files {"test-branch" {"collections/test-collection-1xxxx-_/test-collection-1xxxx.yaml"
+ (let [test-files {"test-branch" {"collections/main/test_collection_1/test_collection_1.yaml"
(test-helpers/generate-collection-yaml "test-collection-1xxxx" "Test Collection 1")
- "collections/test-collection-1xxxx-_/cards/test-card-1.yaml"
+ "collections/main/test_collection_1/test_card_1.yaml"
(test-helpers/generate-card-yaml "test-card-1xxxxxxxxxx" "Test Card 1" "test-collection-1xxxx")}}
mock-main (test-helpers/create-mock-source :initial-files test-files :branch "test-branch")
result (impl/import! (source.p/snapshot mock-main) (:id import-task))]
@@ -326,9 +326,9 @@
{:model_type "Card" :model_id card-id :model_name "Test Card" :status "updated" :status_changed_at (t/offset-date-time)}
{:model_type "Card" :model_id 999 :model_name "Test Card2" :status "deleted" :status_changed_at (t/offset-date-time)}])
(is (= 3 (t2/count :model/RemoteSyncObject)))
- (let [test-files {"main" {"collections/test-collection-1xxxx-_/test-collection-1xxxx.yaml"
+ (let [test-files {"main" {"collections/main/test_collection/test_collection.yaml"
(test-helpers/generate-collection-yaml "test-collection-1xxxx" "Test Collection")
- "collections/test-collection-1xxxx-_/cards/test-card-1.yaml"
+ "collections/main/test_collection/test_card.yaml"
(test-helpers/generate-card-yaml "test-card-1xxxxxxxxxx" "Test Card" "test-collection-1xxxx")}}
mock-source (test-helpers/create-mock-source :initial-files test-files)
result (impl/import! (source.p/snapshot mock-source) task-id)]
@@ -386,21 +386,21 @@
(t2/insert! :model/RemoteSyncObject
[{:model_type "Collection" :model_id active-coll-id :model_name "Active Collection" :status "synced" :status_changed_at (t/offset-date-time)}
{:model_type "Collection" :model_id removed-coll-id :model_name "Removed Collection" :status "removed" :status_changed_at (t/offset-date-time)}])
- (let [initial-files {"main" {"collections/active-collection-xxx_active_collection/active-collection-xxx.yaml"
+ (let [initial-files {"main" {"collections/main/active_collection/active_collection.yaml"
(test-helpers/generate-collection-yaml "active-collection-xxx" "Active Collection")
- "collections/active-collection-xxx_active_collection/cards/active-card.yaml"
+ "collections/main/active_collection/active_card.yaml"
(test-helpers/generate-card-yaml "active-card-xxxxxxxxx" "Active Card" "active-collection-xxx")
- "collections/removed-collection-xx_removed_collection/removed-collection-xx.yaml"
+ "collections/main/removed_collection/removed_collection.yaml"
(test-helpers/generate-collection-yaml "removed-collection-xx" "Removed Collection")
- "collections/removed-collection-xx_removed_collection/cards/removed-card.yaml"
+ "collections/main/removed_collection/removed_card.yaml"
(test-helpers/generate-card-yaml "removed-card-xxxxxxxx" "Removed Card" "removed-collection-xx")}}
mock-source (test-helpers/create-mock-source :initial-files initial-files)
result (impl/export! (source.p/snapshot mock-source) task-id "Test commit")]
(is (= :success (:status result)))
(let [files-after-export (get @(:files-atom mock-source) "main")]
- (is (some #(str/includes? % "active-collection-xxx") (keys files-after-export))
+ (is (some #(str/includes? % "active_collection") (keys files-after-export))
"Active collection files should exist after export")
- (is (not (some #(str/includes? % "removed-collection-xx") (keys files-after-export)))
+ (is (not (some #(str/includes? % "removed_collection") (keys files-after-export)))
"Removed collection files should be deleted after export")))))))))
(deftest export!-only-deletes-top-level-removed-collections-test
@@ -421,18 +421,18 @@
(t2/insert! :model/RemoteSyncObject
[{:model_type "Collection" :model_id parent-coll-id :model_name "Parent Collection" :status "synced" :status_changed_at (t/offset-date-time)}
{:model_type "Collection" :model_id nested-coll-id :model_name "Nested Collection" :status "removed" :status_changed_at (t/offset-date-time)}])
- (let [initial-files {"main" {"collections/parent-collection-xx_parent_collection/parent-collection-xx.yaml"
+ (let [initial-files {"main" {"collections/main/parent_collection/parent_collection.yaml"
(test-helpers/generate-collection-yaml "parent-collection-xx" "Parent Collection")
- "collections/parent-collection-xx_parent_collection/nested-removed-collxx_nested_removed_collection/nested-removed-collxx.yaml"
+ "collections/main/parent_collection/nested_removed_collection/nested_removed_collection.yaml"
(test-helpers/generate-collection-yaml "nested-removed-collxx" "Nested Removed Collection"
:parent-id "parent-collection-xx")}}
mock-source (test-helpers/create-mock-source :initial-files initial-files)
result (impl/export! (source.p/snapshot mock-source) task-id "Test commit")]
(is (= :success (:status result)))
(let [files-after-export (get @(:files-atom mock-source) "main")]
- (is (some #(str/includes? % "parent-collection-xx") (keys files-after-export))
+ (is (some #(str/includes? % "parent_collection") (keys files-after-export))
"Parent collection files should exist after export")
- (is (some #(str/includes? % "nested-removed-collxx") (keys files-after-export))
+ (is (some #(str/includes? % "nested_removed_collection") (keys files-after-export))
"Nested removed collection files should NOT be deleted (only top-level removals are processed)")))))))))
(deftest export!-handles-multiple-removed-top-level-collections-test
@@ -460,23 +460,23 @@
[{:model_type "Collection" :model_id active-coll-id :model_name "Active Collection" :status "synced" :status_changed_at (t/offset-date-time)}
{:model_type "Collection" :model_id removed-coll-1-id :model_name "Removed Col 1" :status "removed" :status_changed_at (t/offset-date-time)}
{:model_type "Collection" :model_id removed-coll-2-id :model_name "Removed Col 2" :status "removed" :status_changed_at (t/offset-date-time)}])
- (let [initial-files {"main" {"collections/removed-coll-1xxxxxxx_removed_collection_1/removed-coll-1xxxxxxx.yaml"
+ (let [initial-files {"main" {"collections/main/removed_collection_1/removed_collection_1.yaml"
(test-helpers/generate-collection-yaml "removed-coll-1xxxxxxx" "Removed Collection 1")
- "collections/removed-coll-1xxxxxxx_removed_collection_1/cards/card-1.yaml"
+ "collections/main/removed_collection_1/card_1.yaml"
(test-helpers/generate-card-yaml "card-1-entity-idxxxxx" "Card 1" "removed-coll-1xxxxxxx")
- "collections/removed-coll-2xxxxxxx_removed_collection_2/removed-coll-2xxxxxxx.yaml"
+ "collections/main/removed_collection_2/removed_collection_2.yaml"
(test-helpers/generate-collection-yaml "removed-coll-2xxxxxxx" "Removed Collection 2")
- "collections/removed-coll-2xxxxxxx_removed_collection_2/dashboards/dash-1.yaml"
+ "collections/main/removed_collection_2/dashboard_1.yaml"
(test-helpers/generate-dashboard-yaml "dash-1-entity-idxxxxx" "Dashboard 1" "removed-coll-2xxxxxxx")}}
mock-source (test-helpers/create-mock-source :initial-files initial-files)
result (impl/export! (source.p/snapshot mock-source) task-id "Test commit")]
(is (= :success (:status result)))
(let [files-after-export (get @(:files-atom mock-source) "main")]
- (is (some #(str/includes? % "active-coll-xxxxxxxxx") (keys files-after-export))
+ (is (some #(str/includes? % "active_collection") (keys files-after-export))
"Active collection files should exist after export")
- (is (not (some #(str/includes? % "removed-coll-1xxxxxxx") (keys files-after-export)))
+ (is (not (some #(str/includes? % "removed_collection_1") (keys files-after-export)))
"First removed collection files should be deleted")
- (is (not (some #(str/includes? % "removed-coll-2xxxxxxx") (keys files-after-export)))
+ (is (not (some #(str/includes? % "removed_collection_2") (keys files-after-export)))
"Second removed collection files should be deleted")))))))))
(deftest finish-remote-config!-sets-default-branch-when-blank-test
@@ -546,7 +546,7 @@
(deftest import!-v57-type-remote-synced-migration-test
(testing "importing v57 export with type=remote-synced sets is_remote_synced=true and clears type"
(let [v57-entity-id "v57-collection-xxxx"
- v57-files {"main" {(str "collections/" v57-entity-id "_v57_collection/" v57-entity-id "_v57_collection.yaml")
+ v57-files {"main" {"collections/main/v57_collection/v57_collection.yaml"
(test-helpers/generate-v57-collection-yaml v57-entity-id "V57 Collection" :type "remote-synced")}}
mock-source (test-helpers/create-mock-source :initial-files v57-files)]
(mt/with-model-cleanup [:model/Collection :model/RemoteSyncTask]
@@ -565,10 +565,10 @@
(testing "importing v57 export with nested type=remote-synced collections migrates all correctly"
(let [parent-entity-id "v57-parent-collxxxxxx"
child-entity-id "v57-child-collxxxxxxx"
- v57-files {"main" {(str "collections/" parent-entity-id "_v57_parent/" parent-entity-id "_v57_parent.yaml")
+ v57-files {"main" {"collections/main/v57_parent/v57_parent.yaml"
(test-helpers/generate-v57-collection-yaml parent-entity-id "V57 Parent" :type "remote-synced")
- (str "collections/" parent-entity-id "_v57_parent/" child-entity-id "_v57_child/" child-entity-id "_v57_child.yaml")
+ "collections/main/v57_parent/v57_child/v57_child.yaml"
(test-helpers/generate-v57-collection-yaml child-entity-id "V57 Child" :parent-id parent-entity-id :type "remote-synced")}}
mock-source (test-helpers/create-mock-source :initial-files v57-files)]
(mt/with-model-cleanup [:model/Collection :model/RemoteSyncTask]
@@ -603,13 +603,13 @@
(t2/insert! :model/RemoteSyncObject
[{:model_type "Collection" :model_id coll-id :model_name "Active Collection" :status "synced" :status_changed_at (t/offset-date-time)}
{:model_type "Table" :model_id table-id :model_name "test-table" :model_table_id table-id :model_table_name "test-table" :status "removed" :status_changed_at (t/offset-date-time)}])
- (let [initial-files {"main" {"databases/test-db/tables/test-table/test-table.yaml"
+ (let [initial-files {"main" {"databases/test_db/tables/test_table/test_table.yaml"
(test-helpers/generate-table-yaml "test-table" "test-db")}}
mock-source (test-helpers/create-mock-source :initial-files initial-files)
result (impl/export! (source.p/snapshot mock-source) task-id "Test commit")]
(is (= :success (:status result)))
(let [files-after-export (get @(:files-atom mock-source) "main")]
- (is (not (some #(str/includes? % "test-table") (keys files-after-export)))
+ (is (not (some #(str/includes? % "test_table") (keys files-after-export)))
"Removed table files should be deleted after export")))))))))
(deftest export!-deletes-files-for-removed-segments-test
@@ -634,13 +634,13 @@
[{:model_type "Collection" :model_id coll-id :model_name "Active Collection" :status "synced" :status_changed_at (t/offset-date-time)}
{:model_type "Segment" :model_id segment-id :model_name "Test Segment" :model_table_id table-id :model_table_name "test-table" :status "removed" :status_changed_at (t/offset-date-time)}])
(let [initial-files {"main" {;; File path uses slugified name to match what serdes/storage-path generates
- "databases/test-db/tables/test-table/segments/test-segment-xxxxxxxx_test_segment.yaml"
+ "databases/test_db/tables/test_table/segments/test_segment.yaml"
(test-helpers/generate-segment-yaml "Test Segment" "test-table" "test-db")}}
mock-source (test-helpers/create-mock-source :initial-files initial-files)
result (impl/export! (source.p/snapshot mock-source) task-id "Test commit")]
(is (= :success (:status result)))
(let [files-after-export (get @(:files-atom mock-source) "main")]
- (is (not (some #(str/includes? % "test-segment-xxxxxxxx") (keys files-after-export)))
+ (is (not (some #(str/includes? % "test_segment") (keys files-after-export)))
"Removed segment files should be deleted after export")))))))))
(deftest export!-updates-removed-table-entries-to-synced-test
@@ -701,13 +701,13 @@
(t2/insert! :model/RemoteSyncObject
[{:model_type "Collection" :model_id coll-id :model_name "Active Collection" :status "synced" :status_changed_at (t/offset-date-time)}
{:model_type "Table" :model_id table-id :model_name "test-table" :model_table_id table-id :model_table_name "test-table" :status "removed" :status_changed_at (t/offset-date-time)}])
- (let [initial-files {"main" {"databases/test-db/schemas/PUBLIC/tables/test-table/test-table.yaml"
+ (let [initial-files {"main" {"databases/test_db/schemas/public/tables/test_table/test_table.yaml"
(test-helpers/generate-table-yaml "test-table" "test-db" :schema "PUBLIC")}}
mock-source (test-helpers/create-mock-source :initial-files initial-files)
result (impl/export! (source.p/snapshot mock-source) task-id "Test commit")]
(is (= :success (:status result)))
(let [files-after-export (get @(:files-atom mock-source) "main")]
- (is (not (some #(str/includes? % "test-table") (keys files-after-export)))
+ (is (not (some #(str/includes? % "test_table") (keys files-after-export)))
"Removed table files should be deleted after export (including schema path)")))))))))
(deftest export!-excludes-archived-segments-test
@@ -747,9 +747,9 @@
(is (= :success (:status result)))
(let [files-after-export (get @(:files-atom mock-source) "main")
file-keys (keys files-after-export)]
- (is (some #(str/includes? % "active-segment-xxxxxx") file-keys)
+ (is (some #(str/includes? % "active_segment") file-keys)
"Active segment should be exported")
- (is (not (some #(str/includes? % "archived-segment-xxxx") file-keys))
+ (is (not (some #(str/includes? % "archived_segment") file-keys))
"Archived segment should NOT be exported")))))))))
(deftest export!-deletes-files-for-archived-segments-test
@@ -774,13 +774,13 @@
(t2/insert! :model/RemoteSyncObject
[{:model_type "Collection" :model_id coll-id :model_name "Active Collection" :status "synced" :status_changed_at (t/offset-date-time)}
{:model_type "Segment" :model_id segment-id :model_name "Archived Segment" :model_table_id table-id :model_table_name "test-table" :status "delete" :status_changed_at (t/offset-date-time)}])
- (let [initial-files {"main" {"databases/test-db/tables/test-table/segments/archived-seg-xxxxxxxx_archived_segment.yaml"
+ (let [initial-files {"main" {"databases/test_db/tables/test_table/segments/archived_segment.yaml"
(test-helpers/generate-segment-yaml "Archived Segment" "test-table" "test-db")}}
mock-source (test-helpers/create-mock-source :initial-files initial-files)
result (impl/export! (source.p/snapshot mock-source) task-id "Test commit")]
(is (= :success (:status result)))
(let [files-after-export (get @(:files-atom mock-source) "main")]
- (is (not (some #(str/includes? % "archived-seg-xxxxxxxx") (keys files-after-export)))
+ (is (not (some #(str/includes? % "archived_segment") (keys files-after-export)))
"Archived segment files should be deleted after export"))
(testing "RemoteSyncObject entry is cleaned up after export"
(is (= "synced" (:status (t2/select-one :model/RemoteSyncObject :model_type "Segment" :model_id segment-id)))
@@ -799,9 +799,9 @@
:is_remote_synced true
:entity_id "test-collection-1xxxx"
:location "/"}]
- (let [test-files {"main" {"collections/test-collection-1xxxx-_/test-collection-1xxxx.yaml"
+ (let [test-files {"main" {"collections/main/test_collection/test_collection.yaml"
(test-helpers/generate-collection-yaml "test-collection-1xxxx" "Test Collection")
- "databases/test-db/tables/test-table/test-table.yaml"
+ "databases/test_db/tables/test_table/test_table.yaml"
(test-helpers/generate-table-yaml "test-table" "test-db")}}
mock-source (test-helpers/create-mock-source :initial-files test-files)
result (impl/import! (source.p/snapshot mock-source) task-id)]
@@ -828,9 +828,9 @@
:is_remote_synced true
:entity_id "test-collection-1xxxx"
:location "/"}]
- (let [test-files {"main" {"collections/test-collection-1xxxx-_/test-collection-1xxxx.yaml"
+ (let [test-files {"main" {"collections/main/test_collection/test_collection.yaml"
(test-helpers/generate-collection-yaml "test-collection-1xxxx" "Test Collection")
- "databases/test-db/tables/test-table/fields/test-field.yaml"
+ "databases/test_db/tables/test_table/fields/test_field.yaml"
(test-helpers/generate-field-yaml "test-field" "test-table" "test-db")}}
mock-source (test-helpers/create-mock-source :initial-files test-files)
result (impl/import! (source.p/snapshot mock-source) task-id)]
@@ -858,13 +858,13 @@
:definition {:source-table table-id
:filter [:> [:field field-id nil] 0]}
:entity_id "TNdMrOCMHrQc_UtvCbTC5"}]
- (let [test-files {"main" {"collections/test-collection-1xxxx-_/test-collection-1xxxx.yaml"
+ (let [test-files {"main" {"collections/main/test_collection/test_collection.yaml"
(test-helpers/generate-collection-yaml "test-collection-1xxxx" "Test Collection")
- "databases/test-db/tables/test-table/test-table.yaml"
+ "databases/test_db/tables/test_table/test_table.yaml"
(test-helpers/generate-table-yaml "test-table" "test-db")
- "databases/test-db/tables/test-table/fields/test-field.yaml"
+ "databases/test_db/tables/test_table/fields/test_field.yaml"
(test-helpers/generate-field-yaml "test-field" "test-table" "test-db" :base-type "type/Integer" :database-type "INTEGER")
- "databases/test-db/tables/test-table/segments/TNdMrOCMHrQc_UtvCbTC5_test_segment.yaml"
+ "databases/test_db/tables/test_table/segments/test_segment.yaml"
(test-helpers/generate-segment-yaml "Test Segment" "test-table" "test-db"
:entity-id "TNdMrOCMHrQc_UtvCbTC5"
:filter-field-name "test-field")}}
@@ -887,9 +887,9 @@
:is_remote_synced true
:entity_id "test-collection-1xxxx"
:location "/"}]
- (let [test-files {"main" {"collections/test-collection-1xxxx-_/test-collection-1xxxx.yaml"
+ (let [test-files {"main" {"collections/main/test_collection/test_collection.yaml"
(test-helpers/generate-collection-yaml "test-collection-1xxxx" "Test Collection")
- "databases/test-db/schemas/PUBLIC/tables/test-table/test-table.yaml"
+ "databases/test_db/schemas/PUBLIC/tables/test_table/test_table.yaml"
(test-helpers/generate-table-yaml "test-table" "test-db" :schema "PUBLIC")}}
mock-source (test-helpers/create-mock-source :initial-files test-files)
result (impl/import! (source.p/snapshot mock-source) task-id)]
@@ -913,11 +913,11 @@
:collection_id coll-id
:type :model
:dataset_query (mt/mbql-query venues)}]
- (let [test-files {"main" {"collections/test-collection-1xxxx-_/test-collection-1xxxx.yaml"
+ (let [test-files {"main" {"collections/main/test_collection/test_collection.yaml"
(test-helpers/generate-collection-yaml "test-collection-1xxxx" "Test Collection")
- "collections/test-collection-1xxxx-_/cards/test-model.yaml"
+ "collections/main/test_collection/test_model.yaml"
(test-helpers/generate-card-yaml "test-model-xxxxxxxxxx" "Test Model" "test-collection-1xxxx" "model")
- "actions/test-action-xxxxxxxxx_test_action.yaml"
+ "actions/test_action.yaml"
(test-helpers/generate-action-yaml "test-action-xxxxxxxxx" "Test Action" "test-model-xxxxxxxxxx")}}
mock-source (test-helpers/create-mock-source :initial-files test-files)
result (impl/import! (source.p/snapshot mock-source) task-id)]
@@ -948,9 +948,9 @@
{:name "Test Measure"
:table_id table-id
:entity_id "TNdMrOCMHrQc_UtvCbTC6"}]
- (let [test-files {"main" {"collections/test-collection-1xxxx-_/test-collection-1xxxx.yaml"
+ (let [test-files {"main" {"collections/main/test_collection/test_collection.yaml"
(test-helpers/generate-collection-yaml "test-collection-1xxxx" "Test Collection")
- "databases/test-db/tables/test-table/measures/TNdMrOCMHrQc_UtvCbTC6_test_measure.yaml"
+ "databases/test_db/tables/test_table/measures/test_measure.yaml"
(test-helpers/generate-measure-yaml "Test Measure" "test-table" "test-db"
:entity-id "TNdMrOCMHrQc_UtvCbTC6"
:agg-field-name "test-field")}}
@@ -983,13 +983,13 @@
[{:model_type "Collection" :model_id coll-id :model_name "Active Collection" :status "synced" :status_changed_at (t/offset-date-time)}
{:model_type "Measure" :model_id measure-id :model_name "Test Measure" :model_table_id table-id :model_table_name "test-table" :status "removed" :status_changed_at (t/offset-date-time)}])
(let [initial-files {"main" {;; File path uses slugified name to match what serdes/storage-path generates
- "databases/test-db/tables/test-table/measures/test-measure-xxxxxxxx_test_measure.yaml"
+ "databases/test_db/tables/test_table/measures/test_measure.yaml"
(test-helpers/generate-measure-yaml "Test Measure" "test-table" "test-db")}}
mock-source (test-helpers/create-mock-source :initial-files initial-files)
result (impl/export! (source.p/snapshot mock-source) task-id "Test commit")]
(is (= :success (:status result)))
(let [files-after-export (get @(:files-atom mock-source) "main")]
- (is (not (some #(str/includes? % "test-measure-xxxxxxxx") (keys files-after-export)))
+ (is (not (some #(str/includes? % "test_measure") (keys files-after-export)))
"Removed measure files should be deleted after export")))))))))
(deftest export!-excludes-archived-measures-test
@@ -1025,9 +1025,9 @@
(is (= :success (:status result)))
(let [files-after-export (get @(:files-atom mock-source) "main")
file-keys (keys files-after-export)]
- (is (some #(str/includes? % "active-measure-xxxxxx") file-keys)
+ (is (some #(str/includes? % "active_measure") file-keys)
(str "Active measure should be exported. Keys: " (pr-str file-keys)))
- (is (not (some #(str/includes? % "archived-measure-xxxx") file-keys))
+ (is (not (some #(str/includes? % "archived_measure") file-keys))
"Archived measure should NOT be exported")))))))))
(deftest export!-deletes-files-for-archived-measures-test
@@ -1050,13 +1050,13 @@
(t2/insert! :model/RemoteSyncObject
[{:model_type "Collection" :model_id coll-id :model_name "Active Collection" :status "synced" :status_changed_at (t/offset-date-time)}
{:model_type "Measure" :model_id measure-id :model_name "Archived Measure" :model_table_id table-id :model_table_name "test-table" :status "delete" :status_changed_at (t/offset-date-time)}])
- (let [initial-files {"main" {"databases/test-db/tables/test-table/measures/archived-meas-xxxxxxx_archived_measure.yaml"
+ (let [initial-files {"main" {"databases/test_db/tables/test_table/measures/archived_measure.yaml"
(test-helpers/generate-measure-yaml "Archived Measure" "test-table" "test-db")}}
mock-source (test-helpers/create-mock-source :initial-files initial-files)
result (impl/export! (source.p/snapshot mock-source) task-id "Test commit")]
(is (= :success (:status result)))
(let [files-after-export (get @(:files-atom mock-source) "main")]
- (is (not (some #(str/includes? % "archived-meas-xxxxxxx") (keys files-after-export)))
+ (is (not (some #(str/includes? % "archived_measure") (keys files-after-export)))
"Archived measure files should be deleted after export"))
(testing "RemoteSyncObject entry is cleaned up after export"
(is (= "synced" (:status (t2/select-one :model/RemoteSyncObject :model_type "Measure" :model_id measure-id)))
@@ -1072,7 +1072,7 @@
remote-sync-enabled true]
(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})
transform-entity-id "auto-enable-trans-xxx"
- test-files {"main" {(str "transforms/" transform-entity-id "_test_transform.yaml")
+ test-files {"main" {"transforms/test_transform.yaml"
(test-helpers/generate-transform-yaml transform-entity-id "Test Transform")}}
mock-source (test-helpers/create-mock-source :initial-files test-files)]
(is (false? (remote-sync.settings/remote-sync-transforms))
@@ -1091,7 +1091,7 @@
remote-sync-enabled true]
(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})
lib-entity-id "auto-enable-lib-xxxxx"
- test-files {"main" {(str "python-libraries/" lib-entity-id ".yaml")
+ test-files {"main" {"python-libraries/uncommon.yaml"
(format "path: uncommon.py
source: |
# shared code
@@ -1118,7 +1118,7 @@ serdes/meta:
(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]
(mt/with-temporary-setting-values [remote-sync-transforms false]
(mt/with-temp [:model/Collection {_coll-id :id} {:name "No Transforms Coll" :is_remote_synced true :entity_id "no-transforms-coll-xx" :location "/"}]
- (let [test-files {"main" {"collections/no-transforms-coll-xx-_/no-transforms-coll-xx.yaml"
+ (let [test-files {"main" {"collections/main/no_transforms_coll/no_transforms_coll.yaml"
(test-helpers/generate-collection-yaml "no-transforms-coll-xx" "No Transforms Coll")}}
mock-source (test-helpers/create-mock-source :initial-files test-files)]
(is (false? (remote-sync.settings/remote-sync-transforms))
@@ -1134,7 +1134,7 @@ serdes/meta:
(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]
(mt/with-temporary-setting-values [remote-sync-transforms true]
(mt/with-temp [:model/Collection {_coll-id :id} {:name "Already Enabled Coll" :is_remote_synced true :entity_id "already-enabled-collx" :location "/"}]
- (let [test-files {"main" {"collections/already-enabled-collx-_/already-enabled-collx.yaml"
+ (let [test-files {"main" {"collections/main/already_enabled_coll/already_enabled_coll.yaml"
(test-helpers/generate-collection-yaml "already-enabled-collx" "Already Enabled Coll")}}
mock-source (test-helpers/create-mock-source :initial-files test-files)]
(is (true? (remote-sync.settings/remote-sync-transforms))
@@ -1150,7 +1150,7 @@ serdes/meta:
(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]
(mt/with-temporary-setting-values [remote-sync-transforms false]
(mt/with-temp [:model/Collection {_coll-id :id} {:name "All Paths Coll" :is_remote_synced true :entity_id "all-paths-coll-xxxxxx" :location "/"}]
- (let [test-files {"main" {"collections/all-paths-coll-xxxxxx-_/all-paths-coll-xxxxxx.yaml"
+ (let [test-files {"main" {"collections/main/all_paths_coll/all_paths_coll.yaml"
(test-helpers/generate-collection-yaml "all-paths-coll-xxxxxx" "All Paths Coll")}}
mock-source (test-helpers/create-mock-source :initial-files test-files)
paths-passed (atom nil)
@@ -1164,7 +1164,7 @@ serdes/meta:
(let [filter-strs (map str @paths-passed)]
(is (some #(str/includes? % "transforms") filter-strs)
"transforms path should be included in filters")
- (is (some #(str/includes? % "python-libraries") filter-strs)
+ (is (some #(str/includes? % "python") filter-strs)
"python-libraries path should be included in filters")
(is (some #(str/includes? % "snippets") filter-strs)
"snippets path should be included in filters"))))))))))))
@@ -1176,7 +1176,7 @@ serdes/meta:
:type "library"
:entity_id collection/library-entity-id
:location "/"}]
- (let [test-files {"main" {"collections/test-collection-1xxxx-_/test-collection-1xxxx.yaml"
+ (let [test-files {"main" {"collections/main/test_collection/test_collection.yaml"
(test-helpers/generate-collection-yaml collection/library-entity-id "Another Library")}}
mock-source (test-helpers/create-mock-source :initial-files test-files)
result (impl/import! (source.p/snapshot mock-source) task-id)]
@@ -1186,7 +1186,7 @@ serdes/meta:
(deftest import!-blocks-if-it-encounters-snippet-conflicts
(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]
(mt/with-temp [:model/NativeQuerySnippet _ {:name "Test Snippet"}]
- (let [test-files {"main" {"collections/test-collection-1xxxx-_/test-collection-1xxxx.yaml"
+ (let [test-files {"main" {"collections/main/test_collection/test_collection.yaml"
(test-helpers/generate-snippet-yaml "blahblahblah" "A Snippet" "select 123")}}
mock-source (test-helpers/create-mock-source :initial-files test-files)
result (impl/import! (source.p/snapshot mock-source) task-id)]
@@ -1196,7 +1196,7 @@ serdes/meta:
(deftest import!-blocks-if-it-encounters-transform-conflicts
(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]
(mt/with-temp [:model/Transform _ {:name "Test Transform"}]
- (let [test-files {"main" {"collections/test-collection-1xxxx-_/test-collection-1xxxx.yaml"
+ (let [test-files {"main" {"collections/main/test_collection/test_collection.yaml"
(test-helpers/generate-transform-yaml "blahblahblah" "A Transform")}}
mock-source (test-helpers/create-mock-source :initial-files test-files)
result (impl/import! (source.p/snapshot mock-source) task-id)]
@@ -1213,11 +1213,11 @@ serdes/meta:
:location "/"}
:model/NativeQuerySnippet _ {:name "Test Snippet"}
:model/Transform _ {:name "Test Transform"}]
- (let [test-files {"main" {"collections/lib-_/lib.yaml"
+ (let [test-files {"main" {"collections/main/lib/lib.yaml"
(test-helpers/generate-collection-yaml collection/library-entity-id "Remote Library")
- "collections/snip-_/snip.yaml"
+ "collections/main/snip/snip.yaml"
(test-helpers/generate-snippet-yaml "snip-entity-id" "Remote Snippet" "select 1")
- "collections/trans-_/trans.yaml"
+ "collections/main/trans/trans.yaml"
(test-helpers/generate-transform-yaml "trans-entity-id" "Remote Transform")}}
mock-source (test-helpers/create-mock-source :initial-files test-files)
result (impl/import! (source.p/snapshot mock-source) task-id)]
@@ -1232,7 +1232,7 @@ serdes/meta:
:type "library"
:entity_id collection/library-entity-id
:location "/"}]
- (let [test-files {"main" {"collections/lib-_/lib.yaml"
+ (let [test-files {"main" {"collections/main/lib/lib.yaml"
(test-helpers/generate-collection-yaml collection/library-entity-id "Remote Library")}}
mock-source (test-helpers/create-mock-source :initial-files test-files)
result (impl/import! (source.p/snapshot mock-source) task-id :force? true)]
@@ -1247,7 +1247,7 @@ serdes/meta:
:entity_id collection/library-entity-id
:location "/"}]
(with-redefs [remote-sync.task/last-version (constantly "previous-version")]
- (let [test-files {"main" {"collections/lib-_/lib.yaml"
+ (let [test-files {"main" {"collections/main/lib/lib.yaml"
(test-helpers/generate-collection-yaml collection/library-entity-id "Remote Library")}}
mock-source (test-helpers/create-mock-source :initial-files test-files)
result (impl/import! (source.p/snapshot mock-source) task-id)]
@@ -1276,7 +1276,7 @@ serdes/meta:
:type nil
:entity_id collection/library-entity-id
:location "/"}]
- (let [test-files {"main" {"collections/lib-_/lib.yaml"
+ (let [test-files {"main" {"collections/main/lib/lib.yaml"
(test-helpers/generate-collection-yaml collection/library-entity-id "Remote Library")}}
mock-source (test-helpers/create-mock-source :initial-files test-files)
result (impl/import! (source.p/snapshot mock-source) task-id)]
@@ -1292,7 +1292,7 @@ serdes/meta:
:type "library"
:entity_id collection/library-entity-id
:location "/"}]
- (let [test-files {"main" {"collections/test-collection-1xxxx-_/test-collection-1xxxx.yaml"
+ (let [test-files {"main" {"collections/main/test_collection/test_collection.yaml"
(test-helpers/generate-collection-yaml collection/library-entity-id "Another Library")}}
mock-source (test-helpers/create-mock-source :initial-files test-files)
result (impl/import! (source.p/snapshot mock-source) task-id)]
@@ -1308,9 +1308,9 @@ serdes/meta:
(testing "import! detects transforms conflict when local has transforms and import has transforms"
(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]
(mt/with-temp [:model/Transform _ {:name "Local Transform"}]
- (let [test-files {"main" {"collections/test-coll-_/test-coll.yaml"
+ (let [test-files {"main" {"collections/main/test_coll/test_coll.yaml"
(test-helpers/generate-collection-yaml "test-collection-1xxxx" "Test Collection")
- "transforms/test-transform-_/test-transform.yaml"
+ "transforms/remote_transform.yaml"
(test-helpers/generate-transform-yaml "test-transform-xxxxx" "Remote Transform")}}
mock-source (test-helpers/create-mock-source :initial-files test-files)
result (impl/import! (source.p/snapshot mock-source) task-id)]
@@ -1322,9 +1322,9 @@ serdes/meta:
(testing "import! detects snippets conflict when local has snippets and import has snippets"
(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]
(mt/with-temp [:model/NativeQuerySnippet _ {:name "Local Snippet"}]
- (let [test-files {"main" {"collections/test-coll-_/test-coll.yaml"
+ (let [test-files {"main" {"collections/main/test_coll/test_coll.yaml"
(test-helpers/generate-collection-yaml "test-collection-1xxxx" "Test Collection")
- "snippets/test-snippet-_/test-snippet.yaml"
+ "snippets/remote_snippet.yaml"
(test-helpers/generate-snippet-yaml "test-snippet-xxxxxxx" "Remote Snippet" "SELECT 1")}}
mock-source (test-helpers/create-mock-source :initial-files test-files)
result (impl/import! (source.p/snapshot mock-source) task-id)]
@@ -1337,9 +1337,9 @@ serdes/meta:
(mt/with-model-cleanup [:model/RemoteSyncTask]
(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]
(mt/with-temp [:model/Collection _ {:name "Local Transforms" :namespace "transforms"}]
- (let [test-files {"main" {"collections/txns-coll-_/txns-coll.yaml"
+ (let [test-files {"main" {"collections/main/txns_coll/txns_coll.yaml"
(test-helpers/generate-collection-yaml "txns-coll-xxxxxxxxx" "Txns Coll")
- "collections/txns-ns-coll-_/txns-ns-coll.yaml"
+ "collections/transforms/remote_transforms/remote_transforms.yaml"
(test-helpers/generate-collection-yaml "txns-ns-coll-xxxxxxx" "Remote Transforms"
:namespace "transforms")}}
mock-source (test-helpers/create-mock-source :initial-files test-files)
@@ -1353,9 +1353,9 @@ serdes/meta:
(mt/with-model-cleanup [:model/RemoteSyncTask]
(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]
(mt/with-temp [:model/Collection _ {:name "Local Snippets" :namespace "snippets"}]
- (let [test-files {"main" {"collections/snip-coll-_/snip-coll.yaml"
+ (let [test-files {"main" {"collections/main/snip_coll/snip_coll.yaml"
(test-helpers/generate-collection-yaml "snip-coll-xxxxxxxxx" "Snip Coll")
- "collections/snip-ns-coll-_/snip-ns-coll.yaml"
+ "collections/snippets/remote_snippets/remote_snippets.yaml"
(test-helpers/generate-collection-yaml "snip-ns-coll-xxxxxxx" "Remote Snippets"
:namespace "snippets")}}
mock-source (test-helpers/create-mock-source :initial-files test-files)
@@ -1383,9 +1383,9 @@ serdes/meta:
:model_name "synced"
:status "synced"
:status_changed_at (t/offset-date-time)})))
- (let [test-files {"main" {"collections/syncd-coll-_/syncd-coll.yaml"
+ (let [test-files {"main" {"collections/main/syncd_coll/syncd_coll.yaml"
(test-helpers/generate-collection-yaml "syncd-coll-xxxxxxxxx" "Syncd Coll")
- "collections/syncd-ns-coll-_/syncd-ns-coll.yaml"
+ "collections/transforms/remote_transforms/remote_transforms.yaml"
(test-helpers/generate-collection-yaml "syncd-ns-coll-xxxxxxx" "Remote Transforms"
:namespace "transforms")}}
mock-source (test-helpers/create-mock-source :initial-files test-files)
@@ -1405,12 +1405,12 @@ serdes/meta:
:model_name "pre-existing"
:status "synced"
:status_changed_at (t/offset-date-time)})))
- (let [test-files {"main" {"collections/noloc-coll-_/noloc-coll.yaml"
+ (let [test-files {"main" {"collections/main/noloc_coll/noloc_coll.yaml"
(test-helpers/generate-collection-yaml "noloc-coll-xxxxxxxxx" "Noloc Coll")
- "collections/noloc-tx-coll-_/noloc-tx-coll.yaml"
+ "collections/transforms/remote_transforms/remote_transforms.yaml"
(test-helpers/generate-collection-yaml "noloc-tx-coll-xxxxxx" "Remote Transforms"
:namespace "transforms")
- "collections/noloc-sn-coll-_/noloc-sn-coll.yaml"
+ "collections/snippets/remote_snippets/remote_snippets.yaml"
(test-helpers/generate-collection-yaml "noloc-sn-coll-xxxxxx" "Remote Snippets"
:namespace "snippets")}}
mock-source (test-helpers/create-mock-source :initial-files test-files)
@@ -1418,3 +1418,18 @@ serdes/meta:
(is (not= :conflict (:status result))
"Should not detect a conflict when local has no unsynced namespace collections")
(is (nil? (seq (:conflict-details result)))))))))
+
+(deftest import!-old-format-paths-test
+ (testing "import! can load content stored at old-format paths (entity_id in name)"
+ (mt/with-model-cleanup [:model/RemoteSyncTask]
+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})
+ coll-eid "old-fmt-coll-xxxxxxxx"
+ ;; Old-format paths: entity_id in filename
+ test-files {"main" {(str "collections/" coll-eid "_test_collection/" coll-eid "_test_collection.yaml")
+ (test-helpers/generate-collection-yaml coll-eid "Test Collection")}}
+ mock-source (test-helpers/create-mock-source :initial-files test-files)
+ result (impl/import! (source.p/snapshot mock-source) task-id)]
+ (is (= :success (:status result))
+ "import should succeed with old-format paths")
+ (is (t2/exists? :model/Collection :entity_id coll-eid)
+ "collection should have been imported from old-format path")))))
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/snippets_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/snippets_test.clj
index daa5c7622ed..623db08741d 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/snippets_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/snippets_test.clj
@@ -283,12 +283,12 @@ is_sample: false
(mt/with-temporary-setting-values [remote-sync-type :read-write
remote-sync-enabled true]
(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})]
- (mt/with-temp [:model/Collection {coll-id :id coll-eid :entity_id}
+ (mt/with-temp [:model/Collection {coll-id :id}
{:name "Snippets Collection"
:namespace :snippets
:entity_id "snippets-coll-xxxxx"
:location "/"}
- :model/NativeQuerySnippet {snippet-id :id snippet-eid :entity_id}
+ :model/NativeQuerySnippet {snippet-id :id}
{:name "Test Snippet"
:content "SELECT 1"
:collection_id coll-id
@@ -309,9 +309,9 @@ is_sample: false
(is (= :success (:status result))
(str "Export should succeed. Result: " result))
(let [files-after-export (get @(:files-atom mock-source) "main")]
- (is (some #(str/includes? % coll-eid) (keys files-after-export))
+ (is (some #(str/includes? % "snippets_collection") (keys files-after-export))
"Export should include the snippets-namespace collection")
- (is (some #(str/includes? % snippet-eid) (keys files-after-export))
+ (is (some #(str/includes? % "test_snippet") (keys files-after-export))
"Export should include the snippet"))))))))))
(deftest import-snippets-from-yaml-test
@@ -322,9 +322,9 @@ is_sample: false
(mt/with-temporary-setting-values [remote-sync-enabled true]
(let [coll-entity-id "snippets-coll-importx"
snippet-entity-id "test-snippet-importxx"
- test-files {"main" {(str "collections/" coll-entity-id "_snippets_collection/" coll-entity-id "_snippets_collection.yaml")
+ test-files {"main" {"collections/snippets/snippets_collection/snippets_collection.yaml"
(generate-snippets-namespace-collection-yaml coll-entity-id "Snippets Collection")
- (str "snippets/" coll-entity-id "_snippets_collection/" snippet-entity-id "_test_snippet.yaml")
+ "collections/snippets/snippets_collection/test_snippet.yaml"
(test-helpers/generate-snippet-yaml snippet-entity-id "Test Snippet" "SELECT 42" :collection-id coll-entity-id)}}
mock-source (test-helpers/create-mock-source :initial-files test-files)
result (impl/import! (source.p/snapshot mock-source) task-id)]
@@ -407,7 +407,7 @@ is_sample: false
:status "delete"
:status_changed_at (t/offset-date-time)}]
(let [removal-paths (spec/build-all-removal-paths)
- snippet-removal-path (first (filter #(str/includes? % snippet-eid) removal-paths))
+ snippet-removal-path (first (filter #(str/includes? % "archived_snippet") removal-paths))
_ (is (some? snippet-removal-path) "Should have a removal path for the archived snippet")
initial-files {"main" {(str snippet-removal-path ".yaml")
(test-helpers/generate-snippet-yaml snippet-eid "Archived Snippet" "SELECT 1" :collection-id coll-eid)}}
@@ -416,7 +416,7 @@ is_sample: false
(is (= :success (:status result))
(str "Export should succeed. Result: " result))
(let [files-after-export (get @(:files-atom mock-source) "main")]
- (is (not (some #(str/includes? % snippet-eid) (keys files-after-export)))
+ (is (not (some #(str/includes? % "archived_snippet") (keys files-after-export)))
"Archived snippet file should be deleted after export")))
(testing "RemoteSyncObject entry is updated to synced after export"
(is (= "synced" (:status (t2/select-one :model/RemoteSyncObject :model_type "NativeQuerySnippet" :model_id snippet-id)))
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj
index efb6ad6cb34..29b8a0b88ca 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj
@@ -251,7 +251,7 @@
(is (= :success (:status result))
(str "Export should succeed. Result: " result))
(let [files-after-export (get @(:files-atom mock-source) "main")]
- (is (some #(str/includes? % coll-eid)
+ (is (some #(str/includes? % "transforms_collection")
(keys files-after-export))
"Export should include the transforms-namespace collection itself"))))))))))
@@ -278,7 +278,7 @@
(is (= :success (:status result))
(str "Export should succeed. Result: " result))
(let [files-after-export (get @(:files-atom mock-source) "main")]
- (is (not (some #(str/includes? % "transforms/") (keys files-after-export)))
+ (is (not (some #(str/includes? % "collections/transforms/") (keys files-after-export)))
"Export should NOT include transform files when setting is disabled"))))))))))
(defn- generate-transforms-namespace-collection-yaml
@@ -314,9 +314,9 @@ is_sample: false
(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})
coll-entity-id "transforms-coll-xxxxx"
transform-entity-id "test-transform-xxxxxx"
- test-files {"main" {(str "collections/" coll-entity-id "_transforms/" coll-entity-id "_transforms.yaml")
+ test-files {"main" {"collections/transforms/transforms/transforms.yaml"
(generate-transforms-namespace-collection-yaml coll-entity-id "Transforms")
- (str "collections/" coll-entity-id "_transforms/transforms/" transform-entity-id "_test_transform.yaml")
+ "collections/transforms/transforms/test_transform.yaml"
(test-helpers/generate-transform-yaml transform-entity-id "Test Transform" :collection-id coll-entity-id)}}
mock-source (test-helpers/create-mock-source :initial-files test-files)
result (impl/import! (source.p/snapshot mock-source) task-id)]
@@ -336,9 +336,9 @@ is_sample: false
transform-entity-id "python-legacy-fmt-01x"
;; Use old map format: {alias: table_id} — this is how pre-unification exports look
source-tables-yaml (format "{orders: %d}" (mt/id :orders))
- test-files {"main" {(str "collections/" coll-entity-id "_transforms/" coll-entity-id "_transforms.yaml")
+ test-files {"main" {"collections/transforms/transforms/transforms.yaml"
(generate-transforms-namespace-collection-yaml coll-entity-id "Transforms")
- (str "collections/" coll-entity-id "_transforms/transforms/" transform-entity-id "_python_transform.yaml")
+ "collections/transforms/transforms/python_transform.yaml"
(test-helpers/generate-python-transform-yaml transform-entity-id "Python Transform"
"test-data (h2)" source-tables-yaml
:collection-id coll-entity-id)}}
@@ -376,7 +376,7 @@ is_sample: false
{:name "Child Transform"
:collection_id coll-id
:entity_id "child-transform-xxxxx"}
- :model/Transform {root-transform-id :id root-transform-eid :entity_id}
+ :model/Transform {root-transform-id :id}
{:name "Root Transform"
:collection_id nil
:entity_id "root-transform-xxxxxx"}
@@ -397,13 +397,13 @@ is_sample: false
:model_collection_id nil
:status "create"
:status_changed_at (t/offset-date-time)}]
- (let [initial-files {"main" {(str "collections/" coll-eid "_transforms_collection/" coll-eid "_transforms_collection.yaml")
+ (let [initial-files {"main" {"collections/transforms/transforms_collection/transforms_collection.yaml"
(generate-transforms-namespace-collection-yaml coll-eid "Transforms Collection")
- (str "collections/" coll-eid "_transforms_collection/transforms/" transform-eid "_child_transform.yaml")
+ "collections/transforms/transforms_collection/child_transform.yaml"
(test-helpers/generate-transform-yaml transform-eid "Child Transform" :collection-id coll-eid)}}
mock-source (test-helpers/create-mock-source :initial-files initial-files)]
- (is (some #(str/includes? % coll-eid) (keys (get @(:files-atom mock-source) "main"))))
- (is (some #(str/includes? % transform-eid) (keys (get @(:files-atom mock-source) "main"))))
+ (is (some #(str/includes? % "transforms_collection") (keys (get @(:files-atom mock-source) "main"))))
+ (is (some #(str/includes? % "child_transform") (keys (get @(:files-atom mock-source) "main"))))
(t2/update! :model/Collection coll-id {:archived true})
(events/publish-event! :event/collection-update
{:object (t2/select-one :model/Collection :id coll-id)
@@ -412,9 +412,9 @@ is_sample: false
(let [result (impl/export! (source.p/snapshot mock-source) task-id "Test export")]
(is (= :success (:status result)))
(let [files-after-export (get @(:files-atom mock-source) "main")]
- (is (not (some #(str/includes? % coll-eid) (keys files-after-export))))
- (is (not (some #(str/includes? % transform-eid) (keys files-after-export))))
- (is (some #(str/includes? % root-transform-eid) (keys files-after-export))
+ (is (not (some #(str/includes? % "transforms_collection") (keys files-after-export))))
+ (is (not (some #(str/includes? % "child_transform") (keys files-after-export))))
+ (is (some #(str/includes? % "root_transform") (keys files-after-export))
"Root transform should be exported")))))))))))
;;; ------------------------------------------- Collection Cleanup Tests -------------------------------------------
@@ -454,7 +454,7 @@ is_sample: false
:entity_id local-coll-entity-id
:location "/"}]
(is (t2/exists? :model/Collection :id local-coll-id))
- (let [test-files {"main" {(str "collections/" remote-coll-entity-id "_remote_transforms/" remote-coll-entity-id "_remote_transforms.yaml")
+ (let [test-files {"main" {"collections/transforms/remote_transforms/remote_transforms.yaml"
(generate-transforms-namespace-collection-yaml remote-coll-entity-id "Remote Transforms")}}
mock-source (test-helpers/create-mock-source :initial-files test-files)
result (impl/import! (source.p/snapshot mock-source) task-id)]
@@ -484,9 +484,9 @@ is_sample: false
(mt/with-model-cleanup [:model/RemoteSyncTask :model/Transform]
(is (t2/exists? :model/Collection :id coll-id))
(is (t2/exists? :model/Transform :id local-transform-id))
- (let [test-files {"main" {(str "collections/" coll-entity-id "_transforms/" coll-entity-id "_transforms.yaml")
+ (let [test-files {"main" {"collections/transforms/transforms_collection/transforms_collection.yaml"
(generate-transforms-namespace-collection-yaml coll-entity-id "Transforms Collection")
- (str "collections/" coll-entity-id "_transforms/transforms/" remote-transform-entity-id "_remote_transform.yaml")
+ "collections/transforms/transforms_collection/remote_transform.yaml"
(test-helpers/generate-transform-yaml remote-transform-entity-id "Remote Transform" :collection-id coll-entity-id)}}
mock-source (test-helpers/create-mock-source :initial-files test-files)]
(testing "fails with `conflict` status because local transforms will be deleted"
@@ -519,7 +519,7 @@ is_sample: false
:collection_id local-coll-id}]
(is (t2/exists? :model/Collection :id local-coll-id))
(is (t2/exists? :model/Transform :id local-transform-id))
- (let [test-files {"main" {(str "collections/" remote-coll-entity-id "_remote_transforms/" remote-coll-entity-id "_remote_transforms.yaml")
+ (let [test-files {"main" {"collections/transforms/remote_transforms/remote_transforms.yaml"
(generate-transforms-namespace-collection-yaml remote-coll-entity-id "Remote Transforms")}}
mock-source (test-helpers/create-mock-source :initial-files test-files)
result (impl/import! (source.p/snapshot mock-source) task-id :force? true)]
@@ -604,7 +604,7 @@ serdes/meta:
(is (= :success (:status result))
(str "Export should succeed. Result: " result))
(let [files-after-export (get @(:files-atom mock-source) "main")]
- (is (some #(str/includes? % "python-libraries/") (keys files-after-export))
+ (is (some #(str/includes? % "python_libraries/") (keys files-after-export))
"Export should include the PythonLibrary file")))))))))
(deftest import-python-library-from-yaml-test
@@ -615,7 +615,7 @@ serdes/meta:
(mt/with-model-cleanup [:model/RemoteSyncTask]
(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})
lib-entity-id (u/generate-nano-id)
- test-files {"main" {(str "python-libraries/" lib-entity-id ".yaml")
+ test-files {"main" {"python_libraries/common_py.yaml"
(generate-python-library-yaml lib-entity-id "common.py" "def shared_func():\n return 42")}}
mock-source (test-helpers/create-mock-source :initial-files test-files)
snapshot (source.p/snapshot mock-source)
@@ -679,7 +679,7 @@ serdes/meta:
local-entity-id (:entity_id local-library)]
(is (t2/exists? :model/PythonLibrary :path "common.py")
"Local PythonLibrary should exist before import")
- (let [test-files {"main" {(str "python-libraries/" local-entity-id ".yaml")
+ (let [test-files {"main" {"python_libraries/common_py.yaml"
(generate-python-library-yaml local-entity-id "common.py" "# remote version\ndef new_func():\n pass")}}
mock-source (test-helpers/create-mock-source :initial-files test-files)
result (impl/import! (source.p/snapshot mock-source) task-id)]
@@ -716,7 +716,7 @@ serdes/meta:
(is (t2/exists? :model/Collection :id coll-id)
"Local transforms collection should exist before import")
(let [remote-coll-entity-id (u/generate-nano-id)
- test-files {"main" {(str "collections/" remote-coll-entity-id "_remote_collection/" remote-coll-entity-id "_remote_collection.yaml")
+ test-files {"main" {"collections/main/remote_collection/remote_collection.yaml"
(test-helpers/generate-collection-yaml remote-coll-entity-id "Remote Collection")}}
mock-source (test-helpers/create-mock-source :initial-files test-files)
result (impl/import! (source.p/snapshot mock-source) task-id)]
@@ -752,9 +752,9 @@ serdes/meta:
"Local transform should exist before import")
(is (t2/exists? :model/TransformTag :id local-tag-id)
"Local tag should exist before import")
- (let [test-files {"main" {(str "collections/" remote-coll-entity-id "_transforms/" remote-coll-entity-id "_transforms.yaml")
+ (let [test-files {"main" {"collections/transforms/remote_transforms/remote_transforms.yaml"
(generate-transforms-namespace-collection-yaml remote-coll-entity-id "Remote Transforms")
- (str "collections/" remote-coll-entity-id "_transforms/transforms/" remote-transform-entity-id "_remote_transform.yaml")
+ "collections/transforms/remote_transforms/remote_transform.yaml"
(test-helpers/generate-transform-yaml remote-transform-entity-id "Remote Transform" :collection-id remote-coll-entity-id)}}
mock-source (test-helpers/create-mock-source :initial-files test-files)
result-without-force (impl/import! (source.p/snapshot mock-source) task-id)
@@ -800,7 +800,7 @@ serdes/meta:
(is (t2/exists? :model/Collection :id coll-id)
"Local transforms collection should exist before import")
(let [remote-coll-entity-id (u/generate-nano-id)
- test-files {"main" {(str "collections/" remote-coll-entity-id "_remote_collection/" remote-coll-entity-id "_remote_collection.yaml")
+ test-files {"main" {"collections/main/remote_collection/remote_collection.yaml"
(test-helpers/generate-collection-yaml remote-coll-entity-id "Remote Collection")}}
mock-source (test-helpers/create-mock-source :initial-files test-files)
result (impl/import! (source.p/snapshot mock-source) task-id)]
@@ -833,9 +833,9 @@ serdes/meta:
"remote-sync-transforms should be disabled initially")
(is (t2/exists? :model/Transform :id local-transform-id)
"Local transform should exist before import")
- (let [test-files {"main" {(str "collections/" remote-coll-entity-id "_transforms/" remote-coll-entity-id "_transforms.yaml")
+ (let [test-files {"main" {"collections/transforms/remote_transforms/remote_transforms.yaml"
(generate-transforms-namespace-collection-yaml remote-coll-entity-id "Remote Transforms")
- (str "collections/" remote-coll-entity-id "_transforms/transforms/" remote-transform-entity-id "_remote_transform.yaml")
+ "collections/transforms/remote_transforms/remote_transform.yaml"
(test-helpers/generate-transform-yaml remote-transform-entity-id "Remote Transform" :collection-id remote-coll-entity-id)}}
mock-source (test-helpers/create-mock-source :initial-files test-files)
result (impl/import! (source.p/snapshot mock-source) task-id :force? true)]
@@ -858,17 +858,16 @@ serdes/meta:
(mt/with-temp [:model/Collection {coll-id :id} {:name "Transforms Collection"
:namespace collection/transforms-ns
:location "/"}
- :model/Transform {transform-id :id transform-eid :entity_id} {:name "Test Transform"
- :collection_id coll-id}
- :model/TransformTag {tag-id :id tag-eid :entity_id} {:name "Test Tag"
- :built_in_type nil}]
- (let [library (t2/insert-returning-instance! :model/PythonLibrary {:path "common.py" :source "# test"})
- lib-eid (:entity_id library)]
+ :model/Transform {transform-id :id} {:name "Test Transform"
+ :collection_id coll-id}
+ :model/TransformTag {tag-id :id} {:name "Test Tag"
+ :built_in_type nil}]
+ (let [library (t2/insert-returning-instance! :model/PythonLibrary {:path "common.py" :source "# test"})]
(is (t2/exists? :model/Transform :id transform-id))
(is (t2/exists? :model/TransformTag :id tag-id))
(is (t2/exists? :model/PythonLibrary :id (:id library)))
(let [paths-before (spec/build-all-removal-paths)]
- (is (not (some #(str/includes? % transform-eid) paths-before))
+ (is (not (some #(str/includes? % "test_transform") paths-before))
"Transform should not be in removal paths before setting is disabled"))
(settings/sync-transform-tracking! true)
(settings/sync-transform-tracking! false)
@@ -878,11 +877,11 @@ serdes/meta:
:status "delete")
"Sentinel RSO should exist with 'delete' status")
(let [paths-after (spec/build-all-removal-paths)]
- (is (some #(str/includes? % transform-eid) paths-after)
+ (is (some #(str/includes? % "test_transform") paths-after)
"Transform should be in removal paths after setting is disabled")
- (is (some #(str/includes? % tag-eid) paths-after)
+ (is (some #(str/includes? % "test_tag") paths-after)
"TransformTag should be in removal paths after setting is disabled")
- (is (some #(str/includes? % lib-eid) paths-after)
+ (is (some #(str/includes? % "common.py") paths-after)
"PythonLibrary should be in removal paths after setting is disabled"))))))))
(deftest build-all-removal-paths-excludes-builtin-transform-tags-test
@@ -890,10 +889,10 @@ serdes/meta:
(mt/with-premium-features #{:transforms-basic}
(mt/with-temporary-setting-values [remote-sync-transforms true
remote-sync-enabled true]
- (mt/with-temp [:model/TransformTag {custom-tag-id :id custom-tag-eid :entity_id} {:name "Custom Tag"
- :built_in_type nil}
- :model/TransformTag {builtin-tag-id :id builtin-tag-eid :entity_id} {:name "Built-in Tag"
- :built_in_type "target"}]
+ (mt/with-temp [:model/TransformTag {custom-tag-id :id} {:name "Custom Tag"
+ :built_in_type nil}
+ :model/TransformTag {builtin-tag-id :id} {:name "Built-in Tag"
+ :built_in_type "target"}]
(is (t2/exists? :model/TransformTag :id custom-tag-id))
(is (t2/exists? :model/TransformTag :id builtin-tag-id))
(settings/sync-transform-tracking! true)
@@ -904,9 +903,9 @@ serdes/meta:
:status "delete")
"Sentinel RSO should exist with 'delete' status")
(let [paths (spec/build-all-removal-paths)]
- (is (some #(str/includes? % custom-tag-eid) paths)
+ (is (some #(str/includes? % "custom_tag") paths)
"Custom tag (built_in_type nil) should be in removal paths")
- (is (not (some #(str/includes? % builtin-tag-eid) paths))
+ (is (not (some #(str/includes? % "built_in_tag") paths))
"Built-in tag should NOT be in removal paths")))))))
(deftest export-excludes-builtin-transform-tags-test
@@ -954,12 +953,12 @@ serdes/meta:
(mt/with-temp [:model/PythonLibrary _ {:path "common.py"
:source "# builtin"
:entity_id transforms-python/builtin-entity-id}
- :model/PythonLibrary custom-lib {:path "custom.py"
- :source "# custom"}]
+ :model/PythonLibrary _ {:path "custom.py"
+ :source "# custom"}]
(settings/sync-transform-tracking! true)
(settings/sync-transform-tracking! false)
(let [paths (spec/build-all-removal-paths)]
- (is (some #(str/includes? % (:entity_id custom-lib)) paths)
+ (is (some #(str/includes? % "custom.py") paths)
"Custom PythonLibrary should be in removal paths")
- (is (not (some #(str/includes? % transforms-python/builtin-entity-id) paths))
+ (is (not (some #(str/includes? % "common.py") paths))
"Built-in PythonLibrary should NOT be in removal paths")))))))
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj
index 26fbcceed86..b68909f93a7 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj
@@ -30,16 +30,11 @@
[#"([^/]+)?/$" :dir
#"/settings.yaml$" :settings
#"/export.log$" :log
- #"/collections/metabots/(.*)\.yaml$" :metabot
- #"/collections/.*/cards/(.*)\.yaml$" :card
- #"/collections/.*/dashboards/(.*)\.yaml$" :dashboard
- #"/collections/.*collection/([^/]*)\.yaml$" :collection
- #"/collections/([^/]*)\.yaml$" :collection
- #"/snippets/(.*)\.yaml" :snippet
+ #"/collections/(.*)\.yaml$" :collection-entity
#"/databases/.*/schemas/(.*)" :schema
#"/databases/(.*)\.yaml" :database
#"/transforms/(.*)\.yaml" :transform
- #"/python-libraries/(.*)\.yaml" :python-library])
+ #"/python_libraries/(.*)\.yaml" :python-library])
(defn- file-type
"Find out entity type by file path"
@@ -128,34 +123,33 @@
(testing "We can export just a single collection"
(let [f (mt/user-http-request :crowberto :post 200 "ee/serialization/export" {}
:collection (:id coll) :data_model false :settings false)]
- (is (= #{:log :dir :dashboard :card :collection :transform :python-library}
+ (is (= #{:log :dir :collection-entity :transform :python-library}
(tar-file-types f)))))
(testing "We can export two collections"
(let [f (mt/user-http-request :crowberto :post 200 "ee/serialization/export" {}
:collection (:id coll) :collection (:id coll2)
:data_model false :settings false)]
- (is (= 2
- (->> (tar-file-types f true)
- (filter #(= :collection (first %)))
- count)))))
+ (is (some #(= :collection-entity (first %))
+ (tar-file-types f true))
+ "Export should contain collection entities")))
(testing "We can export that collection using entity id"
(let [f (mt/user-http-request :crowberto :post 200 "ee/serialization/export" {}
;; eid:... syntax is kept for backward compat
:collection (str "eid:" (:entity_id coll)) :data_model false :settings false)]
- (is (= #{:log :dir :dashboard :card :collection :transform :python-library}
+ (is (= #{:log :dir :collection-entity :transform :python-library}
(tar-file-types f)))))
(testing "We can export that collection using entity id"
(let [f (mt/user-http-request :crowberto :post 200 "ee/serialization/export" {}
:collection (:entity_id coll) :data_model false :settings false)]
- (is (= #{:log :dir :dashboard :card :collection :transform :python-library}
+ (is (= #{:log :dir :collection-entity :transform :python-library}
(tar-file-types f)))))
(testing "Default export: all-collections, data-model, settings"
(let [f (mt/user-http-request :crowberto :post 200 "ee/serialization/export" {})]
- (is (= #{:transform :log :dir :dashboard :card :collection :settings :schema :database :python-library}
+ (is (= #{:transform :log :dir :collection-entity :settings :schema :database :python-library}
(tar-file-types f)))))
(testing "On exception API returns log"
@@ -302,7 +296,7 @@
(deftest import-collection-reference-error-test
(testing "Import fails with 500 when collection reference is invalid"
- (with-serialization-test-data! [coll _dash card]
+ (with-serialization-test-data! [coll _dash _card]
(let [ba (do-export (:id coll))]
;; Pop the export snowplow event
(snowplow-test/pop-event-data-and-user-id!)
@@ -310,7 +304,7 @@
(mt/with-dynamic-fn-redefs [v2.ingest/ingest-file (let [ingest-file (mt/dynamic-value #'v2.ingest/ingest-file)]
(fn [^File file]
(cond-> (ingest-file file)
- (str/includes? (.getName file) (:entity_id card))
+ (= (.getName file) "frobinate.yaml")
(assoc :collection_id "DoesNotExist"))))]
(let [res (binding [api.serialization/*additive-logging* false]
(mt/user-http-request :crowberto :post 500 "ee/serialization/import"
@@ -333,7 +327,7 @@
(deftest import-continue-on-error-test
(testing "Import with continue_on_error=true succeeds partially despite errors"
- (with-serialization-test-data! [coll _dash card]
+ (with-serialization-test-data! [coll _dash _card]
(let [ba (do-export (:id coll))]
;; Pop the export snowplow event
(snowplow-test/pop-event-data-and-user-id!)
@@ -341,7 +335,7 @@
(mt/with-dynamic-fn-redefs [v2.ingest/ingest-file (let [ingest-file (mt/dynamic-value #'v2.ingest/ingest-file)]
(fn [^File file]
(cond-> (ingest-file file)
- (str/includes? (.getName file) (:entity_id card))
+ (= (.getName file) "frobinate.yaml")
(assoc :collection_id "DoesNotExist"))))]
(let [res (mt/user-http-request :crowberto :post 200 "ee/serialization/import"
{:request-options {:headers {"content-type" "multipart/form-data"}}}
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/cmd_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/cmd_test.clj
index 03848ee9654..d61019730b3 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/cmd_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/cmd_test.clj
@@ -1,7 +1,6 @@
(ns metabase-enterprise.serialization.cmd-test
(:require
[clojure.java.io :as io]
- [clojure.string :as str]
[clojure.test :refer :all]
[metabase-enterprise.serialization.test-util :as ts]
[metabase-enterprise.serialization.v2.extract :as v2.extract]
@@ -34,7 +33,7 @@
(snowplow-test/with-fake-snowplow-collector
(ts/with-random-dump-dir [dump-dir "serdesv2-"]
(let [coll (ts/create! :model/Collection :name "coll")
- card (ts/create! :model/Card :name "card" :collection_id (:id coll))]
+ _card (ts/create! :model/Card :name "card" :collection_id (:id coll))]
(cmd/export dump-dir "--collection" (str (:id coll)) "--no-data-model")
(testing "Snowplow export event was sent"
(is (=? {"event" "serialization"
@@ -93,7 +92,7 @@
;; override that reliably
(with-redefs [v2.ingest/ingest-file (fn [^java.io.File file]
(cond-> (ingest-file file)
- (str/includes? (.getName file) (:entity_id card))
+ (= (.getName file) "card.yaml")
(assoc :collection_id "DoesNotExist")))]
(is (thrown? Exception
(cmd/import dump-dir)))
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj
index fedea9f75be..122e1ed35f1 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj
@@ -4,6 +4,7 @@
[clojure.test :refer :all]
[medley.core :as m]
[metabase-enterprise.serialization.cmd :as cmd]
+ [metabase-enterprise.serialization.dump :as dump]
[metabase-enterprise.serialization.test-util :as ts]
[metabase-enterprise.serialization.v2.extract :as extract]
[metabase-enterprise.serialization.v2.ingest :as ingest]
@@ -53,13 +54,10 @@
(filter #(-> % :serdes/meta last :model (= model-name))
entities))
-(defn- collections [dir]
- (for [coll-dir (subdirs dir)
- :when (->> ["cards" "dashboards" "timelines"]
- (map #(io/file coll-dir %))
- (filter #(= % coll-dir))
- empty?)]
- coll-dir))
+(defn- yaml-model-at
+ "Read a YAML file and return its serdes model name."
+ [^File base-dir path-vec]
+ (-> (apply io/file base-dir path-vec) yaml/from-file :serdes/meta last :model))
(defn- file-set [^File dir]
(let [^Path base (.toPath dir)]
@@ -220,50 +218,44 @@
(testing "for Collections"
;; +1 for the Trash collection
- (is (= 110 (count (for [f (file-set (io/file dump-dir))
- :when (and (= (first f) "collections")
- (let [[a b] (take-last 2 f)]
- (= b (str a ".yaml"))))]
- f)))
- "which all go in collections/, even the snippets ones"))
+ (let [colls-dir (io/file dump-dir "collections")
+ coll-count (count (for [f (file-set colls-dir)
+ :when (= "Collection" (yaml-model-at colls-dir f))]
+ f))]
+ ;; +1 for Trash collection; exact count may vary by 1 depending on naming collisions
+ (is (<= 109 coll-count 111)
+ "which all go in collections/, even the snippets ones")))
(testing "for Databases"
(is (= 10 (count (dir->dir-set (io/file dump-dir "databases"))))))
(testing "for Tables"
(is (= 100
- (reduce + (for [db (get @entities "Database")
- :let [tables (dir->dir-set (io/file dump-dir "databases" (:name db) "tables"))]]
- (count tables))))
+ (reduce + (for [db (dir->dir-set (io/file dump-dir "databases"))
+ :let [tables-dir (io/file dump-dir "databases" db "tables")]
+ :when (.exists tables-dir)]
+ (count (dir->dir-set tables-dir)))))
"Tables are scattered, so the directories are harder to count"))
(testing "for Fields"
(is (= 1000
- (reduce + (for [db (get @entities "Database")
- table (subdirs (io/file dump-dir "databases" (:name db) "tables"))]
- (->> (io/file table "fields")
- dir->file-set
- count))))
+ (reduce + (for [db (dir->dir-set (io/file dump-dir "databases"))
+ table (subdirs (io/file dump-dir "databases" db "tables"))
+ :let [fields-dir (io/file table "fields")]
+ :when (.exists fields-dir)]
+ (count (dir->file-set fields-dir)))))
"Fields are scattered, so the directories are harder to count"))
- (testing "for cards"
- ;; 100 from card, and 10 from simple-model
- (is (= 110 (->> (io/file dump-dir "collections")
- collections
- (map (comp count dir->file-set #(io/file % "cards")))
- (reduce +)))))
-
- (testing "for dashboards"
- (is (= 150 (->> (io/file dump-dir "collections")
- collections
- (map (comp count dir->file-set #(io/file % "dashboards")))
- (reduce +)))))
-
- (testing "for timelines"
- (is (= 10 (->> (io/file dump-dir "collections")
- collections
- (map (comp count dir->file-set #(io/file % "timelines")))
- (reduce +)))))
+ (testing "for cards, dashboards, and timelines"
+ ;; In the new storage format, cards/dashboards/timelines are stored directly
+ ;; in collection directories (no per-type subfolders).
+ ;; 100 cards + 10 simple-models + 150 dashboards + 10 timelines = 270
+ ;; We count all yaml files under collections/main/ that are NOT collection definitions.
+ ;; exact count may vary by 1 depending on naming collisions with collection names
+ (let [main-dir (io/file dump-dir "collections" "main")]
+ (is (<= 269 (count (for [f (file-set main-dir)
+ :when (not= "Collection" (yaml-model-at main-dir f))]
+ f)) 271))))
(testing "for segments"
(is (= 30 (reduce + (for [db (dir->dir-set (io/file dump-dir "databases"))
@@ -273,10 +265,12 @@
(count (dir->file-set segments-dir)))))))
(testing "for native query snippets"
- (is (= 10 (->> (io/file dump-dir "snippets")
- collections
- (map (comp count dir->file-set))
- (reduce +)))))
+ ;; Snippets are now under collections/snippets/ (not a top-level snippets/ dir).
+ ;; Count non-collection yaml files under collections/snippets/.
+ (let [snippets-dir (io/file dump-dir "collections" "snippets")]
+ (is (= 10 (count (for [f (file-set snippets-dir)
+ :when (not= "Collection" (yaml-model-at snippets-dir f))]
+ f))))))
(testing "for settings"
(is (.exists (io/file dump-dir "settings.yaml")))))
@@ -998,3 +992,25 @@
(testing "loads well too"
(is (serdes/with-cache (serdes.load/load-metabase! (ingest/ingest-yaml dump-dir)))
"ingested successfully"))))))
+
+(deftest old-format-paths-import-test
+ (testing "serdes can import content stored at old-format paths (entity_id in name, per-type subfolders)"
+ (ts/with-random-dump-dir [dump-dir "serdesv2-"]
+ (mt/with-empty-h2-app-db!
+ ;; Write YAML files using the OLD path format (entity_id_slug, cards/ subfolder)
+ (let [coll-eid "old-test-coll-xxxxxxx"
+ coll-dir (io/file dump-dir "collections" (str coll-eid "_test_collection"))
+ coll-yaml {:serdes/meta [{:model "Collection" :id coll-eid :label "test_collection"}]
+ :name "Test Collection" :entity_id coll-eid :namespace nil
+ :slug "test_collection" :archived false}]
+ ;; Old format: collections/entityid_slug/entityid_slug.yaml
+ (dump/spit-yaml! (io/file coll-dir (str coll-eid "_test_collection.yaml")) coll-yaml)
+ ;; Write settings.yaml (required)
+ (dump/spit-yaml! (io/file dump-dir "settings.yaml") {})
+
+ (testing "old-format files can be ingested and loaded"
+ (let [ingestable (ingest/ingest-yaml dump-dir)]
+ (is (serdes/with-cache (serdes.load/load-metabase! ingestable))
+ "ingestion should succeed")
+ (is (t2/exists? :model/Collection :entity_id coll-eid)
+ "collection should have been imported from old-format path"))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/storage_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/storage_test.clj
index af96f87dfcf..bee0d14297e 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/storage_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/storage_test.clj
@@ -27,15 +27,13 @@
(mt/with-empty-h2-app-db!
(ts/with-temp-dpc [:model/Collection parent {:name "Some Collection"}
:model/Collection child {:name "Child Collection" :location (format "/%d/" (:id parent))}]
- (let [export (into [] (extract/extract {:no-transforms true}))
- parent-filename (format "%s_some_collection" (:entity_id parent))
- child-filename (format "%s_child_collection" (:entity_id child))]
+ (let [export (into [] (extract/extract {:no-transforms true}))]
(storage/store! export dump-dir)
(testing "the right files in the right places"
- (is (= #{[parent-filename (str parent-filename ".yaml")]
- [parent-filename child-filename (str child-filename ".yaml")]}
+ (is (= #{["main" "some_collection.yaml"]
+ ["main" "some_collection" "child_collection.yaml"]}
(file-set (io/file dump-dir "collections")))
- "collections form a tree, with same-named files")
+ "collections form a tree, with files as siblings of their content folders")
(is (contains? (file-set (io/file dump-dir))
["settings.yaml"])
"A few top-level files are expected"))
@@ -45,7 +43,8 @@
(dissoc :id :location)
(assoc :parent_id nil)
(update :created_at t/offset-date-time))
- (-> (yaml/from-file (io/file dump-dir "collections" parent-filename (str parent-filename ".yaml")))
+ (-> (yaml/from-file (io/file dump-dir "collections" "main"
+ "some_collection.yaml"))
(dissoc :serdes/meta)
(update :created_at t/offset-date-time))))
@@ -53,8 +52,8 @@
(dissoc :id :location)
(assoc :parent_id (:entity_id parent))
(update :created_at t/offset-date-time))
- (-> (yaml/from-file (io/file dump-dir "collections" parent-filename
- child-filename (str child-filename ".yaml")))
+ (-> (yaml/from-file (io/file dump-dir "collections" "main"
+ "some_collection" "child_collection.yaml"))
(dissoc :serdes/meta)
(update :created_at t/offset-date-time))))))))))
@@ -67,26 +66,23 @@
:location (str "/" (:id grandparent) "/")}
:model/Collection child {:name "Child Collection"
:location (str "/" (:id grandparent) "/" (:id parent) "/")}
- :model/Card c1 {:name "root card" :collection_id nil}
- :model/Card c2 {:name "grandparent card" :collection_id (:id grandparent)}
- :model/Card c3 {:name "parent card" :collection_id (:id parent)}
- :model/Card c4 {:name "child card" :collection_id (:id child)}
- :model/Dashboard d1 {:name "parent dash" :collection_id (:id parent)}]
+ :model/Card _c1 {:name "root card" :collection_id nil}
+ :model/Card _c2 {:name "grandparent card" :collection_id (:id grandparent)}
+ :model/Card _c3 {:name "parent card" :collection_id (:id parent)}
+ :model/Card _c4 {:name "child card" :collection_id (:id child)}
+ :model/Dashboard _d1 {:name "parent dash" :collection_id (:id parent)}]
(let [export (into [] (extract/extract {:no-transforms true}))]
(storage/store! export dump-dir)
(testing "the right files in the right places"
- (let [gp-dir (str (:entity_id grandparent) "_grandparent_collection")
- p-dir (str (:entity_id parent) "_parent_collection")
- c-dir (str (:entity_id child) "_child_collection")]
- (is (= #{[gp-dir (str gp-dir ".yaml")] ; Grandparent collection
- [gp-dir p-dir (str p-dir ".yaml")] ; Parent collection
- [gp-dir p-dir c-dir (str c-dir ".yaml")] ; Child collection
- ["cards" (str (:entity_id c1) "_root_card.yaml")] ; Root card
- [gp-dir "cards" (str (:entity_id c2) "_grandparent_card.yaml")] ; Grandparent card
- [gp-dir p-dir "cards" (str (:entity_id c3) "_parent_card.yaml")] ; Parent card
- [gp-dir p-dir c-dir "cards" (str (:entity_id c4) "_child_card.yaml")] ; Child card
- [gp-dir p-dir "dashboards" (str (:entity_id d1) "_parent_dash.yaml")]} ; Parent dashboard
- (file-set (io/file dump-dir "collections")))))))))))
+ (is (= #{["main" "grandparent_collection.yaml"]
+ ["main" "grandparent_collection" "parent_collection.yaml"]
+ ["main" "grandparent_collection" "parent_collection" "child_collection.yaml"]
+ ["main" "root_card.yaml"]
+ ["main" "grandparent_collection" "grandparent_card.yaml"]
+ ["main" "grandparent_collection" "parent_collection" "parent_card.yaml"]
+ ["main" "grandparent_collection" "parent_collection" "child_collection" "child_card.yaml"]
+ ["main" "grandparent_collection" "parent_collection" "parent_dash.yaml"]}
+ (file-set (io/file dump-dir "collections"))))))))))
(deftest snippets-collections-nesting-test
(ts/with-random-dump-dir [dump-dir "serdesv2-"]
@@ -100,28 +96,23 @@
:model/Collection child {:name "Child Collection"
:namespace :snippets
:location (str "/" (:id grandparent) "/" (:id parent) "/")}
- :model/NativeQuerySnippet c1 {:name "root snippet" :collection_id nil}
- :model/NativeQuerySnippet c2 {:name "grandparent snippet" :collection_id (:id grandparent)}
- :model/NativeQuerySnippet c3 {:name "parent snippet" :collection_id (:id parent)}
- :model/NativeQuerySnippet c4 {:name "child snippet" :collection_id (:id child)}]
+ :model/NativeQuerySnippet _c1 {:name "root snippet" :collection_id nil}
+ :model/NativeQuerySnippet _c2 {:name "grandparent snippet" :collection_id (:id grandparent)}
+ :model/NativeQuerySnippet _c3 {:name "parent snippet" :collection_id (:id parent)}
+ :model/NativeQuerySnippet _c4 {:name "child snippet" :collection_id (:id child)}]
(let [export (into [] (extract/extract {:no-settings true
:no-data-model true
:no-transforms true}))]
(storage/store! export dump-dir)
- (let [gp-dir (str (:entity_id grandparent) "_grandparent_collection")
- p-dir (str (:entity_id parent) "_parent_collection")
- c-dir (str (:entity_id child) "_child_collection")]
- (testing "collections under collections/"
- (is (= #{[gp-dir (str gp-dir ".yaml")] ; Grandparent collection
- [gp-dir p-dir (str p-dir ".yaml")] ; Parent collection
- [gp-dir p-dir c-dir (str c-dir ".yaml")]} ; Child collection
- (file-set (io/file dump-dir "collections")))))
- (testing "snippets under snippets/"
- (is (= #{[(str (:entity_id c1) "_root_snippet.yaml")] ; Root snippet
- [gp-dir (str (:entity_id c2) "_grandparent_snippet.yaml")] ; Grandparent snippet
- [gp-dir p-dir (str (:entity_id c3) "_parent_snippet.yaml")] ; Parent snippet
- [gp-dir p-dir c-dir (str (:entity_id c4) "_child_snippet.yaml")]} ; Child snippet
- (file-set (io/file dump-dir "snippets")))))))))))
+ (testing "all snippet collections and snippets under collections/snippets/"
+ (is (= #{["snippets" "grandparent_collection.yaml"]
+ ["snippets" "grandparent_collection" "parent_collection.yaml"]
+ ["snippets" "grandparent_collection" "parent_collection" "child_collection.yaml"]
+ ["snippets" "root_snippet.yaml"]
+ ["snippets" "grandparent_collection" "grandparent_snippet.yaml"]
+ ["snippets" "grandparent_collection" "parent_collection" "parent_snippet.yaml"]
+ ["snippets" "grandparent_collection" "parent_collection" "child_collection" "child_snippet.yaml"]}
+ (file-set (io/file dump-dir "collections"))))))))))
(deftest embedded-slash-test
(ts/with-random-dump-dir [dump-dir "serdesv2-"]
@@ -134,20 +125,20 @@
(let [export (into [] (extract/extract {:include-field-values true}))]
(storage/store! export dump-dir)
(testing "the right files in the right places"
- (is (= #{["Company__SLASH__organization website.yaml"]
- ["Company__SLASH__organization website___fieldvalues.yaml"]}
- (file-set (io/file dump-dir "databases" "My Company Data" "tables" "Customers" "fields")))
+ (is (= #{["company__SLASH__organization_website.yaml"]
+ ["company__SLASH__organization_website___fieldvalues.yaml"]}
+ (file-set (io/file dump-dir "databases" "my_company_data" "tables" "customers" "fields")))
"Slashes in file names get escaped")
- (is (contains? (file-set (io/file dump-dir "databases" "My Company Data" "tables"))
- ["Orders__SLASH__Invoices" "Orders__SLASH__Invoices.yaml"])
+ (is (contains? (file-set (io/file dump-dir "databases" "my_company_data" "tables"))
+ ["orders__SLASH__invoices" "orders__SLASH__invoices.yaml"])
"Slashes in directory names get escaped"))
(testing "the Field was properly exported"
(is (= (ts/extract-one "Field" (:id website))
(-> (yaml/from-file (io/file dump-dir
- "databases" "My Company Data"
- "tables" "Customers"
- "fields" "Company__SLASH__organization website.yaml"))
+ "databases" "my_company_data"
+ "tables" "customers"
+ "fields" "company__SLASH__organization_website.yaml"))
(update :visibility_type keyword)
(update :base_type keyword))))))))))
@@ -242,12 +233,12 @@
(ts/with-random-dump-dir [dump-dir "serdesv2-"]
(mt/with-empty-h2-app-db!
(t2/delete! :model/PythonLibrary)
- (ts/with-temp-dpc [:model/PythonLibrary lib {:path "common" :source "def test(): pass"}]
+ (ts/with-temp-dpc [:model/PythonLibrary _lib {:path "common" :source "def test(): pass"}]
(let [export (into [] (extract/extract {:no-settings true :no-data-model true}))]
(storage/store! export dump-dir)
- (testing "python library stored at top-level python-libraries/"
- (is (= #{[(str (:entity_id lib) ".yaml")]}
- (file-set (io/file dump-dir "python-libraries"))))))))))
+ (testing "python library stored at top-level python_libraries/"
+ (is (= #{["common.py.yaml"]}
+ (file-set (io/file dump-dir "python_libraries"))))))))))
(deftest name-too-long-test
(ts/with-random-dump-dir [dump-dir "serdesv2-"]
@@ -258,11 +249,88 @@
:no-transforms true
:targets [["Card" (:id card)]]}))
;; 66 is 'char-count * max-bytes / byte-count'
- card-filename (format "%s_%s" (:entity_id card) (str/join (repeat 66 "ป")))]
+ card-filename (str/join (repeat 66 "ป"))]
(storage/store! export dump-dir)
;; we could also test loading here, but file names do not play significant part in how everything's loaded,
;; `:serdes/meta` does and that one is not shortened or anything
(testing "the right files in the right places"
- (is (= #{["cards" (str card-filename ".yaml")]}
+ (is (= #{["main" (str card-filename ".yaml")]}
(file-set (io/file dump-dir "collections")))
"collections form a tree, with same-named files"))))))
+
+(deftest ^:parallel resolve-path-test
+ (let [resolve-path @#'storage/resolve-path]
+ (testing "basic slugification"
+ (let [fns (atom {})]
+ (is (= ["my_collection" "some_card"]
+ (resolve-path fns [{:label "My Collection" :key "coll-1"}
+ {:label "Some Card" :key "card-1"}])))))
+
+ (testing "special characters are replaced with underscores"
+ (let [fns (atom {})]
+ (is (= ["hello_world_"]
+ (resolve-path fns [{:label "Hello World!" :key "a"}])))))
+
+ (testing "slashes are escaped"
+ (let [fns (atom {})]
+ (is (= ["orders__SLASH__invoices"]
+ (resolve-path fns [{:label "Orders/Invoices" :key "a"}])))))
+
+ (testing "backslashes are escaped"
+ (let [fns (atom {})]
+ (is (= ["c__BACKSLASH__d"]
+ (resolve-path fns [{:label "C\\D" :key "a"}])))))
+
+ (testing "deduplication within the same folder"
+ (let [fns (atom {})]
+ (is (= ["my_card"]
+ (resolve-path fns [{:label "My Card" :key "card-1"}])))
+ (is (= ["my_card_2"]
+ (resolve-path fns [{:label "My Card" :key "card-2"}]))
+ "second entity with same name in same folder gets _2 suffix")))
+
+ (testing "same name in different folders does not conflict"
+ (let [fns (atom {})]
+ (is (= ["folder_a" "readme"]
+ (resolve-path fns [{:label "Folder A" :key "f-a"}
+ {:label "README" :key "doc-1"}])))
+ (is (= ["folder_b" "readme"]
+ (resolve-path fns [{:label "Folder B" :key "f-b"}
+ {:label "README" :key "doc-2"}]))
+ "same leaf name under different parents is fine")))
+
+ (testing "same key with same slug is stable"
+ (let [fns (atom {})]
+ (is (= ["my_card"]
+ (resolve-path fns [{:label "My Card" :key "card-1"}])))
+ (is (= ["my_card"]
+ (resolve-path fns [{:label "My Card" :key "card-1"}]))
+ "re-resolving the same key+label returns the same result")))
+
+ (testing "unicode is preserved"
+ (let [fns (atom {})]
+ (is (= ["données"]
+ (resolve-path fns [{:label "Données" :key "a"}])))))
+
+ (testing "dots are preserved"
+ (let [fns (atom {})]
+ (is (= ["parent.child"]
+ (resolve-path fns [{:label "parent.child" :key "a"}])))))
+
+ (testing "duplicate parent folder names with different keys"
+ (let [fns (atom {})]
+ (is (= ["my_folder" "card_a"]
+ (resolve-path fns [{:label "My Folder" :key "folder-1"}
+ {:label "Card A" :key "card-a"}])))
+ (is (= ["my_folder_2" "card_b"]
+ (resolve-path fns [{:label "My Folder" :key "folder-2"}
+ {:label "Card B" :key "card-b"}]))
+ "second folder with same name gets _2 suffix")
+ (is (= ["my_folder_3" "card_c"]
+ (resolve-path fns [{:label "My Folder" :key "folder-3"}
+ {:label "Card C" :key "card-c"}]))
+ "third folder with same name gets _3 suffix")))
+
+ (testing "empty path returns empty vector"
+ (let [fns (atom {})]
+ (is (= [] (resolve-path fns [])))))))
diff --git a/src/metabase/actions/models.clj b/src/metabase/actions/models.clj
index 17874263dad..d2bcd59b4e3 100644
--- a/src/metabase/actions/models.clj
+++ b/src/metabase/actions/models.clj
@@ -469,8 +469,7 @@
(serdes/mbql-deps dataset_query)))))))
(defmethod serdes/storage-path "Action" [action _ctx]
- (let [{:keys [id label]} (-> action serdes/path last)]
- ["actions" (serdes/storage-leaf-file-name id label)]))
+ [{:label "actions"} {:label (:name action) :key (:entity_id action)}])
;;;; ------------------------------------------------- Search ----------------------------------------------------------
diff --git a/src/metabase/collections/models/collection.clj b/src/metabase/collections/models/collection.clj
index cbc41a9a8a7..b8036da9f83 100644
--- a/src/metabase/collections/models/collection.clj
+++ b/src/metabase/collections/models/collection.clj
@@ -2068,8 +2068,13 @@
(merge child-colls dashboards cards documents timelines tables transforms)))
(defmethod serdes/storage-path "Collection" [coll {:keys [collections]}]
- (let [parental (get collections (:entity_id coll))]
- (concat ["collections"] parental [(last parental)])))
+ (let [path (get collections (:entity_id coll))
+ ns-folder (case (:namespace coll)
+ :snippets "snippets"
+ :transforms "transforms"
+ nil "main"
+ "main")]
+ (into [{:label "collections"} {:label ns-folder}] path)))
(defn- parent-id->location-path [parent-id]
(if-not parent-id
diff --git a/src/metabase/glossary/models/glossary.clj b/src/metabase/glossary/models/glossary.clj
index dec37bc24eb..04407ea09e6 100644
--- a/src/metabase/glossary/models/glossary.clj
+++ b/src/metabase/glossary/models/glossary.clj
@@ -49,4 +49,4 @@
:creator_id (serdes/fk :model/User)}})
(defmethod serdes/storage-path "Glossary" [item _]
- ["glossary" (:term item)])
+ [{:label "glossary"} {:label (:term item) :key (:term item)}])
diff --git a/src/metabase/measures/models/measure.clj b/src/metabase/measures/models/measure.clj
index 43a2e7a4b9d..15cb363338c 100644
--- a/src/metabase/measures/models/measure.clj
+++ b/src/metabase/measures/models/measure.clj
@@ -195,12 +195,8 @@
table_id (conj (serdes/table->path table_id))))
(defmethod serdes/storage-path "Measure" [measure _ctx]
- (let [{:keys [id label]} (-> measure serdes/path last)]
- (-> measure
- :table_id
- serdes/table->path
- serdes/storage-path-prefixes
- (concat ["measures" (serdes/storage-leaf-file-name id label)]))))
+ (into (-> measure :table_id serdes/table->path serdes/storage-path-prefixes)
+ [{:label "measures"} {:label (:name measure) :key (:entity_id measure)}]))
(defn- import-measure-definition
"Import a measure definition from serialization format.
diff --git a/src/metabase/models/serialization.clj b/src/metabase/models/serialization.clj
index 338dd6c5b32..9c809930545 100644
--- a/src/metabase/models/serialization.clj
+++ b/src/metabase/models/serialization.clj
@@ -73,7 +73,6 @@
[metabase.util.date-2 :as u.date]
[metabase.util.json :as json]
[metabase.util.log :as log]
- [metabase.util.string :as u.str]
[toucan2.core :as t2]
[toucan2.model :as t2.model]
[toucan2.realize :as t2.realize]))
@@ -851,34 +850,21 @@
(t2/select-one model :entity_id id-str)
(find-by-identity-hash model id-str)))
-(def ^:private max-label-length 100)
-(def ^:private max-label-bytes 200) ;; 255 is a limit in ext4
-
-(defn- truncate-label [^String s]
- (-> s
- (u.str/limit-bytes max-label-bytes)
- (u.str/limit-chars max-label-length)))
-
-(defn- lower-plural [s]
- (-> s u/lower-case-en (str "s")))
-
-(defn storage-leaf-file-name
- "Captures the common pattern for leaf file names as `entityID_label`."
- ([id] (str id))
- ([id label] (if (nil? label)
- (storage-leaf-file-name id)
- (str id "_" (truncate-label label)))))
-
(defn storage-default-collection-path
- "Implements the most common structure for [[storage-path]] - `collections/c1/c2/c3/models/entityid_label.ext`"
- [entity {:keys [collections]}]
- (let [{:keys [model id label]} (-> entity path last)]
- (concat ["collections"]
- (get collections (:collection_id entity)) ;; This can be nil, but that's fine - that's the root collection.
- [(lower-plural model) (storage-leaf-file-name id label)])))
+ "Implements the most common structure for [[storage-path]].
+ Returns a vector of maps with `:label` and `:key` for each path segment.
+ Result: `[{:label \"collections\"} {:label ns-folder} <collection-hierarchy> {:label entity-name :key entity_id}]`"
+ ([entity ctx]
+ (storage-default-collection-path entity ctx "main"))
+ ([entity {:keys [collections]} ns-folder]
+ (into [{:label "collections"} {:label ns-folder}]
+ cat [(get collections (:collection_id entity))
+ [{:label (:name entity) :key (:entity_id entity)}]])))
(defmulti storage-path
- "Returns a seq of storage path components for a given entity. Dispatches on model name."
+ "Returns a vector of maps with `:label` and optional `:key` for each path segment.
+ `:label` is the human-readable name; `:key` is a deduplication identity (entity_id, name, or nil).
+ Dispatches on model name."
{:arglists '([entity ctx])}
(fn [entity _] (ingested-model entity)))
@@ -886,16 +872,35 @@
(storage-default-collection-path entity ctx))
(defn storage-base-context
- "Creates the basic context for storage. This is a map with a single entry: `:collections` is a map from collection ID
- to the path of collections."
+ "Creates the basic context for storage.
+ - `:collections` maps collection entity_id to a vector of `{:label ... :key ...}` maps representing
+ the collection hierarchy.
+ - `:dashboards` maps dashboard entity_id to `{:label ... :key ...}` for use as virtual subcollections.
+ - `:documents` maps document entity_id to `{:label ... :key ...}` for use as virtual subcollections.
+ - `:unique-name-fns` is an atom of `{parent-key -> unique-name-fn}` where each `unique-name-fn` is a
+ `lib/non-truncating-unique-name-generator`, used to deduplicate names within the same folder during export."
[]
- (let [colls (t2/select ['Collection :id :entity_id :location :slug])
- coll-names (into {} (for [{:keys [id entity_id slug]} colls]
- [(str id) (storage-leaf-file-name entity_id slug)]))
- coll->path (into {} (for [{:keys [entity_id id location]} colls
- :let [parents (rest (str/split location #"/"))]]
- [entity_id (map coll-names (concat parents [(str id)]))]))]
- {:collections coll->path}))
+ (let [colls (t2/select ['Collection :id :entity_id :location :name])
+ id->coll (into {} (for [{:keys [id] :as coll} colls] [(str id) coll]))
+ coll->path (into {}
+ (for [{:keys [entity_id id location]} colls
+ :let [parent-ids (rest (str/split location #"/"))
+ all-ids (concat parent-ids [(str id)])
+ path-maps (mapv (fn [cid]
+ (let [c (id->coll cid)]
+ {:label (:name c) :key (:entity_id c)}))
+ all-ids)]]
+ [entity_id path-maps]))
+ dashboards (into {}
+ (for [{:keys [entity_id name]} (t2/select ['Dashboard :entity_id :name])]
+ [entity_id {:label name :key entity_id}]))
+ documents (into {}
+ (for [{:keys [entity_id name]} (t2/select ['Document :entity_id :name])]
+ [entity_id {:label name :key entity_id}]))]
+ {:collections coll->path
+ :dashboards dashboards
+ :documents documents
+ :unique-name-fns (atom {})}))
;;; # Utilities for implementing serdes
;;; Note that many of these use `^::cache` to cache their lookups during deserialization. This greatly reduces the
@@ -1040,14 +1045,14 @@
Takes the :serdes/meta value for a `Table`!
The return value includes the directory for the Table, but not the file for the Table itself.
- With a schema: `[\"databases\" \"db_name\" \"schemas\" \"public\" \"tables\" \"customers\"]`
- No schema: `[\"databases\" \"db_name\" \"tables\" \"customers\"]`"
+ With a schema: `[{:label \"databases\"} {:label \"db_name\" :key \"db_name\"} {:label \"schemas\"} {:label \"public\" :key \"public\"} {:label \"tables\"} {:label \"customers\" :key \"customers\"}]`
+ No schema: `[{:label \"databases\"} {:label \"db_name\" :key \"db_name\"} {:label \"tables\"} {:label \"customers\" :key \"customers\"}]`"
[path]
- (into [] cat
- (for [entry path]
- [(or (get storage-dirs (:model entry))
- (throw (ex-info "Could not find dir name" {:entry entry})))
- (:id entry)])))
+ (into [] (mapcat (fn [entry]
+ [{:label (or (get storage-dirs (:model entry))
+ (throw (ex-info "Could not find dir name" {:entry entry})))}
+ {:label (:id entry) :key (:id entry)}]))
+ path))
;;; ## Fields
diff --git a/src/metabase/models/transforms/transform.clj b/src/metabase/models/transforms/transform.clj
index a13b2fed7c5..fbc98b696b3 100644
--- a/src/metabase/models/transforms/transform.clj
+++ b/src/metabase/models/transforms/transform.clj
@@ -379,12 +379,7 @@
(serdes/mbql-deps source))))
(defmethod serdes/storage-path "Transform" [transform ctx]
- ;; Path: ["collections" "<nested ... collections>" "transforms" "<entity_id_name>"]
- ;; Use default collection path, then restructure similar to NativeQuerySnippet
- (let [basis (serdes/storage-default-collection-path transform ctx)
- file (last basis)
- colls (->> basis rest (drop-last 2))] ; Drop "collections" at start, and last two elements
- (concat ["collections"] colls ["transforms" file])))
+ (serdes/storage-default-collection-path transform ctx "transforms"))
(defmethod serdes/required "Transform"
[_model id]
diff --git a/src/metabase/models/transforms/transform_job.clj b/src/metabase/models/transforms/transform_job.clj
index 44c1ea06ae6..e80a20ca224 100644
--- a/src/metabase/models/transforms/transform_job.clj
+++ b/src/metabase/models/transforms/transform_job.clj
@@ -198,5 +198,4 @@
[{:model "TransformTag" :id tag-id}])))
(defmethod serdes/storage-path "TransformJob" [job _ctx]
- (let [{:keys [id label]} (-> job serdes/path last)]
- ["transforms" "transform_jobs" (serdes/storage-leaf-file-name id label)]))
+ [{:label "transforms"} {:label "transform_jobs"} {:label (:name job) :key (:entity_id job)}])
diff --git a/src/metabase/models/transforms/transform_tag.clj b/src/metabase/models/transforms/transform_tag.clj
index c2602b92e58..734b4d1c09a 100644
--- a/src/metabase/models/transforms/transform_tag.clj
+++ b/src/metabase/models/transforms/transform_tag.clj
@@ -105,8 +105,7 @@
:created_at (serdes/date)}})
(defmethod serdes/storage-path "TransformTag" [tt _ctx]
- (let [{:keys [id label]} (-> tt serdes/path last)]
- ["transforms" "transform_tags" (serdes/storage-leaf-file-name id label)]))
+ [{:label "transforms"} {:label "transform_tags"} {:label (:name tt) :key (:entity_id tt)}])
;; Event hooks for remote-sync tracking
(t2/define-after-insert :model/TransformTag [tag]
diff --git a/src/metabase/native_query_snippets/models/native_query_snippet.clj b/src/metabase/native_query_snippets/models/native_query_snippet.clj
index d6835f00c3d..68c2014e335 100644
--- a/src/metabase/native_query_snippets/models/native_query_snippet.clj
+++ b/src/metabase/native_query_snippets/models/native_query_snippet.clj
@@ -184,13 +184,7 @@
[[{:model "Collection" :id collection_id}]]))
(defmethod serdes/storage-path "NativeQuerySnippet" [snippet ctx]
- ;; Intended path here is ["snippets" "<nested ... collections>" "<snippet_eid_and_slug>"]
- ;; We just the default path, then pull it apart.
- ;; The default is ["collections" "<nested ... collections>" "nativequerysnippets" "<base_name>"]
- (let [basis (serdes/storage-default-collection-path snippet ctx)
- file (last basis)
- colls (->> basis rest (drop-last 2))] ; Drops the "collections" at the start, and the last two.
- (concat ["snippets"] colls [file])))
+ (serdes/storage-default-collection-path snippet ctx "snippets"))
(defmethod serdes/load-one! "NativeQuerySnippet" [ingested maybe-local]
;; if we got local snippet in db and it has same name as incoming one, we can be sure
diff --git a/src/metabase/queries/models/card.clj b/src/metabase/queries/models/card.clj
index 9ca8e71a6e3..9c078e9dd70 100644
--- a/src/metabase/queries/models/card.clj
+++ b/src/metabase/queries/models/card.clj
@@ -1281,6 +1281,23 @@
(serdes/field->path (:fk_target_field_id m))))))
(disj nil))))
+(defmethod serdes/storage-path "Card" [card ctx]
+ (let [base (serdes/storage-default-collection-path card ctx)]
+ (cond
+ (:dashboard_id card)
+ (let [dash-segment (get (:dashboards ctx) (:dashboard_id card))]
+ (if dash-segment
+ (into (vec (butlast base)) [dash-segment (last base)])
+ base))
+
+ (:document_id card)
+ (let [doc-segment (get (:documents ctx) (:document_id card))]
+ (if doc-segment
+ (into (vec (butlast base)) [doc-segment (last base)])
+ base))
+
+ :else base)))
+
(defmethod serdes/make-spec "Card"
[_model-name _opts]
{:copy [:archived :archived_directly :collection_position :collection_preview :description :display
diff --git a/src/metabase/segments/models/segment.clj b/src/metabase/segments/models/segment.clj
index aeb6c999fe6..565e9d5cb4c 100644
--- a/src/metabase/segments/models/segment.clj
+++ b/src/metabase/segments/models/segment.clj
@@ -222,12 +222,8 @@
table_id (conj (serdes/table->path table_id))))
(defmethod serdes/storage-path "Segment" [segment _ctx]
- (let [{:keys [id label]} (-> segment serdes/path last)]
- (-> segment
- :table_id
- serdes/table->path
- serdes/storage-path-prefixes
- (concat ["segments" (serdes/storage-leaf-file-name id label)]))))
+ (into (-> segment :table_id serdes/table->path serdes/storage-path-prefixes)
+ [{:label "segments"} {:label (:name segment) :key (:entity_id segment)}]))
(defmethod serdes/make-spec "Segment" [_model-name _opts]
{:copy [:name :points_of_interest :archived :caveats :description :entity_id :show_in_getting_started]
diff --git a/src/metabase/warehouse_schema/models/field.clj b/src/metabase/warehouse_schema/models/field.clj
index 5353d2e1d78..2c66b2478fa 100644
--- a/src/metabase/warehouse_schema/models/field.clj
+++ b/src/metabase/warehouse_schema/models/field.clj
@@ -468,6 +468,8 @@
:dimensions (serdes/nested :model/Dimension :field_id opts)}})
(defmethod serdes/storage-path "Field" [field _]
- (let [[path fields] (split-with #(not= "Field" (:model %)) (serdes/path field))]
- (concat (serdes/storage-path-prefixes path)
- ["fields" (str/join "." (map :id fields))])))
+ (let [[path fields] (split-with #(not= "Field" (:model %)) (serdes/path field))
+ field-name (str/join "." (map :id fields))]
+ (conj (serdes/storage-path-prefixes path)
+ {:label "fields"}
+ {:label field-name :key field-name})))
diff --git a/src/metabase/warehouse_schema/models/field_user_settings.clj b/src/metabase/warehouse_schema/models/field_user_settings.clj
index 82f57a4c1ef..67c444ab9eb 100644
--- a/src/metabase/warehouse_schema/models/field_user_settings.clj
+++ b/src/metabase/warehouse_schema/models/field_user_settings.clj
@@ -80,4 +80,5 @@
;; don't have their own directories.
(let [hierarchy (serdes/path fv)
field-path (serdes/storage-path-prefixes (drop-last hierarchy))]
- (update field-path (dec (count field-path)) str field-values-slug)))
+ (update field-path (dec (count field-path))
+ (fn [segment] (update segment :label str field-values-slug)))))
diff --git a/src/metabase/warehouse_schema/models/field_values.clj b/src/metabase/warehouse_schema/models/field_values.clj
index cb86bcaed6d..c2359ce1285 100644
--- a/src/metabase/warehouse_schema/models/field_values.clj
+++ b/src/metabase/warehouse_schema/models/field_values.clj
@@ -619,4 +619,5 @@
;; don't have their own directories.
(let [hierarchy (serdes/path fv)
field-path (serdes/storage-path-prefixes (drop-last hierarchy))]
- (update field-path (dec (count field-path)) str field-values-slug)))
+ (update field-path (dec (count field-path))
+ (fn [segment] (update segment :label str field-values-slug)))))
diff --git a/src/metabase/warehouse_schema/models/table.clj b/src/metabase/warehouse_schema/models/table.clj
index a80255fa728..ccbf59e8317 100644
--- a/src/metabase/warehouse_schema/models/table.clj
+++ b/src/metabase/warehouse_schema/models/table.clj
@@ -705,8 +705,8 @@
:transform_id (serdes/fk :model/Transform)}})
(defmethod serdes/storage-path "Table" [table _ctx]
- (concat (serdes/storage-path-prefixes (serdes/path table))
- [(:name table)]))
+ (conj (serdes/storage-path-prefixes (serdes/path table))
+ {:label (:name table) :key (:name table)}))
;;;; ------------------------------------------------- Search ----------------------------------------------------------
diff --git a/src/metabase/warehouses/models/database.clj b/src/metabase/warehouses/models/database.clj
index d2295efb456..fb9bbeba35e 100644
--- a/src/metabase/warehouses/models/database.clj
+++ b/src/metabase/warehouses/models/database.clj
@@ -689,8 +689,8 @@
(t2/select-one :model/Database :name id))
(defmethod serdes/storage-path "Database" [{:keys [name]} _]
- ;; ["databases" "db_name" "db_name"] directory for the database with same-named file inside.
- ["databases" name name])
+ ;; directory for the database with same-named file inside.
+ [{:label "databases"} {:label name :key name} {:label name :key name}])
(defn assert-not-h2!
"Validate db is not h2 for serialization import"
diff --git a/test_resources/serialization_baseline/actions/hYWoE-l1BpOqmvYedqP2g_my_action.yaml b/test_resources/serialization_baseline/actions/my_action.yaml
similarity index 100%
rename from test_resources/serialization_baseline/actions/hYWoE-l1BpOqmvYedqP2g_my_action.yaml
rename to test_resources/serialization_baseline/actions/my_action.yaml
diff --git a/test_resources/serialization_baseline/collections/cards/kDd3G__y2cqdZu7A1SUsU_ai_model.yaml b/test_resources/serialization_baseline/collections/main/ai_model.yaml
similarity index 100%
rename from test_resources/serialization_baseline/collections/cards/kDd3G__y2cqdZu7A1SUsU_ai_model.yaml
rename to test_resources/serialization_baseline/collections/main/ai_model.yaml
diff --git a/test_resources/serialization_baseline/collections/cards/4ahdpFn43jybRFVQTRjX__ai_model.yaml b/test_resources/serialization_baseline/collections/main/ai_model_2.yaml
similarity index 100%
rename from test_resources/serialization_baseline/collections/cards/4ahdpFn43jybRFVQTRjX__ai_model.yaml
rename to test_resources/serialization_baseline/collections/main/ai_model_2.yaml
diff --git a/test_resources/serialization_baseline/collections/gQuHAIM4C-7fToHzHyNNQ_ai_model/gQuHAIM4C-7fToHzHyNNQ_ai_model.yaml b/test_resources/serialization_baseline/collections/main/ai_model_3.yaml
similarity index 100%
rename from test_resources/serialization_baseline/collections/gQuHAIM4C-7fToHzHyNNQ_ai_model/gQuHAIM4C-7fToHzHyNNQ_ai_model.yaml
rename to test_resources/serialization_baseline/collections/main/ai_model_3.yaml
diff --git a/test_resources/serialization_baseline/collections/gQuHAIM4C-7fToHzHyNNQ_ai_model/cards/Mr5p0TE2FSxGeUFaWd9yu_ai_model.yaml b/test_resources/serialization_baseline/collections/main/ai_model_3/ai_model.yaml
similarity index 100%
rename from test_resources/serialization_baseline/collections/gQuHAIM4C-7fToHzHyNNQ_ai_model/cards/Mr5p0TE2FSxGeUFaWd9yu_ai_model.yaml
rename to test_resources/serialization_baseline/collections/main/ai_model_3/ai_model.yaml
diff --git a/test_resources/serialization_baseline/collections/gQuHAIM4C-7fToHzHyNNQ_ai_model/metabots/gr1EkP92whbdA_TKQljLU.yaml b/test_resources/serialization_baseline/collections/main/ai_model_3/test_metabot.yaml
similarity index 100%
rename from test_resources/serialization_baseline/collections/gQuHAIM4C-7fToHzHyNNQ_ai_model/metabots/gr1EkP92whbdA_TKQljLU.yaml
rename to test_resources/serialization_baseline/collections/main/ai_model_3/test_metabot.yaml
diff --git a/test_resources/serialization_baseline/collections/cards/RC0EJ8I00lOUnvbnGJJNf_ai_model.yaml b/test_resources/serialization_baseline/collections/main/ai_model_4.yaml
similarity index 100%
rename from test_resources/serialization_baseline/collections/cards/RC0EJ8I00lOUnvbnGJJNf_ai_model.yaml
rename to test_resources/serialization_baseline/collections/main/ai_model_4.yaml
diff --git a/test_resources/serialization_baseline/collections/metabots/embeddedmetabotmetabo.yaml b/test_resources/serialization_baseline/collections/main/embedded_metabot.yaml
similarity index 100%
rename from test_resources/serialization_baseline/collections/metabots/embeddedmetabotmetabo.yaml
rename to test_resources/serialization_baseline/collections/main/embedded_metabot.yaml
diff --git a/test_resources/serialization_baseline/collections/olgKH56lYOjakuobH4zPz_grandparent_collection/olgKH56lYOjakuobH4zPz_grandparent_collection.yaml b/test_resources/serialization_baseline/collections/main/grandparent_collection.yaml
similarity index 100%
rename from test_resources/serialization_baseline/collections/olgKH56lYOjakuobH4zPz_grandparent_collection/olgKH56lYOjakuobH4zPz_grandparent_collection.yaml
rename to test_resources/serialization_baseline/collections/main/grandparent_collection.yaml
diff --git a/test_resources/serialization_baseline/collections/olgKH56lYOjakuobH4zPz_grandparent_collection/cards/m98z04nYGCF5n7cvOTfbj_grandparent_card.yaml b/test_resources/serialization_baseline/collections/main/grandparent_collection/grandparent_card.yaml
similarity index 100%
rename from test_resources/serialization_baseline/collections/olgKH56lYOjakuobH4zPz_grandparent_collection/cards/m98z04nYGCF5n7cvOTfbj_grandparent_card.yaml
rename to test_resources/serialization_baseline/collections/main/grandparent_collection/grandparent_card.yaml
diff --git a/test_resources/serialization_baseline/collections/olgKH56lYOjakuobH4zPz_grandparent_collection/qPZhJMPNGkfd3sq8jWiZS_parent_collection/qPZhJMPNGkfd3sq8jWiZS_parent_collection.yaml b/test_resources/serialization_baseline/collections/main/grandparent_collection/parent_collection.yaml
similarity index 100%
rename from test_resources/serialization_baseline/collections/olgKH56lYOjakuobH4zPz_grandparent_collection/qPZhJMPNGkfd3sq8jWiZS_parent_collection/qPZhJMPNGkfd3sq8jWiZS_parent_collection.yaml
rename to test_resources/serialization_baseline/collections/main/grandparent_collection/parent_collection.yaml
diff --git a/test_resources/serialization_baseline/collections/olgKH56lYOjakuobH4zPz_grandparent_collection/qPZhJMPNGkfd3sq8jWiZS_parent_collection/fJVsXIDzzipSZsaro9Pvu_child_collection/fJVsXIDzzipSZsaro9Pvu_child_collection.yaml b/test_resources/serialization_baseline/collections/main/grandparent_collection/parent_collection/child_collection.yaml
similarity index 100%
rename from test_resources/serialization_baseline/collections/olgKH56lYOjakuobH4zPz_grandparent_collection/qPZhJMPNGkfd3sq8jWiZS_parent_collection/fJVsXIDzzipSZsaro9Pvu_child_collection/fJVsXIDzzipSZsaro9Pvu_child_collection.yaml
rename to test_resources/serialization_baseline/collections/main/grandparent_collection/parent_collection/child_collection.yaml
diff --git a/test_resources/serialization_baseline/collections/olgKH56lYOjakuobH4zPz_grandparent_collection/qPZhJMPNGkfd3sq8jWiZS_parent_collection/fJVsXIDzzipSZsaro9Pvu_child_collection/cards/Ra0JVOCXKTxH5TPbwZm0x_child_card.yaml b/test_resources/serialization_baseline/collections/main/grandparent_collection/parent_collection/child_collection/child_card.yaml
similarity index 100%
rename from test_resources/serialization_baseline/collections/olgKH56lYOjakuobH4zPz_grandparent_collection/qPZhJMPNGkfd3sq8jWiZS_parent_collection/fJVsXIDzzipSZsaro9Pvu_child_collection/cards/Ra0JVOCXKTxH5TPbwZm0x_child_card.yaml
rename to test_resources/serialization_baseline/collections/main/grandparent_collection/parent_collection/child_collection/child_card.yaml
diff --git a/test_resources/serialization_baseline/collections/olgKH56lYOjakuobH4zPz_grandparent_collection/qPZhJMPNGkfd3sq8jWiZS_parent_collection/cards/IBZSKSt2-QOMviIaGcQE6_parent_card.yaml b/test_resources/serialization_baseline/collections/main/grandparent_collection/parent_collection/parent_card.yaml
similarity index 100%
rename from test_resources/serialization_baseline/collections/olgKH56lYOjakuobH4zPz_grandparent_collection/qPZhJMPNGkfd3sq8jWiZS_parent_collection/cards/IBZSKSt2-QOMviIaGcQE6_parent_card.yaml
rename to test_resources/serialization_baseline/collections/main/grandparent_collection/parent_collection/parent_card.yaml
diff --git a/test_resources/serialization_baseline/collections/olgKH56lYOjakuobH4zPz_grandparent_collection/qPZhJMPNGkfd3sq8jWiZS_parent_collection/dashboards/JT3BKJwNCfhUHnd9oqQ_1_parent_dash.yaml b/test_resources/serialization_baseline/collections/main/grandparent_collection/parent_collection/parent_dash.yaml
similarity index 100%
rename from test_resources/serialization_baseline/collections/olgKH56lYOjakuobH4zPz_grandparent_collection/qPZhJMPNGkfd3sq8jWiZS_parent_collection/dashboards/JT3BKJwNCfhUHnd9oqQ_1_parent_dash.yaml
rename to test_resources/serialization_baseline/collections/main/grandparent_collection/parent_collection/parent_dash.yaml
diff --git a/test_resources/serialization_baseline/collections/cards/1z82DAj_YUJAsCZZaacxK_linked_card.yaml b/test_resources/serialization_baseline/collections/main/linked_card.yaml
similarity index 100%
rename from test_resources/serialization_baseline/collections/cards/1z82DAj_YUJAsCZZaacxK_linked_card.yaml
rename to test_resources/serialization_baseline/collections/main/linked_card.yaml
diff --git a/test_resources/serialization_baseline/collections/metabots/metabotmetabotmetabot.yaml b/test_resources/serialization_baseline/collections/main/metabot.yaml
similarity index 100%
rename from test_resources/serialization_baseline/collections/metabots/metabotmetabotmetabot.yaml
rename to test_resources/serialization_baseline/collections/main/metabot.yaml
diff --git a/test_resources/serialization_baseline/collections/channels/My HTTP channel_my_http_channel.yaml b/test_resources/serialization_baseline/collections/main/my_http_channel.yaml
similarity index 100%
rename from test_resources/serialization_baseline/collections/channels/My HTTP channel_my_http_channel.yaml
rename to test_resources/serialization_baseline/collections/main/my_http_channel.yaml
diff --git a/test_resources/serialization_baseline/collections/JS5JHDitDkWEtTt1-N7zr_qdlkexrocaxgnecmryqu/JS5JHDitDkWEtTt1-N7zr_qdlkexrocaxgnecmryqu.yaml b/test_resources/serialization_baseline/collections/main/qdlkexrocaxgnecmryqu.yaml
similarity index 100%
rename from test_resources/serialization_baseline/collections/JS5JHDitDkWEtTt1-N7zr_qdlkexrocaxgnecmryqu/JS5JHDitDkWEtTt1-N7zr_qdlkexrocaxgnecmryqu.yaml
rename to test_resources/serialization_baseline/collections/main/qdlkexrocaxgnecmryqu.yaml
diff --git a/test_resources/serialization_baseline/collections/JS5JHDitDkWEtTt1-N7zr_qdlkexrocaxgnecmryqu/cards/_DexsxY2EQdmhwMZU4BCs_dependent_card.yaml b/test_resources/serialization_baseline/collections/main/qdlkexrocaxgnecmryqu/dependent_card.yaml
similarity index 100%
rename from test_resources/serialization_baseline/collections/JS5JHDitDkWEtTt1-N7zr_qdlkexrocaxgnecmryqu/cards/_DexsxY2EQdmhwMZU4BCs_dependent_card.yaml
rename to test_resources/serialization_baseline/collections/main/qdlkexrocaxgnecmryqu/dependent_card.yaml
diff --git a/test_resources/serialization_baseline/collections/JS5JHDitDkWEtTt1-N7zr_qdlkexrocaxgnecmryqu/documents/FLTBqagEX0mNl33ZZvFDa_test_document.yaml b/test_resources/serialization_baseline/collections/main/qdlkexrocaxgnecmryqu/test_document.yaml
similarity index 100%
rename from test_resources/serialization_baseline/collections/JS5JHDitDkWEtTt1-N7zr_qdlkexrocaxgnecmryqu/documents/FLTBqagEX0mNl33ZZvFDa_test_document.yaml
rename to test_resources/serialization_baseline/collections/main/qdlkexrocaxgnecmryqu/test_document.yaml
diff --git a/test_resources/serialization_baseline/collections/cards/ze7O4-8qRFH1v2Ho2apT1_root_card.yaml b/test_resources/serialization_baseline/collections/main/root_card.yaml
similarity index 100%
rename from test_resources/serialization_baseline/collections/cards/ze7O4-8qRFH1v2Ho2apT1_root_card.yaml
rename to test_resources/serialization_baseline/collections/main/root_card.yaml
diff --git a/test_resources/serialization_baseline/collections/p8Zq2ud7JlSoSBXdJWIaW_shared_collection/p8Zq2ud7JlSoSBXdJWIaW_shared_collection.yaml b/test_resources/serialization_baseline/collections/main/shared_collection.yaml
similarity index 100%
rename from test_resources/serialization_baseline/collections/p8Zq2ud7JlSoSBXdJWIaW_shared_collection/p8Zq2ud7JlSoSBXdJWIaW_shared_collection.yaml
rename to test_resources/serialization_baseline/collections/main/shared_collection.yaml
diff --git a/test_resources/serialization_baseline/collections/p8Zq2ud7JlSoSBXdJWIaW_shared_collection/timelines/PsHfXHhETFiTIRQODuYB8_empty_timeline.yaml b/test_resources/serialization_baseline/collections/main/shared_collection/empty_timeline.yaml
similarity index 100%
rename from test_resources/serialization_baseline/collections/p8Zq2ud7JlSoSBXdJWIaW_shared_collection/timelines/PsHfXHhETFiTIRQODuYB8_empty_timeline.yaml
rename to test_resources/serialization_baseline/collections/main/shared_collection/empty_timeline.yaml
diff --git a/test_resources/serialization_baseline/collections/p8Zq2ud7JlSoSBXdJWIaW_shared_collection/timelines/jqbQYdaqb_vXFlZnjvshP_populated_timeline.yaml b/test_resources/serialization_baseline/collections/main/shared_collection/populated_timeline.yaml
similarity index 100%
rename from test_resources/serialization_baseline/collections/p8Zq2ud7JlSoSBXdJWIaW_shared_collection/timelines/jqbQYdaqb_vXFlZnjvshP_populated_timeline.yaml
rename to test_resources/serialization_baseline/collections/main/shared_collection/populated_timeline.yaml
diff --git a/test_resources/serialization_baseline/collections/metabots/slackbotmetabotmetabo.yaml b/test_resources/serialization_baseline/collections/main/slack_metabot.yaml
similarity index 100%
rename from test_resources/serialization_baseline/collections/metabots/slackbotmetabotmetabo.yaml
rename to test_resources/serialization_baseline/collections/main/slack_metabot.yaml
diff --git a/test_resources/serialization_baseline/collections/dashboards/ziLZTiwSdz6aJaB9VhGiO_smart_linked_dashboard.yaml b/test_resources/serialization_baseline/collections/main/smart_linked_dashboard.yaml
similarity index 100%
rename from test_resources/serialization_baseline/collections/dashboards/ziLZTiwSdz6aJaB9VhGiO_smart_linked_dashboard.yaml
rename to test_resources/serialization_baseline/collections/main/smart_linked_dashboard.yaml
diff --git a/test_resources/serialization_baseline/collections/M-Q4pcV0qkiyJ0kiSWECl_some_collection/M-Q4pcV0qkiyJ0kiSWECl_some_collection.yaml b/test_resources/serialization_baseline/collections/main/some_collection.yaml
similarity index 100%
rename from test_resources/serialization_baseline/collections/M-Q4pcV0qkiyJ0kiSWECl_some_collection/M-Q4pcV0qkiyJ0kiSWECl_some_collection.yaml
rename to test_resources/serialization_baseline/collections/main/some_collection.yaml
diff --git a/test_resources/serialization_baseline/collections/M-Q4pcV0qkiyJ0kiSWECl_some_collection/cards/OMuZ0wHe2O5Z_59-cLmn4_series_question_a.yaml b/test_resources/serialization_baseline/collections/main/some_collection/series_question_a.yaml
similarity index 100%
rename from test_resources/serialization_baseline/collections/M-Q4pcV0qkiyJ0kiSWECl_some_collection/cards/OMuZ0wHe2O5Z_59-cLmn4_series_question_a.yaml
rename to test_resources/serialization_baseline/collections/main/some_collection/series_question_a.yaml
diff --git a/test_resources/serialization_baseline/collections/M-Q4pcV0qkiyJ0kiSWECl_some_collection/cards/XsxiHuzwlGIFNq245HdZC_series_question_b.yaml b/test_resources/serialization_baseline/collections/main/some_collection/series_question_b.yaml
similarity index 100%
rename from test_resources/serialization_baseline/collections/M-Q4pcV0qkiyJ0kiSWECl_some_collection/cards/XsxiHuzwlGIFNq245HdZC_series_question_b.yaml
rename to test_resources/serialization_baseline/collections/main/some_collection/series_question_b.yaml
diff --git a/test_resources/serialization_baseline/collections/M-Q4pcV0qkiyJ0kiSWECl_some_collection/dashboards/Q_jD-f-9clKLFZ2TfUG2h_shared_dashboard.yaml b/test_resources/serialization_baseline/collections/main/some_collection/shared_dashboard.yaml
similarity index 100%
rename from test_resources/serialization_baseline/collections/M-Q4pcV0qkiyJ0kiSWECl_some_collection/dashboards/Q_jD-f-9clKLFZ2TfUG2h_shared_dashboard.yaml
rename to test_resources/serialization_baseline/collections/main/some_collection/shared_dashboard.yaml
diff --git a/test_resources/serialization_baseline/collections/M-Q4pcV0qkiyJ0kiSWECl_some_collection/cards/f1C68pznmrpN1F5xFDj6d_some_question.yaml b/test_resources/serialization_baseline/collections/main/some_collection/some_question.yaml
similarity index 100%
rename from test_resources/serialization_baseline/collections/M-Q4pcV0qkiyJ0kiSWECl_some_collection/cards/f1C68pznmrpN1F5xFDj6d_some_question.yaml
rename to test_resources/serialization_baseline/collections/main/some_collection/some_question.yaml
diff --git a/test_resources/serialization_baseline/collections/cards/aqpA3mIKUnfzYUlwjuGwT_source_question.yaml b/test_resources/serialization_baseline/collections/main/source_question.yaml
similarity index 100%
rename from test_resources/serialization_baseline/collections/cards/aqpA3mIKUnfzYUlwjuGwT_source_question.yaml
rename to test_resources/serialization_baseline/collections/main/source_question.yaml
diff --git a/test_resources/serialization_baseline/collections/metabots/CyLr7tYRKiOoTWlcbl36q.yaml b/test_resources/serialization_baseline/collections/main/test_metabot.yaml
similarity index 100%
rename from test_resources/serialization_baseline/collections/metabots/CyLr7tYRKiOoTWlcbl36q.yaml
rename to test_resources/serialization_baseline/collections/main/test_metabot.yaml
diff --git a/test_resources/serialization_baseline/collections/metabots/zXxMBaxni6nRGRhD7vmPc.yaml b/test_resources/serialization_baseline/collections/main/test_metabot_2.yaml
similarity index 100%
rename from test_resources/serialization_baseline/collections/metabots/zXxMBaxni6nRGRhD7vmPc.yaml
rename to test_resources/serialization_baseline/collections/main/test_metabot_2.yaml
diff --git a/test_resources/serialization_baseline/collections/metabots/OYeHvLi8mMGwwMwJpetYw.yaml b/test_resources/serialization_baseline/collections/main/test_metabot_3.yaml
similarity index 100%
rename from test_resources/serialization_baseline/collections/metabots/OYeHvLi8mMGwwMwJpetYw.yaml
rename to test_resources/serialization_baseline/collections/main/test_metabot_3.yaml
diff --git a/test_resources/serialization_baseline/collections/Y6d4QwJgGKw-X1tRh3ir2_shared_collection/Y6d4QwJgGKw-X1tRh3ir2_shared_collection.yaml b/test_resources/serialization_baseline/collections/snippets/shared_collection_2.yaml
similarity index 100%
rename from test_resources/serialization_baseline/collections/Y6d4QwJgGKw-X1tRh3ir2_shared_collection/Y6d4QwJgGKw-X1tRh3ir2_shared_collection.yaml
rename to test_resources/serialization_baseline/collections/snippets/shared_collection_2.yaml
diff --git a/test_resources/serialization_baseline/snippets/Y6d4QwJgGKw-X1tRh3ir2_shared_collection/hcSr4xgVEd8AyElRRLu9w_snippet_1.yaml b/test_resources/serialization_baseline/collections/snippets/shared_collection_2/snippet_1.yaml
similarity index 100%
rename from test_resources/serialization_baseline/snippets/Y6d4QwJgGKw-X1tRh3ir2_shared_collection/hcSr4xgVEd8AyElRRLu9w_snippet_1.yaml
rename to test_resources/serialization_baseline/collections/snippets/shared_collection_2/snippet_1.yaml
diff --git a/test_resources/serialization_baseline/snippets/WyQWnT23PF-SfbYLROllJ_snippet_2.yaml b/test_resources/serialization_baseline/collections/snippets/snippet_2.yaml
similarity index 100%
rename from test_resources/serialization_baseline/snippets/WyQWnT23PF-SfbYLROllJ_snippet_2.yaml
rename to test_resources/serialization_baseline/collections/snippets/snippet_2.yaml
diff --git a/test_resources/serialization_baseline/collections/transforms/2HzIFwJ6720JAx07UMavl_test_transform.yaml b/test_resources/serialization_baseline/collections/transforms/test_transform.yaml
similarity index 100%
rename from test_resources/serialization_baseline/collections/transforms/2HzIFwJ6720JAx07UMavl_test_transform.yaml
rename to test_resources/serialization_baseline/collections/transforms/test_transform.yaml
index 26415e7ccdb..5f494b8a4f4 100644
--- a/test_resources/serialization_baseline/collections/transforms/2HzIFwJ6720JAx07UMavl_test_transform.yaml
+++ b/test_resources/serialization_baseline/collections/transforms/test_transform.yaml
@@ -4,7 +4,6 @@ creator_id: [email protected]
description: A test transform for serialization
entity_id: 2HzIFwJ6720JAx07UMavl
name: Test Transform
-source_database_id: My Database
owner_email: null
owner_user_id: null
source:
@@ -17,6 +16,7 @@ source:
- Schemaless Table
type: query
type: query
+source_database_id: My Database
tags:
- entity_id: TUtH6I5SqautNtUZoZ6Ti
position: 0
diff --git a/test_resources/serialization_baseline/databases/My Database/My Database.yaml b/test_resources/serialization_baseline/databases/my_database/my_database.yaml
similarity index 87%
rename from test_resources/serialization_baseline/databases/My Database/My Database.yaml
rename to test_resources/serialization_baseline/databases/my_database/my_database.yaml
index c51bcd6d075..33fa07dcf4f 100644
--- a/test_resources/serialization_baseline/databases/My Database/My Database.yaml
+++ b/test_resources/serialization_baseline/databases/my_database/my_database.yaml
@@ -12,8 +12,8 @@ is_on_demand: false
is_sample: false
is_audit: false
is_attached_dwh: false
-metadata_sync_schedule: 0 5 * * * ? *
-cache_field_values_schedule: 0 0 23 * * ? *
+metadata_sync_schedule: 0 28 * * * ? *
+cache_field_values_schedule: 0 0 10 * * ? *
settings:
database-source-dataset-name: test-data
caveats: null
diff --git a/test_resources/serialization_baseline/databases/My Database/schemas/public/tables/target_table/target_table.yaml b/test_resources/serialization_baseline/databases/my_database/schemas/public_2/tables/target_table/target_table.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/My Database/schemas/public/tables/target_table/target_table.yaml
rename to test_resources/serialization_baseline/databases/my_database/schemas/public_2/tables/target_table/target_table.yaml
diff --git a/test_resources/serialization_baseline/databases/My Database/tables/Schemaless Table/fields/Some Field.yaml b/test_resources/serialization_baseline/databases/my_database/tables/schemaless_table/fields/some_field.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/My Database/tables/Schemaless Table/fields/Some Field.yaml
rename to test_resources/serialization_baseline/databases/my_database/tables/schemaless_table/fields/some_field.yaml
diff --git a/test_resources/serialization_baseline/databases/My Database/tables/Schemaless Table/fields/Some Field___fieldusersettings.yaml b/test_resources/serialization_baseline/databases/my_database/tables/schemaless_table/fields/some_field___fieldusersettings.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/My Database/tables/Schemaless Table/fields/Some Field___fieldusersettings.yaml
rename to test_resources/serialization_baseline/databases/my_database/tables/schemaless_table/fields/some_field___fieldusersettings.yaml
diff --git a/test_resources/serialization_baseline/databases/My Database/tables/Schemaless Table/fields/Some Field___fieldvalues.yaml b/test_resources/serialization_baseline/databases/my_database/tables/schemaless_table/fields/some_field___fieldvalues.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/My Database/tables/Schemaless Table/fields/Some Field___fieldvalues.yaml
rename to test_resources/serialization_baseline/databases/my_database/tables/schemaless_table/fields/some_field___fieldvalues.yaml
diff --git a/test_resources/serialization_baseline/databases/My Database/tables/Schemaless Table/measures/0SHLtATxX8wbutAOlZAtM_my_measure.yaml b/test_resources/serialization_baseline/databases/my_database/tables/schemaless_table/measures/my_measure.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/My Database/tables/Schemaless Table/measures/0SHLtATxX8wbutAOlZAtM_my_measure.yaml
rename to test_resources/serialization_baseline/databases/my_database/tables/schemaless_table/measures/my_measure.yaml
diff --git a/test_resources/serialization_baseline/databases/My Database/tables/Schemaless Table/Schemaless Table.yaml b/test_resources/serialization_baseline/databases/my_database/tables/schemaless_table/schemaless_table.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/My Database/tables/Schemaless Table/Schemaless Table.yaml
rename to test_resources/serialization_baseline/databases/my_database/tables/schemaless_table/schemaless_table.yaml
diff --git a/test_resources/serialization_baseline/databases/My Database/tables/Schemaless Table/segments/TNdMrOCMHrQc_UtvCbTC5_my_segment.yaml b/test_resources/serialization_baseline/databases/my_database/tables/schemaless_table/segments/my_segment.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/My Database/tables/Schemaless Table/segments/TNdMrOCMHrQc_UtvCbTC5_my_segment.yaml
rename to test_resources/serialization_baseline/databases/my_database/tables/schemaless_table/segments/my_segment.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/CATEGORIES/CATEGORIES.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/categories/categories.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/CATEGORIES/CATEGORIES.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/categories/categories.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/CATEGORIES/fields/ID.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/categories/fields/id.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/CATEGORIES/fields/ID.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/categories/fields/id.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/CATEGORIES/fields/NAME.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/categories/fields/name.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/CATEGORIES/fields/NAME.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/categories/fields/name.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/CATEGORIES/fields/NAME___fieldvalues.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/categories/fields/name___fieldvalues.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/CATEGORIES/fields/NAME___fieldvalues.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/categories/fields/name___fieldvalues.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/CHECKINS/CHECKINS.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/checkins/checkins.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/CHECKINS/CHECKINS.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/checkins/checkins.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/CHECKINS/fields/DATE.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/checkins/fields/date.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/CHECKINS/fields/DATE.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/checkins/fields/date.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/CHECKINS/fields/ID.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/checkins/fields/id.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/CHECKINS/fields/ID.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/checkins/fields/id.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/CHECKINS/fields/USER_ID.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/checkins/fields/user_id.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/CHECKINS/fields/USER_ID.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/checkins/fields/user_id.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/CHECKINS/fields/VENUE_ID.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/checkins/fields/venue_id.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/CHECKINS/fields/VENUE_ID.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/checkins/fields/venue_id.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/ORDERS/fields/CREATED_AT.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/orders/fields/created_at.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/ORDERS/fields/CREATED_AT.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/orders/fields/created_at.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/ORDERS/fields/DISCOUNT.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/orders/fields/discount.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/ORDERS/fields/DISCOUNT.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/orders/fields/discount.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/ORDERS/fields/ID.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/orders/fields/id.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/ORDERS/fields/ID.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/orders/fields/id.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/ORDERS/fields/PRODUCT_ID.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/orders/fields/product_id.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/ORDERS/fields/PRODUCT_ID.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/orders/fields/product_id.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/ORDERS/fields/QUANTITY.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/orders/fields/quantity.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/ORDERS/fields/QUANTITY.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/orders/fields/quantity.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/ORDERS/fields/QUANTITY___fieldvalues.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/orders/fields/quantity___fieldvalues.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/ORDERS/fields/QUANTITY___fieldvalues.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/orders/fields/quantity___fieldvalues.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/ORDERS/fields/SUBTOTAL.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/orders/fields/subtotal.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/ORDERS/fields/SUBTOTAL.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/orders/fields/subtotal.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/ORDERS/fields/TAX.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/orders/fields/tax.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/ORDERS/fields/TAX.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/orders/fields/tax.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/ORDERS/fields/TOTAL.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/orders/fields/total.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/ORDERS/fields/TOTAL.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/orders/fields/total.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/ORDERS/fields/USER_ID.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/orders/fields/user_id.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/ORDERS/fields/USER_ID.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/orders/fields/user_id.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/ORDERS/ORDERS.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/orders/orders.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/ORDERS/ORDERS.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/orders/orders.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PEOPLE/fields/ADDRESS.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/people/fields/address.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PEOPLE/fields/ADDRESS.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/people/fields/address.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PEOPLE/fields/BIRTH_DATE.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/people/fields/birth_date.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PEOPLE/fields/BIRTH_DATE.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/people/fields/birth_date.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PEOPLE/fields/CITY.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/people/fields/city.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PEOPLE/fields/CITY.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/people/fields/city.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PEOPLE/fields/CREATED_AT.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/people/fields/created_at.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PEOPLE/fields/CREATED_AT.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/people/fields/created_at.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PEOPLE/fields/EMAIL.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/people/fields/email.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PEOPLE/fields/EMAIL.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/people/fields/email.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PEOPLE/fields/ID.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/people/fields/id.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PEOPLE/fields/ID.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/people/fields/id.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PEOPLE/fields/LATITUDE.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/people/fields/latitude.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PEOPLE/fields/LATITUDE.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/people/fields/latitude.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PEOPLE/fields/LONGITUDE.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/people/fields/longitude.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PEOPLE/fields/LONGITUDE.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/people/fields/longitude.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PEOPLE/fields/NAME.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/people/fields/name.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PEOPLE/fields/NAME.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/people/fields/name.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PEOPLE/fields/PASSWORD.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/people/fields/password.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PEOPLE/fields/PASSWORD.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/people/fields/password.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PEOPLE/fields/SOURCE.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/people/fields/source.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PEOPLE/fields/SOURCE.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/people/fields/source.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PEOPLE/fields/SOURCE___fieldvalues.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/people/fields/source___fieldvalues.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PEOPLE/fields/SOURCE___fieldvalues.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/people/fields/source___fieldvalues.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PEOPLE/fields/STATE.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/people/fields/state.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PEOPLE/fields/STATE.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/people/fields/state.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PEOPLE/fields/STATE___fieldvalues.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/people/fields/state___fieldvalues.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PEOPLE/fields/STATE___fieldvalues.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/people/fields/state___fieldvalues.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PEOPLE/fields/ZIP.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/people/fields/zip.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PEOPLE/fields/ZIP.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/people/fields/zip.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PEOPLE/PEOPLE.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/people/people.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PEOPLE/PEOPLE.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/people/people.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PRODUCTS/fields/CATEGORY.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/products/fields/category.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PRODUCTS/fields/CATEGORY.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/products/fields/category.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PRODUCTS/fields/CATEGORY___fieldvalues.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/products/fields/category___fieldvalues.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PRODUCTS/fields/CATEGORY___fieldvalues.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/products/fields/category___fieldvalues.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PRODUCTS/fields/CREATED_AT.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/products/fields/created_at.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PRODUCTS/fields/CREATED_AT.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/products/fields/created_at.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PRODUCTS/fields/EAN.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/products/fields/ean.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PRODUCTS/fields/EAN.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/products/fields/ean.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PRODUCTS/fields/EAN___fieldvalues.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/products/fields/ean___fieldvalues.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PRODUCTS/fields/EAN___fieldvalues.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/products/fields/ean___fieldvalues.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PRODUCTS/fields/ID.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/products/fields/id.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PRODUCTS/fields/ID.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/products/fields/id.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PRODUCTS/fields/PRICE.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/products/fields/price.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PRODUCTS/fields/PRICE.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/products/fields/price.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PRODUCTS/fields/RATING.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/products/fields/rating.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PRODUCTS/fields/RATING.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/products/fields/rating.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PRODUCTS/fields/TITLE.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/products/fields/title.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PRODUCTS/fields/TITLE.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/products/fields/title.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PRODUCTS/fields/TITLE___fieldvalues.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/products/fields/title___fieldvalues.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PRODUCTS/fields/TITLE___fieldvalues.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/products/fields/title___fieldvalues.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PRODUCTS/fields/VENDOR.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/products/fields/vendor.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PRODUCTS/fields/VENDOR.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/products/fields/vendor.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PRODUCTS/fields/VENDOR___fieldvalues.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/products/fields/vendor___fieldvalues.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PRODUCTS/fields/VENDOR___fieldvalues.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/products/fields/vendor___fieldvalues.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PRODUCTS/PRODUCTS.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/products/products.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PRODUCTS/PRODUCTS.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/products/products.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/REVIEWS/fields/BODY.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/reviews/fields/body.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/REVIEWS/fields/BODY.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/reviews/fields/body.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/REVIEWS/fields/CREATED_AT.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/reviews/fields/created_at.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/REVIEWS/fields/CREATED_AT.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/reviews/fields/created_at.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/REVIEWS/fields/ID.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/reviews/fields/id.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/REVIEWS/fields/ID.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/reviews/fields/id.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/REVIEWS/fields/PRODUCT_ID.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/reviews/fields/product_id.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/REVIEWS/fields/PRODUCT_ID.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/reviews/fields/product_id.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/REVIEWS/fields/RATING.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/reviews/fields/rating.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/REVIEWS/fields/RATING.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/reviews/fields/rating.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/REVIEWS/fields/RATING___fieldvalues.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/reviews/fields/rating___fieldvalues.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/REVIEWS/fields/RATING___fieldvalues.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/reviews/fields/rating___fieldvalues.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/REVIEWS/fields/REVIEWER.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/reviews/fields/reviewer.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/REVIEWS/fields/REVIEWER.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/reviews/fields/reviewer.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/REVIEWS/REVIEWS.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/reviews/reviews.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/REVIEWS/REVIEWS.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/reviews/reviews.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/USERS/fields/ID.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/users/fields/id.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/USERS/fields/ID.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/users/fields/id.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/USERS/fields/LAST_LOGIN.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/users/fields/last_login.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/USERS/fields/LAST_LOGIN.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/users/fields/last_login.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/USERS/fields/NAME.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/users/fields/name.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/USERS/fields/NAME.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/users/fields/name.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/USERS/fields/NAME___fieldvalues.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/users/fields/name___fieldvalues.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/USERS/fields/NAME___fieldvalues.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/users/fields/name___fieldvalues.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/USERS/fields/PASSWORD.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/users/fields/password.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/USERS/fields/PASSWORD.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/users/fields/password.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/USERS/fields/PASSWORD___fieldvalues.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/users/fields/password___fieldvalues.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/USERS/fields/PASSWORD___fieldvalues.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/users/fields/password___fieldvalues.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/USERS/USERS.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/users/users.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/USERS/USERS.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/users/users.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/VENUES/fields/CATEGORY_ID.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/venues/fields/category_id.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/VENUES/fields/CATEGORY_ID.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/venues/fields/category_id.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/VENUES/fields/ID.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/venues/fields/id.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/VENUES/fields/ID.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/venues/fields/id.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/VENUES/fields/LATITUDE.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/venues/fields/latitude.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/VENUES/fields/LATITUDE.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/venues/fields/latitude.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/VENUES/fields/LONGITUDE.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/venues/fields/longitude.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/VENUES/fields/LONGITUDE.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/venues/fields/longitude.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/VENUES/fields/NAME.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/venues/fields/name.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/VENUES/fields/NAME.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/venues/fields/name.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/VENUES/fields/NAME___fieldvalues.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/venues/fields/name___fieldvalues.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/VENUES/fields/NAME___fieldvalues.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/venues/fields/name___fieldvalues.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/VENUES/fields/PRICE.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/venues/fields/price.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/VENUES/fields/PRICE.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/venues/fields/price.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/VENUES/fields/PRICE___fieldvalues.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/venues/fields/price___fieldvalues.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/VENUES/fields/PRICE___fieldvalues.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/venues/fields/price___fieldvalues.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/VENUES/VENUES.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/venues/venues.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/VENUES/VENUES.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/schemas/public/tables/venues/venues.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/tables/linked_table/linked_table.yaml b/test_resources/serialization_baseline/databases/test_data__h2_/tables/linked_table/linked_table.yaml
similarity index 100%
rename from test_resources/serialization_baseline/databases/test-data (h2)/tables/linked_table/linked_table.yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/tables/linked_table/linked_table.yaml
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/test-data (h2).yaml b/test_resources/serialization_baseline/databases/test_data__h2_/test_data__h2_.yaml
similarity index 88%
rename from test_resources/serialization_baseline/databases/test-data (h2)/test-data (h2).yaml
rename to test_resources/serialization_baseline/databases/test_data__h2_/test_data__h2_.yaml
index beb1365a61e..39850b0040b 100644
--- a/test_resources/serialization_baseline/databases/test-data (h2)/test-data (h2).yaml
+++ b/test_resources/serialization_baseline/databases/test_data__h2_/test_data__h2_.yaml
@@ -17,8 +17,8 @@ is_on_demand: false
is_sample: false
is_audit: false
is_attached_dwh: false
-metadata_sync_schedule: 0 23 * * * ? *
-cache_field_values_schedule: 0 0 15 * * ? *
+metadata_sync_schedule: 0 49 * * * ? *
+cache_field_values_schedule: 0 0 4 * * ? *
settings:
database-source-dataset-name: test-data
caveats: null
diff --git a/test_resources/serialization_baseline/python-libraries/cWWH9qJPvHNB3rP2vLZrK.yaml b/test_resources/serialization_baseline/python_libraries/common.py.yaml
similarity index 100%
rename from test_resources/serialization_baseline/python-libraries/cWWH9qJPvHNB3rP2vLZrK.yaml
rename to test_resources/serialization_baseline/python_libraries/common.py.yaml
index df7768a7956..496fe72201e 100644
--- a/test_resources/serialization_baseline/python-libraries/cWWH9qJPvHNB3rP2vLZrK.yaml
+++ b/test_resources/serialization_baseline/python_libraries/common.py.yaml
@@ -1,9 +1,9 @@
-created_at: '2026-01-21T21:50:59.518133Z'
-entity_id: cWWH9qJPvHNB3rP2vLZrK
path: common.py
source: |-
def helper():
return 42
+entity_id: cWWH9qJPvHNB3rP2vLZrK
+created_at: '2026-01-21T21:50:59.518133Z'
serdes/meta:
- id: cWWH9qJPvHNB3rP2vLZrK
model: PythonLibrary
diff --git a/test_resources/serialization_baseline/settings.yaml b/test_resources/serialization_baseline/settings.yaml
index b8d4349370e..524cb03adc8 100644
--- a/test_resources/serialization_baseline/settings.yaml
+++ b/test_resources/serialization_baseline/settings.yaml
@@ -19,6 +19,7 @@ custom-homepage-dashboard: null
default-maps-enabled: null
disable-cors-on-localhost: null
email-max-recipients-per-second: null
+embedded-metabot-enabled?: null
embedding-homepage: null
embedding-hub-production-embed-snippet-created: null
embedding-hub-sso-auth-manual-tested: null
@@ -32,14 +33,13 @@ enable-xrays: null
gsheets: null
hide-embed-branding?: null
humanization-strategy: null
-embedded-metabot-enabled?: null
-metabot-enabled?: null
landing-page: null
landing-page-illustration: null
landing-page-illustration-custom: null
loading-message: null
login-page-illustration: null
login-page-illustration-custom: null
+metabot-enabled?: null
native-query-autocomplete-match-style: null
no-data-illustration: null
no-data-illustration-custom: null
diff --git a/test_resources/serialization_baseline/transforms/transform_jobs/PJahvcoF61oX5T4uCT_FY_daily_job.yaml b/test_resources/serialization_baseline/transforms/transform_jobs/daily_job.yaml
similarity index 100%
rename from test_resources/serialization_baseline/transforms/transform_jobs/PJahvcoF61oX5T4uCT_FY_daily_job.yaml
rename to test_resources/serialization_baseline/transforms/transform_jobs/daily_job.yaml
diff --git a/test_resources/serialization_baseline/transforms/transform_jobs/RognTgBT2fXaxoQh4ws7z_hourly_job.yaml b/test_resources/serialization_baseline/transforms/transform_jobs/hourly_job.yaml
similarity index 100%
rename from test_resources/serialization_baseline/transforms/transform_jobs/RognTgBT2fXaxoQh4ws7z_hourly_job.yaml
rename to test_resources/serialization_baseline/transforms/transform_jobs/hourly_job.yaml
diff --git a/test_resources/serialization_baseline/transforms/transform_jobs/CL1TVitmzJrDjdjvO2xmN_monthly_job.yaml b/test_resources/serialization_baseline/transforms/transform_jobs/monthly_job.yaml
similarity index 100%
rename from test_resources/serialization_baseline/transforms/transform_jobs/CL1TVitmzJrDjdjvO2xmN_monthly_job.yaml
rename to test_resources/serialization_baseline/transforms/transform_jobs/monthly_job.yaml
diff --git a/test_resources/serialization_baseline/transforms/transform_jobs/lXIFa2SUrXLgaLZJx33oU_weekly_job.yaml b/test_resources/serialization_baseline/transforms/transform_jobs/weekly_job.yaml
similarity index 100%
rename from test_resources/serialization_baseline/transforms/transform_jobs/lXIFa2SUrXLgaLZJx33oU_weekly_job.yaml
rename to test_resources/serialization_baseline/transforms/transform_jobs/weekly_job.yaml
diff --git a/test_resources/serialization_baseline/transforms/transform_tags/custometlcustometlcus_custom_etl.yaml b/test_resources/serialization_baseline/transforms/transform_tags/custom_etl.yaml
similarity index 100%
rename from test_resources/serialization_baseline/transforms/transform_tags/custometlcustometlcus_custom_etl.yaml
rename to test_resources/serialization_baseline/transforms/transform_tags/custom_etl.yaml
diff --git a/test_resources/serialization_baseline/transforms/transform_tags/dUW7nvQHQBdA0Rx0gJckI_daily.yaml b/test_resources/serialization_baseline/transforms/transform_tags/daily.yaml
similarity index 100%
rename from test_resources/serialization_baseline/transforms/transform_tags/dUW7nvQHQBdA0Rx0gJckI_daily.yaml
rename to test_resources/serialization_baseline/transforms/transform_tags/daily.yaml
diff --git a/test_resources/serialization_baseline/transforms/transform_tags/dailydailydailydailyx_daily.yaml b/test_resources/serialization_baseline/transforms/transform_tags/daily_2.yaml
similarity index 100%
rename from test_resources/serialization_baseline/transforms/transform_tags/dailydailydailydailyx_daily.yaml
rename to test_resources/serialization_baseline/transforms/transform_tags/daily_2.yaml
diff --git a/test_resources/serialization_baseline/transforms/transform_tags/wYLcUPbtF7jVPO0duGGHR_hourly.yaml b/test_resources/serialization_baseline/transforms/transform_tags/hourly.yaml
similarity index 100%
rename from test_resources/serialization_baseline/transforms/transform_tags/wYLcUPbtF7jVPO0duGGHR_hourly.yaml
rename to test_resources/serialization_baseline/transforms/transform_tags/hourly.yaml
diff --git a/test_resources/serialization_baseline/transforms/transform_tags/hourlyhourlyhourlyxxx_hourly.yaml b/test_resources/serialization_baseline/transforms/transform_tags/hourly_2.yaml
similarity index 100%
rename from test_resources/serialization_baseline/transforms/transform_tags/hourlyhourlyhourlyxxx_hourly.yaml
rename to test_resources/serialization_baseline/transforms/transform_tags/hourly_2.yaml
diff --git a/test_resources/serialization_baseline/transforms/transform_tags/DeeJHc81qKhD9qWINISO8_monthly.yaml b/test_resources/serialization_baseline/transforms/transform_tags/monthly.yaml
similarity index 100%
rename from test_resources/serialization_baseline/transforms/transform_tags/DeeJHc81qKhD9qWINISO8_monthly.yaml
rename to test_resources/serialization_baseline/transforms/transform_tags/monthly.yaml
diff --git a/test_resources/serialization_baseline/transforms/transform_tags/qPGCJu7t97kn-X07_fgSj_weekly.yaml b/test_resources/serialization_baseline/transforms/transform_tags/weekly.yaml
similarity index 100%
rename from test_resources/serialization_baseline/transforms/transform_tags/qPGCJu7t97kn-X07_fgSj_weekly.yaml
rename to test_resources/serialization_baseline/transforms/transform_tags/weekly.yaml