The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
fixes yaml-merge-driver migrations during rebase (#66904)
bryanc6d105ebd941
diff --git a/mage/src/mage/merge_yaml_migrations.clj b/mage/src/mage/merge_yaml_migrations.cljindex 436d7cbcac5..8ac918496da 100644--- a/mage/src/mage/merge_yaml_migrations.clj+++ b/mage/src/mage/merge_yaml_migrations.clj@@ -70,10 +70,19 @@(defn- cs-id [cs](-> cs :changeSet :id))-(defn- merge-changesets [base-data ours-data theirs-data]+(defn- merge-changesets [base-data ours-data theirs-data working-tree-data](let [base-cs (m/index-by cs-id (extract-changesets base-data))- ours-cs (m/index-by cs-id (extract-changesets ours-data))+ git-ours-cs (m/index-by cs-id (extract-changesets ours-data))theirs-cs (m/index-by cs-id (extract-changesets theirs-data))+ ;; During rebase, git passes the WRONG "ours" file on 2nd+ merge driver calls.+ ;; The working tree contains the actual correct state from previous cherry-picks.+ ;; Use working-tree as the authoritative "ours" when available.+ working-tree-cs (if working-tree-data+ (m/index-by cs-id (extract-changesets working-tree-data))+ {})+ ;; Effective ours: prefer working-tree over git-provided ours+ ;; This fixes the rebase bug where git passes stale ours data+ ours-cs (merge git-ours-cs working-tree-cs)all-ids (into #{}(comp (map keys) cat)[base-cs ours-cs theirs-cs])]@@ -123,20 +132,32 @@">>>>>>>\n")id ours theirs))-(defn- merge-files [base ours theirs {:keys [_marker-size]}]+(defn- merge-files [base ours theirs {:keys [_marker-size filepath]}](let [ours-text (slurp ours)theirs-text (slurp theirs)+ ;; Read from working tree to get the actual current state during rebase+ ;; This contains migrations correctly merged in previous cherry-picks+ working-tree-text (try (slurp filepath) (catch Exception _ nil));; Parse yamlbase-data (parse-yaml base)ours-data (parse-yaml ours)theirs-data (parse-yaml theirs)+ working-tree-data (when working-tree-text+ (try+ (yaml/parse-string working-tree-text)+ (catch Exception _ nil)));; Extract raw text for each changeset- ours-cs-texts (extract-changeset-texts ours-text ours-data)+ git-ours-cs-texts (extract-changeset-texts ours-text ours-data)theirs-cs-texts (extract-changeset-texts theirs-text theirs-data)- ;; Extract footer from ours file- footer (extract-footer ours-text)+ working-tree-cs-texts (when (and working-tree-text working-tree-data)+ (extract-changeset-texts working-tree-text working-tree-data))+ ;; Merge text maps: prefer working-tree over git-provided ours+ ;; This ensures we use the correct text for migrations from previous cherry-picks+ ours-cs-texts (merge git-ours-cs-texts working-tree-cs-texts)+ ;; Extract footer from ours file (or working tree if available)+ footer (extract-footer (or working-tree-text ours-text));; Perform merge- merged (merge-changesets base-data ours-data theirs-data)+ merged (merge-changesets base-data ours-data theirs-data working-tree-data);; Sort merged changesets by ID (chronological order)sorted-merged (sort-by :id merged);; Build result by concatenating raw text blocks@@ -151,7 +172,7 @@(get ours-cs-texts (:id cs))(get theirs-cs-texts (:id cs))))];; Remove trailing blank lines from each changeset- (str/replace text #"\n+$" "")))]+ (some-> text (str/replace #"\n+$" ""))))]{:result (str header"\n"(str/join "\n\n" (filter some? changeset-texts))@@ -161,8 +182,6 @@:conflicts (vec (keep #(when (= (:source %) :conflict) (:id %)) sorted-merged)):cnt (count sorted-merged)}))-;;-;;;; Usage (called by git):;; merge-yaml-migrations %O %A %B %L %P@@ -184,13 +203,23 @@(println "Will modify <ours>!"))(u/exit 2))- (let [[base ours theirs marker-size target] arguments+ (let [;; Resolve relative paths against project root (where git creates temp files)+ resolve-path (fn [p]+ (if (str/starts-with? p "/")+ p+ (str u/project-root-directory "/" p)))+ [base ours theirs marker-size filepath] arguments+ base (resolve-path base)+ ours (resolve-path ours)+ theirs (resolve-path theirs)+ filepath (when filepath (resolve-path filepath)){:keys [result conflicts cnt]} (merge-filesbaseourstheirs- {:marker-size marker-size})]- (spit (or target *out*) result)+ {:marker-size marker-size+ :filepath filepath})]+ (spit (or filepath ours) result);; Exit with appropriate code(when (seq conflicts)diff --git a/mage/test/mage/merge_yaml_migrations_integration_test.clj b/mage/test/mage/merge_yaml_migrations_integration_test.cljnew file mode 100644index 00000000000..36b218cde45--- /dev/null+++ b/mage/test/mage/merge_yaml_migrations_integration_test.clj@@ -0,0 +1,274 @@+(ns mage.merge-yaml-migrations-integration-test+ "Integration tests for the YAML merge driver during git rebase scenarios.++ These tests are NOT run by `mage -test` because they require creating+ temporary git repositories and branches. Run manually with:+ bb -e '(require (quote mage.merge-yaml-migrations-integration-test)) (mage.merge-yaml-migrations-integration-test/run-tests)'++ This tests the root cause of migrations being dropped during rebase:+ Git passes the WRONG 'ours' file on 2nd+ merge driver calls.++ Scenario:+ Feature branch has commits: A → B → C (each touches migrations.yaml)++ During rebase onto master:+ - Cherry-pick A: base=merge-base, ours=master, theirs=A → correct+ - Cherry-pick B: base=A, ours=master (WRONG!), theirs=B → loses A's migrations+ - Cherry-pick C: base=B, ours=master (WRONG!), theirs=C → loses A+B's migrations"+ (:require+ [babashka.fs :as fs]+ [babashka.process :as p]+ [clojure.string :as str]+ [clojure.test :refer [deftest is testing]]))++(def ^:private base-yaml+ "databaseChangeLog:+ - objectQuotingStrategy: QUOTE_ALL_OBJECTS++ - changeSet:+ id: v58.2025-01-01T00:00:00+ author: base+ changes:+ - sql:+ sql: SELECT 1++# >>>>>>>>>> DO NOT ADD NEW MIGRATIONS BELOW THIS LINE! ADD THEM ABOVE <<<<<<<<<<+")++(def ^:private commit-a-yaml+ "databaseChangeLog:+ - objectQuotingStrategy: QUOTE_ALL_OBJECTS++ - changeSet:+ id: v58.2025-01-01T00:00:00+ author: base+ changes:+ - sql:+ sql: SELECT 1++ - changeSet:+ id: v58.2025-01-02T00:00:00+ author: commit-a+ changes:+ - sql:+ sql: SELECT 'from commit A'++# >>>>>>>>>> DO NOT ADD NEW MIGRATIONS BELOW THIS LINE! ADD THEM ABOVE <<<<<<<<<<+")++(def ^:private commit-b-yaml+ "databaseChangeLog:+ - objectQuotingStrategy: QUOTE_ALL_OBJECTS++ - changeSet:+ id: v58.2025-01-01T00:00:00+ author: base+ changes:+ - sql:+ sql: SELECT 1++ - changeSet:+ id: v58.2025-01-02T00:00:00+ author: commit-a+ changes:+ - sql:+ sql: SELECT 'from commit A'++ - changeSet:+ id: v58.2025-01-03T00:00:00+ author: commit-b+ changes:+ - sql:+ sql: SELECT 'from commit B'++# >>>>>>>>>> DO NOT ADD NEW MIGRATIONS BELOW THIS LINE! ADD THEM ABOVE <<<<<<<<<<+")++(def ^:private commit-c-yaml+ "databaseChangeLog:+ - objectQuotingStrategy: QUOTE_ALL_OBJECTS++ - changeSet:+ id: v58.2025-01-01T00:00:00+ author: base+ changes:+ - sql:+ sql: SELECT 1++ - changeSet:+ id: v58.2025-01-02T00:00:00+ author: commit-a+ changes:+ - sql:+ sql: SELECT 'from commit A'++ - changeSet:+ id: v58.2025-01-03T00:00:00+ author: commit-b+ changes:+ - sql:+ sql: SELECT 'from commit B'++ - changeSet:+ id: v58.2025-01-04T00:00:00+ author: commit-c+ changes:+ - sql:+ sql: SELECT 'from commit C'++# >>>>>>>>>> DO NOT ADD NEW MIGRATIONS BELOW THIS LINE! ADD THEM ABOVE <<<<<<<<<<+")++(def ^:private master-new-yaml+ "databaseChangeLog:+ - objectQuotingStrategy: QUOTE_ALL_OBJECTS++ - changeSet:+ id: v58.2025-01-01T00:00:00+ author: base+ changes:+ - sql:+ sql: SELECT 1++ - changeSet:+ id: v58.2025-01-05T00:00:00+ author: master-new+ changes:+ - sql:+ sql: SELECT 'from master after branch'++# >>>>>>>>>> DO NOT ADD NEW MIGRATIONS BELOW THIS LINE! ADD THEM ABOVE <<<<<<<<<<+")++(defn- sh [dir & args]+ (let [result (p/sh (vec args) {:dir dir :err :string :out :string})]+ (when (not= 0 (:exit result))+ (throw (ex-info (str "Command failed: " (str/join " " args))+ {:args args+ :exit (:exit result)+ :out (:out result)+ :err (:err result)})))+ (:out result)))++(defn- sh-ok? [dir & args]+ (let [result (p/sh (vec args) {:dir dir :err :string :out :string})]+ (= 0 (:exit result))))++(defn- count-changesets [content]+ (count (re-seq #"id: v58\." content)))++(defn- setup-test-repo+ "Creates a test repository with the problematic rebase scenario.++ Returns the path to the temp directory."+ []+ (let [tmp-dir (str (fs/create-temp-dir {:prefix "yaml-merge-test-"}))+ migrations-file "migrations.yaml"+ mage-path (str (fs/absolutize "."))]++ ;; Initialize repo+ (sh tmp-dir "git" "init")+ (sh tmp-dir "git" "config" "user.email" "[email protected]")+ (sh tmp-dir "git" "config" "user.name" "Test User")++ ;; Create a wrapper script that sets MB_DIR to the original PWD+ ;; This allows mage to resolve relative paths correctly+ (let [wrapper-path (str tmp-dir "/merge-wrapper.sh")]+ (spit wrapper-path+ (str "#!/bin/bash\n"+ "# Set MB_DIR to preserve original working directory\n"+ "export MB_DIR=\"$PWD\"\n"+ mage-path "/bin/mage -merge-yaml-migrations \"$@\"\n"))+ (sh tmp-dir "chmod" "+x" wrapper-path)++ ;; Configure merge driver to use the wrapper+ (spit (str tmp-dir "/.gitattributes")+ (str migrations-file " merge=yaml-migrations\n"))+ (sh tmp-dir "git" "config" "merge.yaml-migrations.name" "YAML Migrations Merge Driver")+ (sh tmp-dir "git" "config" "merge.yaml-migrations.driver"+ (str wrapper-path " %O %A %B %L %P")))++ ;; Create initial commit on master with base yaml+ (spit (str tmp-dir "/" migrations-file) base-yaml)+ (sh tmp-dir "git" "add" ".")+ (sh tmp-dir "git" "commit" "-m" "Initial commit")++ ;; Create feature branch+ (sh tmp-dir "git" "checkout" "-b" "feature")++ ;; Commit A - add migration+ (spit (str tmp-dir "/" migrations-file) commit-a-yaml)+ (sh tmp-dir "git" "add" ".")+ (sh tmp-dir "git" "commit" "-m" "Commit A: add migration 01-02")++ ;; Commit B - add another migration+ (spit (str tmp-dir "/" migrations-file) commit-b-yaml)+ (sh tmp-dir "git" "add" ".")+ (sh tmp-dir "git" "commit" "-m" "Commit B: add migration 01-03")++ ;; Commit C - add another migration+ (spit (str tmp-dir "/" migrations-file) commit-c-yaml)+ (sh tmp-dir "git" "add" ".")+ (sh tmp-dir "git" "commit" "-m" "Commit C: add migration 01-04")++ ;; Go back to master and add a new migration there+ (sh tmp-dir "git" "checkout" "master")+ (spit (str tmp-dir "/" migrations-file) master-new-yaml)+ (sh tmp-dir "git" "add" ".")+ (sh tmp-dir "git" "commit" "-m" "Master: add migration 01-05")++ tmp-dir))++(deftest rebase-preserves-all-migrations-test+ (testing "Rebasing feature branch with multiple commits preserves all migrations"+ (let [tmp-dir (setup-test-repo)+ migrations-file "migrations.yaml"]+ (try+ ;; Checkout feature branch+ (sh tmp-dir "git" "checkout" "feature")++ ;; Count changesets before rebase (should be 4: base + A + B + C)+ (let [before-content (slurp (str tmp-dir "/" migrations-file))+ before-count (count-changesets before-content)]+ (is (= 4 before-count) "Feature branch should have 4 changesets before rebase"))++ ;; Perform the rebase+ (let [rebase-result (sh-ok? tmp-dir "git" "rebase" "master")]+ (is rebase-result "Rebase should succeed without conflicts"))++ ;; Count changesets after rebase (should be 5: base + master + A + B + C)+ (let [after-content (slurp (str tmp-dir "/" migrations-file))+ after-count (count-changesets after-content)]+ (is (= 5 after-count)+ (str "After rebase should have 5 changesets (base + master + A + B + C), got "+ after-count ". Content:\n" after-content))++ ;; Verify specific migrations are present+ (is (str/includes? after-content "v58.2025-01-01T00:00:00") "Base migration present")+ (is (str/includes? after-content "v58.2025-01-02T00:00:00") "Commit A migration present")+ (is (str/includes? after-content "v58.2025-01-03T00:00:00") "Commit B migration present")+ (is (str/includes? after-content "v58.2025-01-04T00:00:00") "Commit C migration present")+ (is (str/includes? after-content "v58.2025-01-05T00:00:00") "Master migration present"))++ (finally+ ;; Cleanup+ (fs/delete-tree tmp-dir))))))++(defn run-tests+ "Run the integration tests. Call this manually since these tests+ are not included in the standard test suite."+ []+ (println "Running YAML merge driver integration tests...")+ (let [results (clojure.test/run-tests 'mage.merge-yaml-migrations-integration-test)]+ (if (and (zero? (:fail results)) (zero? (:error results)))+ (println "✓ All integration tests passed!")+ (println "✗ Some tests failed"))+ results))++(comment+ ;; Run tests manually:+ (run-tests)++ ;; Or from command line:+ ;; bb -e '(require (quote mage.merge-yaml-migrations-integration-test)) (mage.merge-yaml-migrations-integration-test/run-tests)'+ )