Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

yaml migrations helper, into mage (#66486)

bryanbcd33be0ae1f
diff --git a/.gitattributes b/.gitattributes
new file mode 100644
index 00000000000..c8dd1874eeb
--- /dev/null
+++ b/.gitattributes
@@ -0,0 +1,5 @@
+# Custom merge driver for YAML migration files
+# Treats each changeset as an atomic unit to avoid conflicts within changesets
+# Execute this in your shell to enable custom merge driver:
+# echo '\n[include]\npath=../.gitconfig' >> .git/config
+resources/migrations/*.yaml merge=yaml-migrations
diff --git a/.gitconfig b/.gitconfig
new file mode 100644
index 00000000000..6feb99900d6
--- /dev/null
+++ b/.gitconfig
@@ -0,0 +1,5 @@
+# execute this in your shell to enable custom merge driver:
+# echo '\n[include]\npath=../.gitconfig' >> .git/config
+[merge "yaml-migrations"]
+ name = YAML migration changeset-aware merge
+ driver = mage -merge-yaml-migrations %O %A %B %L %P
diff --git a/bb.edn b/bb.edn
index b9700e701df..752b5c217d4 100644
--- a/bb.edn
+++ b/bb.edn
@@ -2,12 +2,14 @@
;; we put path as bin, and everything is in the ./mage subdirectory,
;; so the namespaces are mage.cli, mage.format, etc.
:paths ["mage/src" "mage/test" "bin/lint-migrations-file/src"]
- :deps {dev.weavejester/cljfmt {:mvn/version "0.15.3"}
- io.github.paintparty/bling {:mvn/version "0.8.8"}
- metosin/malli {:mvn/version "0.17.0"}
- mvxcvi/puget {:mvn/version "1.3.4"}
- org.babashka/json {:mvn/version "0.1.6"}
- table/table {:mvn/version "0.5.0"}}
+ :deps {dev.weavejester/cljfmt {:mvn/version "0.15.3"} ;; run cljfmt from bb
+ io.github.paintparty/bling {:mvn/version "0.8.8"} ;; printing bells and whistles
+ medley/medley {:mvn/version "1.3.0"} ;; utilities
+ metosin/malli {:mvn/version "0.17.0"} ;; data validation
+ mvxcvi/puget {:mvn/version "1.3.4"} ;; pretty printing
+ org.babashka/json {:mvn/version "0.1.6"} ;; json parsing
+ table/table {:mvn/version "0.5.0"} ;; table printing
+ }
:tasks
{:requires [[mage.cli :as cli]
[mage.color :as c]]
@@ -25,9 +27,9 @@
["./bin/mage alias print zsh" "Prints shell script to put into .zshrc for zsh users"]]
:requires [[mage.alias :as alias]]
:arg-schema [:or
- [:tuple]
- [:tuple [:enum "bash" "zsh" "fish"]]
- [:tuple [:enum "install" "print"] [:enum "bash" "zsh" "fish"]]]
+ [:tuple [:enum {:name "shell-type"} "bash" "zsh" "fish"]]
+ [:tuple [:enum {:name "mode"} "install" "print"]
+ [:enum {:name "shell-type"} "bash" "zsh" "fish"]]]
:task (task! (alias/instructions parsed))}
cljfmt-staged
@@ -45,7 +47,8 @@
["./bin/mage cljfmt-files src" "Format all files in src"]
["./bin/mage cljfmt-files -c src" "Check all files in src"]]
:options [["-c" "--force-check" "Check staged files"]]
- :arg-schema [:sequential [:string {:description "Files or directories to format."}]]
+ :arg-schema [:sequential [:string {:name "file-or-directory"
+ :description "Files or directories to format."}]]
:task (task! (format/files parsed))}
cljfmt-all
@@ -61,7 +64,7 @@
:examples [["./bin/mage cljfmt-updated" "Format updated files relative to HEAD"]
["./bin/mage cljfmt-updated master" "Format updated files relative to master"]]
:options [["-c" "--force-check" "Check staged files"]]
- :arg-schema [:or [:tuple] [:tuple :string]]
+ :arg-schema [:or [:tuple] [:tuple [:string {:name "git-ref" :description "Git ref to compare against"}]]]
:task (task! (format/updated parsed))}
kondo
@@ -110,8 +113,8 @@
["./bin/mage start-db mysql oldest" "start the oldest mysql db we support"]]
:requires [[mage.start-db :as start-db]]
:arg-schema [:tuple
- [:enum :postgres :mysql :mariadb :mongo]
- [:enum :oldest :latest]]
+ [:enum {:name "db"} :postgres :mysql :mariadb :mongo]
+ [:enum {:name "relative-version"} :oldest :latest]]
:db-info {:postgres {:ports {:oldest 5432 :latest 5433} :eol-url "https://endoflife.date/api/postgres.json"}
:mysql {:ports {:oldest 3308 :latest 3309} :eol-url "https://endoflife.date/api/mysql.json"}
:mariadb {:ports {:oldest 3306 :latest 3307} :eol-url "https://endoflife.date/api/mariadb.json"}
@@ -158,8 +161,8 @@
["-d" "--delete" "delete the old jar if found, by default does not re-download it"]
["-p" "--port PORT" "you can optionally provide a port to run the jar on, if not, we use a sane port."]]
:arg-schema [:or
- [:tuple [:string {:desc "version"}]]
- [:tuple [:string {:desc "version"}] [:string {:desc "path"}]]]
+ [:tuple [:string {:name "version"}]]
+ [:tuple [:string {:name "version"}] [:string {:name "path"}]]]
:usage-fn (fn [_]
(-> "Version can have the following forms:"
(str "\n -" (c/green "50") " for the latest enterprise version of release 50")
@@ -202,21 +205,22 @@
["-p" "--port PORT" "Port to use for the task, defaults to value in .nrepl-port"]]
:arg-schema [:sequential
{:details "If none are given, you will be prompted to select directories recursively from test/ and enterprise/backend/test/"}
- [:string {:description "Files or directories (recursive) to test"}]]
- :requires [[mage.quick-test-runner :as qtr]]
+ [:string {:name "file-or-dir" :description "Files or directories (recursive) to test"}]]
+ :requires [[mage.quick-test-runner :as qtr]]
:task (task! (qtr/go parsed))}
affected-modules
- {:doc "Prints information about which backend modules have changes compared to a Git ref e.g. `master`"
- :examples [["./bin/mage affected-modules master"]
+ {:doc "Which backend modules have changes?"
+ :examples [["./bin/mage affected-modules master" "prints changes compared to `master`"]
["./bin/mage affected-modules"]]
+ :arg-schema [:or [:tuple] [:tuple [:string {:name "git-ref"}]]]
:requires [[mage.modules]]
:task (task! (mage.modules/cli-print-affected-modules (:arguments parsed)))}
can-skip-driver-tests
- {:doc "Returns zero (success) if we can skip driver tests based on changes related to a Git ref e.g. `master`"
- :examples [["./bin/mage can-skip-driver-tests master"]
- ["./bin/mage can-skip-driver-tests"]]
+ {:doc "Can we skip driver tests for this branch?"
+ :examples [["./bin/mage can-skip-driver-tests master" "Returns zero (success) if we can skip driver tests based on changes related to `master`"]
+ ["./bin/mage can-skip-driver-tests" "Returns zero (success) if we can skip driver tests based on changes? (defaults to master)"]]
:requires [[mage.modules]]
:task (task! (mage.modules/cli-can-skip-driver-tests (:arguments parsed)))}
@@ -231,6 +235,33 @@
;; These tasks are more for internal tooling, e.g. for use from git hooks etc.
;; - hidden from `./bin/mage` listing and `bb tasks`, these all - start with a `-`
+ -install-merge-drivers
+ {:doc "Installs custom git merge drivers for YAML migration files."
+ :requires [[clojure.string :as str]
+ [mage.util :as u]]
+ :task (task!
+ (let [git-config-file (str u/project-root-directory "/.git/config")
+ the-include (str "\n"
+ "[include]\n"
+ " path=../.gitconfig\n")]
+ (when-not (str/includes? (slurp git-config-file) the-include)
+ (spit git-config-file the-include :append true))))}
+
+ -merge-yaml-migrations
+ {:doc "Custom git merge driver for YAML migration files."
+ :usage-fn (fn [_]
+ (str
+ "Treats each changeset as an atomic unit to avoid conflicts within changesets.\n"
+ "Preserves original formatting including blank lines and footer warnings"))
+ :arg-schema [:tuple
+ [:string {:name "base" :description "ancestor's version (base)"}]
+ [:string {:name "ours" :description "current version (ours)"}]
+ [:string {:name "theirs" :description "other branch's version (theirs)"}]
+ [:int {:name "conflict-marker-size" :description "conflict marker size" :min 0}]
+ [:string {:name "file-path" :description "file path"}]]
+ :requires [[mage.merge-yaml-migrations]]
+ :task (task! (mage.merge-yaml-migrations/-main parsed))}
+
-play
{:doc "Play sounds from mage.sound"
:examples [["./bin/mage -play success" "play success sound"] ["./bin/mage -play s" "play success sound"]
@@ -244,7 +275,7 @@
("e" "error") (sound/error)
("p" "ping") (sound/ping)
("w" "warning") (sound/warning)))}
-
+
-token-scan
{:doc "Security token scanner for detecting leaked API keys and secrets"
:examples [["./bin/mage -token-scan deps.edn bb.edn" "Scan specific files"]
@@ -281,10 +312,13 @@
{:doc "Check that code is readable parens a file"
:requires [[mage.readability-check :as readability]]
:arg-schema [:or
- [:tuple [:string {:description "File to check"}]]
+ [:tuple [:string {:name "file"
+ :description "File to check"}]]
[:tuple
- [:string {:description "File to check"}]
- [:int {:description "A line in the file" :min 1}]]]
+ [:string {:name "file"
+ :description "File to check"}]
+ [:int {:name "line-number"
+ :description "A line in the file" :min 1}]]]
:examples [["./bin/mage -check-readable dev/src/dev.clj"
"Check file for readability"]
["./bin/mage -check-readable dev/src/dev.clj 300"
@@ -315,7 +349,7 @@
:options [["-n" "--negate" "Negate the result of the calculation"]
["-a" "--also-add N" "Add N to the result of the calculation"]]
;; (optional) `:arg-schema` is a malli schema for the arguments passed to the task, after the options
- :arg-schema [:tuple :int [:enum :+ :-] :int]
+ :arg-schema [:tuple [:int {:name "number-a"}] [:enum {:name "operation"} :+ :-] [:int {:name "number-b"}]]
;; (optional) `:usage-fn` is a function called with the current-task map.
;; Returns a string containing extra-detailed usage information.
:usage-fn (fn [{:keys [doc] :as _task}]
@@ -353,5 +387,5 @@
{:doc "Removes all lines like '^<space>+$' from files in glob pattern."
:requires [[mage.escherize.fix-whitespace :as fix-whitespace]]
:examples [["mage -escherize-fix-whitespace '**/*.clj{,c,s}'" "Removes all lines like ' ' from all clj files"]]
- :arg-schema [:sequential [:string {:description "Glob pattern to match files"}]]
+ :arg-schema [:sequential [:string {:name "glob-pattern" :description "Glob pattern to match files"}]]
:task (task! (fix-whitespace/run-clean parsed))}}}
diff --git a/mage/resources/merge-yaml-migrations-test/README.md b/mage/resources/merge-yaml-migrations-test/README.md
new file mode 100644
index 00000000000..8d6f2669cce
--- /dev/null
+++ b/mage/resources/merge-yaml-migrations-test/README.md
@@ -0,0 +1,45 @@
+# Tests for merge-yaml-migrations
+
+This directory contains tests for the custom git merge driver `merge-yaml-migrations`.
+
+## Running Tests
+
+From the repository root:
+
+```bash
+./bin/merge-yaml-migrations-test/test_merge.clj
+```
+
+Or using babashka:
+
+```bash
+bb bin/merge-yaml-migrations-test/test_merge.clj
+```
+
+## Test Coverage
+
+The tests cover:
+
+1. **Clean merge scenarios:**
+ - Both branches add different changesets
+ - Only one branch adds changesets
+
+2. **Conflict scenarios:**
+ - Both branches modify the same changeset differently
+
+3. **Deletion scenarios:**
+ - One branch deletes a changeset while the other keeps it
+
+4. **Formatting preservation:**
+ - Single blank lines between changesets (not double)
+ - Footer warning preserved from the "ours" file
+ - 2-space indentation maintained
+ - objectQuotingStrategy header preserved
+
+## Test Structure
+
+Tests use temporary files and the `clojure.test` framework. Each test:
+1. Creates temporary YAML files for base, ours, and theirs versions
+2. Runs the merge driver
+3. Verifies the exit code and merged result
+4. Cleans up temp files automatically
diff --git a/mage/resources/merge-yaml-migrations-test/fixtures/base-conflict.yaml b/mage/resources/merge-yaml-migrations-test/fixtures/base-conflict.yaml
new file mode 100644
index 00000000000..0bf8be09125
--- /dev/null
+++ b/mage/resources/merge-yaml-migrations-test/fixtures/base-conflict.yaml
@@ -0,0 +1,14 @@
+databaseChangeLog:
+ - objectQuotingStrategy: QUOTE_ALL_OBJECTS
+
+ - changeSet:
+ id: v58.2025-10-30T09:03:00
+ author: alice
+ comment: Original
+ changes:
+ - addColumn:
+ tableName: test_table
+ columns:
+ - column:
+ name: col1
+ type: varchar(50)
diff --git a/mage/resources/merge-yaml-migrations-test/fixtures/base-simple.yaml b/mage/resources/merge-yaml-migrations-test/fixtures/base-simple.yaml
new file mode 100644
index 00000000000..87f0bc8ab32
--- /dev/null
+++ b/mage/resources/merge-yaml-migrations-test/fixtures/base-simple.yaml
@@ -0,0 +1,14 @@
+databaseChangeLog:
+ - objectQuotingStrategy: QUOTE_ALL_OBJECTS
+
+ - changeSet:
+ id: v58.2025-10-30T09:03:00
+ author: alice
+ comment: Original changeset
+ changes:
+ - addColumn:
+ tableName: test_table
+ columns:
+ - column:
+ name: test_column
+ type: varchar(50)
diff --git a/mage/resources/merge-yaml-migrations-test/fixtures/base-sort.yaml b/mage/resources/merge-yaml-migrations-test/fixtures/base-sort.yaml
new file mode 100644
index 00000000000..a08983e22ab
--- /dev/null
+++ b/mage/resources/merge-yaml-migrations-test/fixtures/base-sort.yaml
@@ -0,0 +1,8 @@
+databaseChangeLog:
+ - objectQuotingStrategy: QUOTE_ALL_OBJECTS
+
+ - changeSet:
+ id: v58.2025-10-30T09:00:00
+ author: alice
+ comment: First
+ changes: []
diff --git a/mage/resources/merge-yaml-migrations-test/fixtures/base-two-changesets.yaml b/mage/resources/merge-yaml-migrations-test/fixtures/base-two-changesets.yaml
new file mode 100644
index 00000000000..9ef4fc52171
--- /dev/null
+++ b/mage/resources/merge-yaml-migrations-test/fixtures/base-two-changesets.yaml
@@ -0,0 +1,18 @@
+databaseChangeLog:
+ - objectQuotingStrategy: QUOTE_ALL_OBJECTS
+
+ - changeSet:
+ id: v58.2025-10-30T09:03:00
+ author: alice
+ comment: First
+ changes:
+ - addColumn:
+ tableName: table1
+
+ - changeSet:
+ id: v58.2025-10-31T09:03:00
+ author: bob
+ comment: Second
+ changes:
+ - addColumn:
+ tableName: table2
diff --git a/mage/resources/merge-yaml-migrations-test/fixtures/base-with-footer.yaml b/mage/resources/merge-yaml-migrations-test/fixtures/base-with-footer.yaml
new file mode 100644
index 00000000000..ef6fa2740d0
--- /dev/null
+++ b/mage/resources/merge-yaml-migrations-test/fixtures/base-with-footer.yaml
@@ -0,0 +1,15 @@
+databaseChangeLog:
+ - objectQuotingStrategy: QUOTE_ALL_OBJECTS
+
+ - changeSet:
+ id: v58.2025-10-30T09:03:00
+ author: alice
+ comment: Test
+ changes:
+ - addColumn:
+ tableName: test
+
+# >>>>>>>>>> DO NOT ADD NEW MIGRATIONS BELOW THIS LINE! ADD THEM ABOVE <<<<<<<<<<
+
+# ADVICE:
+# Keep this at the bottom
diff --git a/mage/resources/merge-yaml-migrations-test/fixtures/ours-adds-later.yaml b/mage/resources/merge-yaml-migrations-test/fixtures/ours-adds-later.yaml
new file mode 100644
index 00000000000..3e70720f5a7
--- /dev/null
+++ b/mage/resources/merge-yaml-migrations-test/fixtures/ours-adds-later.yaml
@@ -0,0 +1,14 @@
+databaseChangeLog:
+ - objectQuotingStrategy: QUOTE_ALL_OBJECTS
+
+ - changeSet:
+ id: v58.2025-10-30T09:00:00
+ author: alice
+ comment: First
+ changes: []
+
+ - changeSet:
+ id: v58.2025-11-01T12:00:00
+ author: bob
+ comment: Added later by us
+ changes: []
diff --git a/mage/resources/merge-yaml-migrations-test/fixtures/ours-adds-one.yaml b/mage/resources/merge-yaml-migrations-test/fixtures/ours-adds-one.yaml
new file mode 100644
index 00000000000..863352a3b35
--- /dev/null
+++ b/mage/resources/merge-yaml-migrations-test/fixtures/ours-adds-one.yaml
@@ -0,0 +1,22 @@
+databaseChangeLog:
+ - objectQuotingStrategy: QUOTE_ALL_OBJECTS
+
+ - changeSet:
+ id: v58.2025-10-30T09:03:00
+ author: alice
+ comment: Original changeset
+ changes:
+ - addColumn:
+ tableName: test_table
+ columns:
+ - column:
+ name: test_column
+ type: varchar(50)
+
+ - changeSet:
+ id: v58.2025-11-01T10:00:00
+ author: bob
+ comment: Added by us
+ changes:
+ - createTable:
+ tableName: our_table
diff --git a/mage/resources/merge-yaml-migrations-test/fixtures/ours-adds-with-footer.yaml b/mage/resources/merge-yaml-migrations-test/fixtures/ours-adds-with-footer.yaml
new file mode 100644
index 00000000000..ad284e67817
--- /dev/null
+++ b/mage/resources/merge-yaml-migrations-test/fixtures/ours-adds-with-footer.yaml
@@ -0,0 +1,21 @@
+databaseChangeLog:
+ - objectQuotingStrategy: QUOTE_ALL_OBJECTS
+
+ - changeSet:
+ id: v58.2025-10-30T09:03:00
+ author: alice
+ comment: Test
+ changes:
+ - addColumn:
+ tableName: test
+
+ - changeSet:
+ id: v58.2025-11-01T10:00:00
+ author: bob
+ comment: Our change
+ changes: []
+
+# >>>>>>>>>> DO NOT ADD NEW MIGRATIONS BELOW THIS LINE! ADD THEM ABOVE <<<<<<<<<<
+
+# ADVICE:
+# Keep this at the bottom
diff --git a/mage/resources/merge-yaml-migrations-test/fixtures/ours-deletes-second.yaml b/mage/resources/merge-yaml-migrations-test/fixtures/ours-deletes-second.yaml
new file mode 100644
index 00000000000..2c0d4ec328d
--- /dev/null
+++ b/mage/resources/merge-yaml-migrations-test/fixtures/ours-deletes-second.yaml
@@ -0,0 +1,10 @@
+databaseChangeLog:
+ - objectQuotingStrategy: QUOTE_ALL_OBJECTS
+
+ - changeSet:
+ id: v58.2025-10-30T09:03:00
+ author: alice
+ comment: First
+ changes:
+ - addColumn:
+ tableName: table1
diff --git a/mage/resources/merge-yaml-migrations-test/fixtures/ours-modifies.yaml b/mage/resources/merge-yaml-migrations-test/fixtures/ours-modifies.yaml
new file mode 100644
index 00000000000..1182e59319b
--- /dev/null
+++ b/mage/resources/merge-yaml-migrations-test/fixtures/ours-modifies.yaml
@@ -0,0 +1,14 @@
+databaseChangeLog:
+ - objectQuotingStrategy: QUOTE_ALL_OBJECTS
+
+ - changeSet:
+ id: v58.2025-10-30T09:03:00
+ author: alice
+ comment: Modified by us
+ changes:
+ - addColumn:
+ tableName: test_table
+ columns:
+ - column:
+ name: col1
+ type: varchar(100)
diff --git a/mage/resources/merge-yaml-migrations-test/fixtures/theirs-adds-between.yaml b/mage/resources/merge-yaml-migrations-test/fixtures/theirs-adds-between.yaml
new file mode 100644
index 00000000000..b5cf2bb63ff
--- /dev/null
+++ b/mage/resources/merge-yaml-migrations-test/fixtures/theirs-adds-between.yaml
@@ -0,0 +1,14 @@
+databaseChangeLog:
+ - objectQuotingStrategy: QUOTE_ALL_OBJECTS
+
+ - changeSet:
+ id: v58.2025-10-30T09:00:00
+ author: alice
+ comment: First
+ changes: []
+
+ - changeSet:
+ id: v58.2025-10-31T10:00:00
+ author: charlie
+ comment: Added between by them
+ changes: []
diff --git a/mage/resources/merge-yaml-migrations-test/fixtures/theirs-adds-one.yaml b/mage/resources/merge-yaml-migrations-test/fixtures/theirs-adds-one.yaml
new file mode 100644
index 00000000000..3a48a583d77
--- /dev/null
+++ b/mage/resources/merge-yaml-migrations-test/fixtures/theirs-adds-one.yaml
@@ -0,0 +1,22 @@
+databaseChangeLog:
+ - objectQuotingStrategy: QUOTE_ALL_OBJECTS
+
+ - changeSet:
+ id: v58.2025-10-30T09:03:00
+ author: alice
+ comment: Original changeset
+ changes:
+ - addColumn:
+ tableName: test_table
+ columns:
+ - column:
+ name: test_column
+ type: varchar(50)
+
+ - changeSet:
+ id: v58.2025-11-02T11:00:00
+ author: charlie
+ comment: Added by them
+ changes:
+ - createTable:
+ tableName: their_table
diff --git a/mage/resources/merge-yaml-migrations-test/fixtures/theirs-modifies.yaml b/mage/resources/merge-yaml-migrations-test/fixtures/theirs-modifies.yaml
new file mode 100644
index 00000000000..67b112c92b9
--- /dev/null
+++ b/mage/resources/merge-yaml-migrations-test/fixtures/theirs-modifies.yaml
@@ -0,0 +1,14 @@
+databaseChangeLog:
+ - objectQuotingStrategy: QUOTE_ALL_OBJECTS
+
+ - changeSet:
+ id: v58.2025-10-30T09:03:00
+ author: alice
+ comment: Modified by them
+ changes:
+ - addColumn:
+ tableName: test_table
+ columns:
+ - column:
+ name: col1
+ type: varchar(200)
diff --git a/mage/resources/merge-yaml-migrations-test/test_merge.bats b/mage/resources/merge-yaml-migrations-test/test_merge.bats
new file mode 100755
index 00000000000..c59565892ee
--- /dev/null
+++ b/mage/resources/merge-yaml-migrations-test/test_merge.bats
@@ -0,0 +1,130 @@
+#!/usr/bin/env bats
+
+# Test suite for merge-yaml-migrations script
+
+setup() {
+ SCRIPT_DIR="$(cd "$(dirname "$BATS_TEST_FILENAME")" && pwd)"
+ MERGE_SCRIPT="$SCRIPT_DIR/../merge-yaml-migrations"
+ FIXTURES_DIR="$SCRIPT_DIR/fixtures"
+ TEMP_DIR="$(mktemp -d)"
+}
+
+teardown() {
+ rm -rf "$TEMP_DIR"
+}
+
+@test "clean merge: both branches add different changesets" {
+ run "$MERGE_SCRIPT" \
+ "$FIXTURES_DIR/base-simple.yaml" \
+ "$FIXTURES_DIR/ours-adds-one.yaml" \
+ "$FIXTURES_DIR/theirs-adds-one.yaml" \
+ 7 \
+ "$TEMP_DIR/result.yaml"
+
+ [ "$status" -eq 0 ]
+ [[ "$output" == *"Clean merge"* ]]
+
+ # Should have 3 changesets total
+ changeset_count=$(grep -c "^ - changeSet:" "$TEMP_DIR/result.yaml")
+ [ "$changeset_count" -eq 3 ]
+}
+
+@test "clean merge: only our branch adds a changeset" {
+ run "$MERGE_SCRIPT" \
+ "$FIXTURES_DIR/base-simple.yaml" \
+ "$FIXTURES_DIR/ours-adds-one.yaml" \
+ "$FIXTURES_DIR/base-simple.yaml" \
+ 7 \
+ "$TEMP_DIR/result.yaml"
+
+ [ "$status" -eq 0 ]
+
+ # Should have 2 changesets
+ changeset_count=$(grep -c "^ - changeSet:" "$TEMP_DIR/result.yaml")
+ [ "$changeset_count" -eq 2 ]
+}
+
+@test "conflict: both branches modify the same changeset differently" {
+ run "$MERGE_SCRIPT" \
+ "$FIXTURES_DIR/base-conflict.yaml" \
+ "$FIXTURES_DIR/ours-modifies.yaml" \
+ "$FIXTURES_DIR/theirs-modifies.yaml" \
+ 7 \
+ "$TEMP_DIR/result.yaml"
+
+ [ "$status" -eq 1 ]
+ [[ "$output" == *"Merge conflicts detected"* ]]
+
+ grep -q "MERGE CONFLICT" "$TEMP_DIR/result.yaml"
+}
+
+@test "deletion: changeset deletion is respected" {
+ run "$MERGE_SCRIPT" \
+ "$FIXTURES_DIR/base-two-changesets.yaml" \
+ "$FIXTURES_DIR/ours-deletes-second.yaml" \
+ "$FIXTURES_DIR/base-two-changesets.yaml" \
+ 7 \
+ "$TEMP_DIR/result.yaml"
+
+ [ "$status" -eq 0 ]
+
+ # Should have only 1 changeset (deletion respected)
+ changeset_count=$(grep -c "^ - changeSet:" "$TEMP_DIR/result.yaml")
+ [ "$changeset_count" -eq 1 ]
+}
+
+@test "preserves footer warning" {
+ run "$MERGE_SCRIPT" \
+ "$FIXTURES_DIR/base-with-footer.yaml" \
+ "$FIXTURES_DIR/ours-adds-with-footer.yaml" \
+ "$FIXTURES_DIR/base-with-footer.yaml" \
+ 7 \
+ "$TEMP_DIR/result.yaml"
+
+ [ "$status" -eq 0 ]
+
+ # Footer should be preserved
+ grep -q "# >>>>>>>>>> DO NOT ADD NEW MIGRATIONS BELOW THIS LINE!" "$TEMP_DIR/result.yaml"
+ grep -q "# ADVICE:" "$TEMP_DIR/result.yaml"
+}
+
+@test "preserves formatting and blank lines" {
+ run "$MERGE_SCRIPT" \
+ "$FIXTURES_DIR/base-simple.yaml" \
+ "$FIXTURES_DIR/ours-adds-one.yaml" \
+ "$FIXTURES_DIR/theirs-adds-one.yaml" \
+ 7 \
+ "$TEMP_DIR/result.yaml"
+
+ [ "$status" -eq 0 ]
+
+ # Should preserve quoting strategy line
+ grep -q " - objectQuotingStrategy: QUOTE_ALL_OBJECTS" "$TEMP_DIR/result.yaml"
+
+ # Should not have consecutive blank lines
+ ! grep -Pzo '\n\n\n' "$TEMP_DIR/result.yaml"
+}
+
+@test "sorts changesets by ID chronologically" {
+ run "$MERGE_SCRIPT" \
+ "$FIXTURES_DIR/base-sort.yaml" \
+ "$FIXTURES_DIR/ours-adds-later.yaml" \
+ "$FIXTURES_DIR/theirs-adds-between.yaml" \
+ 7 \
+ "$TEMP_DIR/result.yaml"
+
+ [ "$status" -eq 0 ]
+
+ # Extract changeset IDs in order
+ ids=$(grep "id: v58" "$TEMP_DIR/result.yaml" | sed 's/.*id: //' | tr '\n' ' ')
+
+ # Should be sorted chronologically
+ [[ "$ids" == *"v58.2025-10-30T09:00:00"* ]]
+ [[ "$ids" == *"v58.2025-10-31T10:00:00"* ]]
+ [[ "$ids" == *"v58.2025-11-01T12:00:00"* ]]
+
+ # Verify order by checking the "between" changeset comes before "later"
+ between_line=$(grep -n "v58.2025-10-31T10:00:00" "$TEMP_DIR/result.yaml" | cut -d: -f1)
+ later_line=$(grep -n "v58.2025-11-01T12:00:00" "$TEMP_DIR/result.yaml" | cut -d: -f1)
+ [ "$between_line" -lt "$later_line" ]
+}
diff --git a/mage/src/mage/merge_yaml_migrations.clj b/mage/src/mage/merge_yaml_migrations.clj
new file mode 100644
index 00000000000..436d7cbcac5
--- /dev/null
+++ b/mage/src/mage/merge_yaml_migrations.clj
@@ -0,0 +1,206 @@
+(ns mage.merge-yaml-migrations
+ (:require
+ [clj-yaml.core :as yaml]
+ [clojure.string :as str]
+ [mage.util :as u]
+ [medley.core :as m]))
+
+(set! *warn-on-reflection* true)
+
+;; Parse YAML file to Clojure data structure
+(defn- parse-yaml [file-path]
+ (try
+ (yaml/parse-string (slurp file-path))
+ (catch Exception e
+ (binding [*out* *err*]
+ (println "Error parsing YAML file:" file-path)
+ (println (.getMessage e)))
+ (u/exit 2))))
+
+(def ^:private FOOTER-PREFIX
+ "# >>>>>>>>>> DO NOT ADD NEW MIGRATIONS BELOW THIS LINE! ADD THEM ABOVE <<<<<<<<<<")
+
+(defn- extract-footer [content]
+ (let [idx (some-> content (str/index-of FOOTER-PREFIX))]
+ (if (and idx (pos? idx))
+ (subs content idx)
+ "")))
+
+;; Extract raw text for each changeset from the file
+(defn- extract-changeset-texts [file-content parsed-yaml]
+ (let [lines (str/split-lines file-content)
+ changelog (get parsed-yaml :databaseChangeLog)
+ changesets (filterv #(contains? % :changeSet) changelog)]
+ ;; For each changeset, find its text in the original file
+ (->> (for [cs changesets]
+ (let [cs-id (get-in cs [:changeSet :id])
+ ;; Find the line with "id: <cs-id>"
+ id-line-idx (first (keep-indexed
+ (fn [idx line]
+ (when (str/includes? line (str "id: " cs-id))
+ idx))
+ lines))
+ ;; Find the changeset start (line before with " - changeSet:")
+ cs-start-idx (when id-line-idx
+ (loop [idx (dec id-line-idx)]
+ (cond
+ (< idx 0) 0
+ (str/starts-with? (get lines idx) " - changeSet:") idx
+ :else (recur (dec idx)))))
+ ;; Find the changeset end (next line with " - changeSet:" or " - " at root level or "#" or end)
+ cs-end-idx (when cs-start-idx
+ (loop [idx (inc cs-start-idx)]
+ (cond
+ (>= idx (count lines)) (dec (count lines))
+ (let [line (get lines idx)]
+ (or (str/starts-with? line " - changeSet:")
+ (and (str/starts-with? line " - ")
+ (not (re-find #"^ - (changeSet|sql|addColumn|createTable|dropTable|addForeignKeyConstraint|dropForeignKeyConstraint|createIndex|dropIndex|addUniqueConstraint|dropUniqueConstraint|addPrimaryKey|dropPrimaryKey):" line)))
+ (str/starts-with? line "# ")))
+ (dec idx)
+ :else (recur (inc idx)))))]
+ (when (and cs-start-idx cs-end-idx)
+ [cs-id (str/join "\n" (subvec (vec lines) cs-start-idx (inc cs-end-idx)))])))
+ (into {}))))
+
+(defn- extract-changesets [parsed-yaml]
+ (->> (:databaseChangeLog parsed-yaml)
+ (filterv :changeSet)))
+
+(defn- cs-id [cs]
+ (-> cs :changeSet :id))
+
+(defn- merge-changesets [base-data ours-data theirs-data]
+ (let [base-cs (m/index-by cs-id (extract-changesets base-data))
+ ours-cs (m/index-by cs-id (extract-changesets ours-data))
+ theirs-cs (m/index-by cs-id (extract-changesets theirs-data))
+ all-ids (into #{}
+ (comp (map keys) cat)
+ [base-cs ours-cs theirs-cs])]
+ (reduce
+ (fn [acc id]
+ (let [base (get base-cs id)
+ ours (get ours-cs id)
+ theirs (get theirs-cs id)]
+ (cond
+ (and ours (not theirs) (not base)) ; Added by us
+ (conj acc {:id id :source :ours})
+ (and theirs (not ours) (not base)) ; Added by them
+ (conj acc {:id id :source :theirs})
+ (and base ours (not theirs)) ; they have deleted it
+ acc
+ (and base theirs (not ours)) ; we have deleted it
+ acc
+ (and base (not ours) (not theirs)) ; only in base, so both deleted
+ acc
+ (and base ours theirs) ; check for modifications
+ (cond
+ (= ours theirs) ; identical
+ (conj acc {:id id :source :ours})
+ (and (= theirs base) (not= ours base)) ; our mod
+ (conj acc {:id id :source :ours})
+ (and (= ours base) (not= theirs base)) ; their mod
+ (conj acc {:id id :source :theirs})
+ :else ; CONFLICT
+ (conj acc {:id id
+ :source :conflict
+ :ours ours
+ :theirs theirs}))
+ :else ; was I diligent enough?
+ (do
+ (binding [*out* *err*]
+ (println "Unexpected case for changeset:" id))
+ acc))))
+ []
+ all-ids)))
+
+(defn- format-conflict-changeset [id ours theirs]
+ (format
+ (str "<<<<<<< MERGE CONFLICT for changeset: %s, our version:\n"
+ "%s\n"
+ "======= Their version:\n"
+ "%s\n"
+ ">>>>>>>\n")
+ id ours theirs))
+
+(defn- merge-files [base ours theirs {:keys [_marker-size]}]
+ (let [ours-text (slurp ours)
+ theirs-text (slurp theirs)
+ ;; Parse yaml
+ base-data (parse-yaml base)
+ ours-data (parse-yaml ours)
+ theirs-data (parse-yaml theirs)
+ ;; Extract raw text for each changeset
+ ours-cs-texts (extract-changeset-texts ours-text ours-data)
+ theirs-cs-texts (extract-changeset-texts theirs-text theirs-data)
+ ;; Extract footer from ours file
+ footer (extract-footer ours-text)
+ ;; Perform merge
+ merged (merge-changesets base-data ours-data theirs-data)
+ ;; Sort merged changesets by ID (chronological order)
+ sorted-merged (sort-by :id merged)
+ ;; Build result by concatenating raw text blocks
+ header "databaseChangeLog:\n - objectQuotingStrategy: QUOTE_ALL_OBJECTS\n"
+ changeset-texts
+ (for [cs sorted-merged]
+ (let [text (case (:source cs)
+ :ours (get ours-cs-texts (:id cs))
+ :theirs (get theirs-cs-texts (:id cs))
+ :conflict (format-conflict-changeset ;; TODO: use marker-size here
+ (:id cs)
+ (get ours-cs-texts (:id cs))
+ (get theirs-cs-texts (:id cs))))]
+ ;; Remove trailing blank lines from each changeset
+ (str/replace text #"\n+$" "")))]
+ {:result (str header
+ "\n"
+ (str/join "\n\n" (filter some? changeset-texts))
+ "\n"
+ (when (seq footer) (str "\n" footer))
+ (when-not (str/ends-with? footer "\n") "\n"))
+ :conflicts (vec (keep #(when (= (:source %) :conflict) (:id %)) sorted-merged))
+ :cnt (count sorted-merged)}))
+
+;;
+
+;;
+;; Usage (called by git):
+;; merge-yaml-migrations %O %A %B %L %P
+;; %O = ancestor's version (base)
+;; %A = current version (ours)
+;; %B = other branch's version (theirs)
+;; %L = conflict marker size
+;; %P = file path
+;;
+;; Exit codes:
+;; 0 = clean merge
+;; 1 = conflicts detected (conflict markers added to file)
+;; >1 = merge failed
+
+(defn -main [{:keys [arguments]}]
+ (when (< (count arguments) 3)
+ (binding [*out* *err*]
+ (println "Usage: merge-yaml-migrations <base> <ours> <theirs> [conflict-marker-size] [file-path]")
+ (println "Will modify <ours>!"))
+ (u/exit 2))
+
+ (let [[base ours theirs marker-size target] arguments
+ {:keys [result conflicts cnt]} (merge-files
+ base
+ ours
+ theirs
+ {:marker-size marker-size})]
+ (spit (or target *out*) result)
+
+ ;; Exit with appropriate code
+ (when (seq conflicts)
+ (binding [*out* *err*]
+ (println "Merge conflicts detected in changesets:" (str/join ", " conflicts))
+ (println "Conflict markers added to file. Please resolve manually."))
+ (u/exit 1))
+
+ (println "Clean merge of" cnt "changesets")
+ (u/exit 0)))
+
+(when (= *file* (System/getProperty "babashka.file"))
+ (-main *command-line-args*))
diff --git a/mage/src/mage/token_scan.clj b/mage/src/mage/token_scan.clj
index af506c18594..e24e09860e2 100644
--- a/mage/src/mage/token_scan.clj
+++ b/mage/src/mage/token_scan.clj
@@ -89,29 +89,8 @@
:matches []
:error (str e)})))
-(defn- git-ignored-files
- "Returns a set of files that are ignored by git."
- [files]
- (println (c/yellow "Checking git ignore status for " (c/white (count files)) " files..."))
- (let [{:keys [exit]
- :as proc} (p/sh {:out :string
- :err :string
- :continue true
- :dir u/project-root-directory
- :in (str/join "\n" files)}
- "git" "check-ignore" "--stdin")
- output (:out proc)]
- (when (= 128 exit)
- (throw (ex-info "git check-ignore has failed with an exceptional status code: maybe git is not initialized in this directory or no gitignore file found."
- {:babashka/exit exit :git-error (:err proc)})))
- (->> output
- str/split-lines
- (remove str/blank?)
- (map str/trim)
- set)))
-
(defn remove-ignored [files]
- (let [ignored (git-ignored-files files) ;; Returns a set of git-ignored files
+ (let [ignored (u/git-ignored-files files) ;; Returns a set of git-ignored files
_ (u/debug "Ignored files:" (count ignored))
files' (remove ignored files)]
(u/debug "Files to scan:" (count files'))
diff --git a/mage/src/mage/util.clj b/mage/src/mage/util.clj
index 3d283d0d392..088c808fbb2 100644
--- a/mage/src/mage/util.clj
+++ b/mage/src/mage/util.clj
@@ -1,6 +1,7 @@
(ns mage.util
(:require
[babashka.fs :as fs]
+ [babashka.process :as p]
[babashka.tasks :refer [shell]]
[clojure.edn :as edn]
[clojure.java.io :as io]
@@ -261,3 +262,24 @@
([message exit-code]
(println message)
(throw (ex-info "" {:mage/quiet true :babashka/exit exit-code}))))
+
+(defn git-ignored-files
+ "Returns a set of files that are ignored by git."
+ [files]
+ (println (c/yellow "Checking git ignore status for " (c/white (count files)) " files..."))
+ (let [{:keys [exit]
+ :as proc} (p/sh {:out :string
+ :err :string
+ :continue true
+ :dir project-root-directory
+ :in (str/join "\n" files)}
+ "git" "check-ignore" "--stdin")
+ output (:out proc)]
+ (when (= 128 exit)
+ (throw (ex-info "git check-ignore has failed with an exceptional status code: maybe git is not initialized in this directory or no gitignore file found."
+ {:babashka/exit exit :git-error (:err proc)})))
+ (->> output
+ str/split-lines
+ (remove str/blank?)
+ (map str/trim)
+ set)))
diff --git a/mage/test/mage/core_test.clj b/mage/test/mage/core_test.clj
index c5cd923bed5..1dfb80f48ed 100644
--- a/mage/test/mage/core_test.clj
+++ b/mage/test/mage/core_test.clj
@@ -3,11 +3,17 @@
[babashka.tasks :as bt]
[clojure.string :as str]
[clojure.test :refer [deftest is testing]]
+ [mage.merge-yaml-migrations-test :as merge-yaml-migrations-test]
[mage.alias-test]
[mage.token-scan-test]
[mage.util :as u]
[mage.util-test]))
+(comment
+ mage.util-test/keep-me
+ merge-yaml-migrations-test/keep-me
+ token-scan-test/keep-me)
+
(set! *warn-on-reflection* true)
(deftest bin-mage-has-help-test
diff --git a/mage/test/mage/merge_yaml_migrations_test.clj b/mage/test/mage/merge_yaml_migrations_test.clj
new file mode 100644
index 00000000000..1bee276306e
--- /dev/null
+++ b/mage/test/mage/merge_yaml_migrations_test.clj
@@ -0,0 +1,100 @@
+(ns mage.merge-yaml-migrations-test
+ (:require
+ [clojure.string :as str]
+ [clojure.test :refer [deftest is testing]]
+ [mage.merge-yaml-migrations :as merge-yaml]))
+
+(def fixtures-dir "mage/resources/merge-yaml-migrations-test/fixtures")
+
+(defn- fixture-path [filename]
+ (str fixtures-dir "/" filename))
+
+(defn- merge-files-test
+ "Test helper that calls the private merge-files function"
+ [base ours theirs]
+ (#'merge-yaml/merge-files base ours theirs {:marker-size 7}))
+
+(deftest clean-merge-both-add-different-changesets
+ (testing "clean merge: both branches add different changesets"
+ (let [{:keys [result conflicts cnt]} (merge-files-test
+ (fixture-path "base-simple.yaml")
+ (fixture-path "ours-adds-one.yaml")
+ (fixture-path "theirs-adds-one.yaml"))]
+ (is (empty? conflicts) "Should have no conflicts")
+ (is (= 3 cnt) "Should have 3 changesets total")
+ (is (= 3 (count (re-seq #"(?m)^ - changeSet:" result)))
+ "Result should contain 3 changesets"))))
+
+(deftest clean-merge-only-ours-adds
+ (testing "clean merge: only our branch adds a changeset"
+ (let [{:keys [result conflicts cnt]} (merge-files-test
+ (fixture-path "base-simple.yaml")
+ (fixture-path "ours-adds-one.yaml")
+ (fixture-path "base-simple.yaml"))]
+ (is (empty? conflicts) "Should have no conflicts")
+ (is (= 2 cnt) "Should have 2 changesets")
+ (is (= 2 (count (re-seq #"(?m)^ - changeSet:" result)))
+ "Result should contain 2 changesets"))))
+
+(deftest conflict-both-modify-same-changeset
+ (testing "conflict: both branches modify the same changeset differently"
+ (let [{:keys [result conflicts cnt]} (merge-files-test
+ (fixture-path "base-conflict.yaml")
+ (fixture-path "ours-modifies.yaml")
+ (fixture-path "theirs-modifies.yaml"))]
+ (is (= 1 cnt))
+ (is (seq conflicts) "Should have conflicts")
+ (is (str/includes? result "MERGE CONFLICT")
+ "Result should contain conflict markers"))))
+
+(deftest deletion-changeset-deleted
+ (testing "deletion: changeset deletion is respected"
+ (let [{:keys [result conflicts cnt]} (merge-files-test
+ (fixture-path "base-two-changesets.yaml")
+ (fixture-path "ours-deletes-second.yaml")
+ (fixture-path "base-two-changesets.yaml"))]
+ (is (empty? conflicts) "Should have no conflicts")
+ (is (= 1 cnt) "Should have only 1 changeset (deletion respected)")
+ (is (= 1 (count (re-seq #"(?m)^ - changeSet:" result)))
+ "Result should contain 1 changeset"))))
+
+(deftest preserves-footer-warning
+ (testing "preserves footer warning"
+ (let [{:keys [result conflicts]} (merge-files-test
+ (fixture-path "base-with-footer.yaml")
+ (fixture-path "ours-adds-with-footer.yaml")
+ (fixture-path "base-with-footer.yaml"))]
+ (is (empty? conflicts) "Should have no conflicts")
+ (is (str/includes? result "# >>>>>>>>>> DO NOT ADD NEW MIGRATIONS BELOW THIS LINE!")
+ "Footer warning should be preserved")
+ (is (str/includes? result "# ADVICE:")
+ "Footer advice should be preserved"))))
+
+(deftest preserves-formatting
+ (testing "preserves formatting and blank lines"
+ (let [{:keys [result conflicts]} (merge-files-test
+ (fixture-path "base-simple.yaml")
+ (fixture-path "ours-adds-one.yaml")
+ (fixture-path "theirs-adds-one.yaml"))]
+ (is (empty? conflicts) "Should have no conflicts")
+ (is (str/includes? result " - objectQuotingStrategy: QUOTE_ALL_OBJECTS")
+ "Should preserve quoting strategy line")
+ (is (not (str/includes? result "\n\n\n"))
+ "Should not have consecutive blank lines"))))
+
+(deftest sorts-changesets-chronologically
+ (testing "sorts changesets by ID chronologically"
+ (let [{:keys [result conflicts]} (merge-files-test
+ (fixture-path "base-sort.yaml")
+ (fixture-path "ours-adds-later.yaml")
+ (fixture-path "theirs-adds-between.yaml"))]
+ (is (empty? conflicts) "Should have no conflicts")
+
+ ;; Extract changeset IDs in order
+ (let [ids (re-seq #"id: v58\.2025-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}" result)
+ ids-str (map #(subs % 4) ids)] ;; Remove "id: " prefix
+ (is (= ["v58.2025-10-30T09:00:00"
+ "v58.2025-10-31T10:00:00"
+ "v58.2025-11-01T12:00:00"]
+ ids-str)
+ "Changesets should be sorted chronologically")))))
diff --git a/mage/test/mage/util_test.clj b/mage/test/mage/util_test.clj
index 6b7bdeb081a..a7bffb68dad 100644
--- a/mage/test/mage/util_test.clj
+++ b/mage/test/mage/util_test.clj
@@ -17,7 +17,10 @@
(str/replace (re-pattern (str new-content "$")) ""))))
(deftest updated-files-returns-all-kinds-of-files
- (let [extension->path (-> (group-by fs/extension (fs/glob u/project-root-directory "**.*"))
+ (let [all-files (map str (fs/glob u/project-root-directory "**.*"))
+ git-ignored-files (u/git-ignored-files all-files)
+ ok-files (remove git-ignored-files all-files)
+ extension->path (-> (group-by fs/extension ok-files)
(update-vals (comp str first))
(select-keys ["clj" "cljc" "cljs" "edn" "js" "ts" "jsx" "json" "css" "html" "md"]))]
(doseq [[_ path] extension->path]
diff --git a/package.json b/package.json
index 99406ac8b97..62392040945 100644
--- a/package.json
+++ b/package.json
@@ -518,7 +518,8 @@
"prettier --write"
],
"**/*": [
- "./bin/mage -token-scan"
+ "./bin/mage -token-scan",
+ "./bin/mage -install-merge-drivers"
]
},
"browserslist": [