The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
yaml migrations helper, into mage (#66486)
bryanbcd33be0ae1f
diff --git a/.gitattributes b/.gitattributesnew file mode 100644index 00000000000..c8dd1874eeb--- /dev/null+++ b/.gitattributes@@ -0,0 +1,5 @@+# Custom merge driver for YAML migration files+# Treats each changeset as an atomic unit to avoid conflicts within changesets+# Execute this in your shell to enable custom merge driver:+# echo '\n[include]\npath=../.gitconfig' >> .git/config+resources/migrations/*.yaml merge=yaml-migrationsdiff --git a/.gitconfig b/.gitconfignew file mode 100644index 00000000000..6feb99900d6--- /dev/null+++ b/.gitconfig@@ -0,0 +1,5 @@+# execute this in your shell to enable custom merge driver:+# echo '\n[include]\npath=../.gitconfig' >> .git/config+[merge "yaml-migrations"]+ name = YAML migration changeset-aware merge+ driver = mage -merge-yaml-migrations %O %A %B %L %Pdiff --git a/bb.edn b/bb.ednindex b9700e701df..752b5c217d4 100644--- a/bb.edn+++ b/bb.edn@@ -2,12 +2,14 @@;; we put path as bin, and everything is in the ./mage subdirectory,;; so the namespaces are mage.cli, mage.format, etc.:paths ["mage/src" "mage/test" "bin/lint-migrations-file/src"]- :deps {dev.weavejester/cljfmt {:mvn/version "0.15.3"}- io.github.paintparty/bling {:mvn/version "0.8.8"}- metosin/malli {:mvn/version "0.17.0"}- mvxcvi/puget {:mvn/version "1.3.4"}- org.babashka/json {:mvn/version "0.1.6"}- table/table {:mvn/version "0.5.0"}}+ :deps {dev.weavejester/cljfmt {:mvn/version "0.15.3"} ;; run cljfmt from bb+ io.github.paintparty/bling {:mvn/version "0.8.8"} ;; printing bells and whistles+ medley/medley {:mvn/version "1.3.0"} ;; utilities+ metosin/malli {:mvn/version "0.17.0"} ;; data validation+ mvxcvi/puget {:mvn/version "1.3.4"} ;; pretty printing+ org.babashka/json {:mvn/version "0.1.6"} ;; json parsing+ table/table {:mvn/version "0.5.0"} ;; table printing+ }:tasks{:requires [[mage.cli :as cli][mage.color :as c]]@@ -25,9 +27,9 @@["./bin/mage alias print zsh" "Prints shell script to put into .zshrc for zsh users"]]:requires [[mage.alias :as alias]]:arg-schema [:or- [:tuple]- [:tuple [:enum "bash" "zsh" "fish"]]- [:tuple [:enum "install" "print"] [:enum "bash" "zsh" "fish"]]]+ [:tuple [:enum {:name "shell-type"} "bash" "zsh" "fish"]]+ [:tuple [:enum {:name "mode"} "install" "print"]+ [:enum {:name "shell-type"} "bash" "zsh" "fish"]]]:task (task! (alias/instructions parsed))}cljfmt-staged@@ -45,7 +47,8 @@["./bin/mage cljfmt-files src" "Format all files in src"]["./bin/mage cljfmt-files -c src" "Check all files in src"]]:options [["-c" "--force-check" "Check staged files"]]- :arg-schema [:sequential [:string {:description "Files or directories to format."}]]+ :arg-schema [:sequential [:string {:name "file-or-directory"+ :description "Files or directories to format."}]]:task (task! (format/files parsed))}cljfmt-all@@ -61,7 +64,7 @@:examples [["./bin/mage cljfmt-updated" "Format updated files relative to HEAD"]["./bin/mage cljfmt-updated master" "Format updated files relative to master"]]:options [["-c" "--force-check" "Check staged files"]]- :arg-schema [:or [:tuple] [:tuple :string]]+ :arg-schema [:or [:tuple] [:tuple [:string {:name "git-ref" :description "Git ref to compare against"}]]]:task (task! (format/updated parsed))}kondo@@ -110,8 +113,8 @@["./bin/mage start-db mysql oldest" "start the oldest mysql db we support"]]:requires [[mage.start-db :as start-db]]:arg-schema [:tuple- [:enum :postgres :mysql :mariadb :mongo]- [:enum :oldest :latest]]+ [:enum {:name "db"} :postgres :mysql :mariadb :mongo]+ [:enum {:name "relative-version"} :oldest :latest]]:db-info {:postgres {:ports {:oldest 5432 :latest 5433} :eol-url "https://endoflife.date/api/postgres.json"}:mysql {:ports {:oldest 3308 :latest 3309} :eol-url "https://endoflife.date/api/mysql.json"}:mariadb {:ports {:oldest 3306 :latest 3307} :eol-url "https://endoflife.date/api/mariadb.json"}@@ -158,8 +161,8 @@["-d" "--delete" "delete the old jar if found, by default does not re-download it"]["-p" "--port PORT" "you can optionally provide a port to run the jar on, if not, we use a sane port."]]:arg-schema [:or- [:tuple [:string {:desc "version"}]]- [:tuple [:string {:desc "version"}] [:string {:desc "path"}]]]+ [:tuple [:string {:name "version"}]]+ [:tuple [:string {:name "version"}] [:string {:name "path"}]]]:usage-fn (fn [_](-> "Version can have the following forms:"(str "\n -" (c/green "50") " for the latest enterprise version of release 50")@@ -202,21 +205,22 @@["-p" "--port PORT" "Port to use for the task, defaults to value in .nrepl-port"]]:arg-schema [:sequential{:details "If none are given, you will be prompted to select directories recursively from test/ and enterprise/backend/test/"}- [:string {:description "Files or directories (recursive) to test"}]]- :requires [[mage.quick-test-runner :as qtr]]+ [:string {:name "file-or-dir" :description "Files or directories (recursive) to test"}]]+ :requires [[mage.quick-test-runner :as qtr]]:task (task! (qtr/go parsed))}affected-modules- {:doc "Prints information about which backend modules have changes compared to a Git ref e.g. `master`"- :examples [["./bin/mage affected-modules master"]+ {:doc "Which backend modules have changes?"+ :examples [["./bin/mage affected-modules master" "prints changes compared to `master`"]["./bin/mage affected-modules"]]+ :arg-schema [:or [:tuple] [:tuple [:string {:name "git-ref"}]]]:requires [[mage.modules]]:task (task! (mage.modules/cli-print-affected-modules (:arguments parsed)))}can-skip-driver-tests- {:doc "Returns zero (success) if we can skip driver tests based on changes related to a Git ref e.g. `master`"- :examples [["./bin/mage can-skip-driver-tests master"]- ["./bin/mage can-skip-driver-tests"]]+ {:doc "Can we skip driver tests for this branch?"+ :examples [["./bin/mage can-skip-driver-tests master" "Returns zero (success) if we can skip driver tests based on changes related to `master`"]+ ["./bin/mage can-skip-driver-tests" "Returns zero (success) if we can skip driver tests based on changes? (defaults to master)"]]:requires [[mage.modules]]:task (task! (mage.modules/cli-can-skip-driver-tests (:arguments parsed)))}@@ -231,6 +235,33 @@;; These tasks are more for internal tooling, e.g. for use from git hooks etc.;; - hidden from `./bin/mage` listing and `bb tasks`, these all - start with a `-`+ -install-merge-drivers+ {:doc "Installs custom git merge drivers for YAML migration files."+ :requires [[clojure.string :as str]+ [mage.util :as u]]+ :task (task!+ (let [git-config-file (str u/project-root-directory "/.git/config")+ the-include (str "\n"+ "[include]\n"+ " path=../.gitconfig\n")]+ (when-not (str/includes? (slurp git-config-file) the-include)+ (spit git-config-file the-include :append true))))}++ -merge-yaml-migrations+ {:doc "Custom git merge driver for YAML migration files."+ :usage-fn (fn [_]+ (str+ "Treats each changeset as an atomic unit to avoid conflicts within changesets.\n"+ "Preserves original formatting including blank lines and footer warnings"))+ :arg-schema [:tuple+ [:string {:name "base" :description "ancestor's version (base)"}]+ [:string {:name "ours" :description "current version (ours)"}]+ [:string {:name "theirs" :description "other branch's version (theirs)"}]+ [:int {:name "conflict-marker-size" :description "conflict marker size" :min 0}]+ [:string {:name "file-path" :description "file path"}]]+ :requires [[mage.merge-yaml-migrations]]+ :task (task! (mage.merge-yaml-migrations/-main parsed))}+-play{:doc "Play sounds from mage.sound":examples [["./bin/mage -play success" "play success sound"] ["./bin/mage -play s" "play success sound"]@@ -244,7 +275,7 @@("e" "error") (sound/error)("p" "ping") (sound/ping)("w" "warning") (sound/warning)))}-+-token-scan{:doc "Security token scanner for detecting leaked API keys and secrets":examples [["./bin/mage -token-scan deps.edn bb.edn" "Scan specific files"]@@ -281,10 +312,13 @@{:doc "Check that code is readable parens a file":requires [[mage.readability-check :as readability]]:arg-schema [:or- [:tuple [:string {:description "File to check"}]]+ [:tuple [:string {:name "file"+ :description "File to check"}]][:tuple- [:string {:description "File to check"}]- [:int {:description "A line in the file" :min 1}]]]+ [:string {:name "file"+ :description "File to check"}]+ [:int {:name "line-number"+ :description "A line in the file" :min 1}]]]:examples [["./bin/mage -check-readable dev/src/dev.clj""Check file for readability"]["./bin/mage -check-readable dev/src/dev.clj 300"@@ -315,7 +349,7 @@:options [["-n" "--negate" "Negate the result of the calculation"]["-a" "--also-add N" "Add N to the result of the calculation"]];; (optional) `:arg-schema` is a malli schema for the arguments passed to the task, after the options- :arg-schema [:tuple :int [:enum :+ :-] :int]+ :arg-schema [:tuple [:int {:name "number-a"}] [:enum {:name "operation"} :+ :-] [:int {:name "number-b"}]];; (optional) `:usage-fn` is a function called with the current-task map.;; Returns a string containing extra-detailed usage information.:usage-fn (fn [{:keys [doc] :as _task}]@@ -353,5 +387,5 @@{:doc "Removes all lines like '^<space>+$' from files in glob pattern.":requires [[mage.escherize.fix-whitespace :as fix-whitespace]]:examples [["mage -escherize-fix-whitespace '**/*.clj{,c,s}'" "Removes all lines like ' ' from all clj files"]]- :arg-schema [:sequential [:string {:description "Glob pattern to match files"}]]+ :arg-schema [:sequential [:string {:name "glob-pattern" :description "Glob pattern to match files"}]]:task (task! (fix-whitespace/run-clean parsed))}}}diff --git a/mage/resources/merge-yaml-migrations-test/README.md b/mage/resources/merge-yaml-migrations-test/README.mdnew file mode 100644index 00000000000..8d6f2669cce--- /dev/null+++ b/mage/resources/merge-yaml-migrations-test/README.md@@ -0,0 +1,45 @@+# Tests for merge-yaml-migrations++This directory contains tests for the custom git merge driver `merge-yaml-migrations`.++## Running Tests++From the repository root:++```bash+./bin/merge-yaml-migrations-test/test_merge.clj+```++Or using babashka:++```bash+bb bin/merge-yaml-migrations-test/test_merge.clj+```++## Test Coverage++The tests cover:++1. **Clean merge scenarios:**+ - Both branches add different changesets+ - Only one branch adds changesets++2. **Conflict scenarios:**+ - Both branches modify the same changeset differently++3. **Deletion scenarios:**+ - One branch deletes a changeset while the other keeps it++4. **Formatting preservation:**+ - Single blank lines between changesets (not double)+ - Footer warning preserved from the "ours" file+ - 2-space indentation maintained+ - objectQuotingStrategy header preserved++## Test Structure++Tests use temporary files and the `clojure.test` framework. Each test:+1. Creates temporary YAML files for base, ours, and theirs versions+2. Runs the merge driver+3. Verifies the exit code and merged result+4. Cleans up temp files automaticallydiff --git a/mage/resources/merge-yaml-migrations-test/fixtures/base-conflict.yaml b/mage/resources/merge-yaml-migrations-test/fixtures/base-conflict.yamlnew file mode 100644index 00000000000..0bf8be09125--- /dev/null+++ b/mage/resources/merge-yaml-migrations-test/fixtures/base-conflict.yaml@@ -0,0 +1,14 @@+databaseChangeLog:+ - objectQuotingStrategy: QUOTE_ALL_OBJECTS++ - changeSet:+ id: v58.2025-10-30T09:03:00+ author: alice+ comment: Original+ changes:+ - addColumn:+ tableName: test_table+ columns:+ - column:+ name: col1+ type: varchar(50)diff --git a/mage/resources/merge-yaml-migrations-test/fixtures/base-simple.yaml b/mage/resources/merge-yaml-migrations-test/fixtures/base-simple.yamlnew file mode 100644index 00000000000..87f0bc8ab32--- /dev/null+++ b/mage/resources/merge-yaml-migrations-test/fixtures/base-simple.yaml@@ -0,0 +1,14 @@+databaseChangeLog:+ - objectQuotingStrategy: QUOTE_ALL_OBJECTS++ - changeSet:+ id: v58.2025-10-30T09:03:00+ author: alice+ comment: Original changeset+ changes:+ - addColumn:+ tableName: test_table+ columns:+ - column:+ name: test_column+ type: varchar(50)diff --git a/mage/resources/merge-yaml-migrations-test/fixtures/base-sort.yaml b/mage/resources/merge-yaml-migrations-test/fixtures/base-sort.yamlnew file mode 100644index 00000000000..a08983e22ab--- /dev/null+++ b/mage/resources/merge-yaml-migrations-test/fixtures/base-sort.yaml@@ -0,0 +1,8 @@+databaseChangeLog:+ - objectQuotingStrategy: QUOTE_ALL_OBJECTS++ - changeSet:+ id: v58.2025-10-30T09:00:00+ author: alice+ comment: First+ changes: []diff --git a/mage/resources/merge-yaml-migrations-test/fixtures/base-two-changesets.yaml b/mage/resources/merge-yaml-migrations-test/fixtures/base-two-changesets.yamlnew file mode 100644index 00000000000..9ef4fc52171--- /dev/null+++ b/mage/resources/merge-yaml-migrations-test/fixtures/base-two-changesets.yaml@@ -0,0 +1,18 @@+databaseChangeLog:+ - objectQuotingStrategy: QUOTE_ALL_OBJECTS++ - changeSet:+ id: v58.2025-10-30T09:03:00+ author: alice+ comment: First+ changes:+ - addColumn:+ tableName: table1++ - changeSet:+ id: v58.2025-10-31T09:03:00+ author: bob+ comment: Second+ changes:+ - addColumn:+ tableName: table2diff --git a/mage/resources/merge-yaml-migrations-test/fixtures/base-with-footer.yaml b/mage/resources/merge-yaml-migrations-test/fixtures/base-with-footer.yamlnew file mode 100644index 00000000000..ef6fa2740d0--- /dev/null+++ b/mage/resources/merge-yaml-migrations-test/fixtures/base-with-footer.yaml@@ -0,0 +1,15 @@+databaseChangeLog:+ - objectQuotingStrategy: QUOTE_ALL_OBJECTS++ - changeSet:+ id: v58.2025-10-30T09:03:00+ author: alice+ comment: Test+ changes:+ - addColumn:+ tableName: test++# >>>>>>>>>> DO NOT ADD NEW MIGRATIONS BELOW THIS LINE! ADD THEM ABOVE <<<<<<<<<<++# ADVICE:+# Keep this at the bottomdiff --git a/mage/resources/merge-yaml-migrations-test/fixtures/ours-adds-later.yaml b/mage/resources/merge-yaml-migrations-test/fixtures/ours-adds-later.yamlnew file mode 100644index 00000000000..3e70720f5a7--- /dev/null+++ b/mage/resources/merge-yaml-migrations-test/fixtures/ours-adds-later.yaml@@ -0,0 +1,14 @@+databaseChangeLog:+ - objectQuotingStrategy: QUOTE_ALL_OBJECTS++ - changeSet:+ id: v58.2025-10-30T09:00:00+ author: alice+ comment: First+ changes: []++ - changeSet:+ id: v58.2025-11-01T12:00:00+ author: bob+ comment: Added later by us+ changes: []diff --git a/mage/resources/merge-yaml-migrations-test/fixtures/ours-adds-one.yaml b/mage/resources/merge-yaml-migrations-test/fixtures/ours-adds-one.yamlnew file mode 100644index 00000000000..863352a3b35--- /dev/null+++ b/mage/resources/merge-yaml-migrations-test/fixtures/ours-adds-one.yaml@@ -0,0 +1,22 @@+databaseChangeLog:+ - objectQuotingStrategy: QUOTE_ALL_OBJECTS++ - changeSet:+ id: v58.2025-10-30T09:03:00+ author: alice+ comment: Original changeset+ changes:+ - addColumn:+ tableName: test_table+ columns:+ - column:+ name: test_column+ type: varchar(50)++ - changeSet:+ id: v58.2025-11-01T10:00:00+ author: bob+ comment: Added by us+ changes:+ - createTable:+ tableName: our_tablediff --git a/mage/resources/merge-yaml-migrations-test/fixtures/ours-adds-with-footer.yaml b/mage/resources/merge-yaml-migrations-test/fixtures/ours-adds-with-footer.yamlnew file mode 100644index 00000000000..ad284e67817--- /dev/null+++ b/mage/resources/merge-yaml-migrations-test/fixtures/ours-adds-with-footer.yaml@@ -0,0 +1,21 @@+databaseChangeLog:+ - objectQuotingStrategy: QUOTE_ALL_OBJECTS++ - changeSet:+ id: v58.2025-10-30T09:03:00+ author: alice+ comment: Test+ changes:+ - addColumn:+ tableName: test++ - changeSet:+ id: v58.2025-11-01T10:00:00+ author: bob+ comment: Our change+ changes: []++# >>>>>>>>>> DO NOT ADD NEW MIGRATIONS BELOW THIS LINE! ADD THEM ABOVE <<<<<<<<<<++# ADVICE:+# Keep this at the bottomdiff --git a/mage/resources/merge-yaml-migrations-test/fixtures/ours-deletes-second.yaml b/mage/resources/merge-yaml-migrations-test/fixtures/ours-deletes-second.yamlnew file mode 100644index 00000000000..2c0d4ec328d--- /dev/null+++ b/mage/resources/merge-yaml-migrations-test/fixtures/ours-deletes-second.yaml@@ -0,0 +1,10 @@+databaseChangeLog:+ - objectQuotingStrategy: QUOTE_ALL_OBJECTS++ - changeSet:+ id: v58.2025-10-30T09:03:00+ author: alice+ comment: First+ changes:+ - addColumn:+ tableName: table1diff --git a/mage/resources/merge-yaml-migrations-test/fixtures/ours-modifies.yaml b/mage/resources/merge-yaml-migrations-test/fixtures/ours-modifies.yamlnew file mode 100644index 00000000000..1182e59319b--- /dev/null+++ b/mage/resources/merge-yaml-migrations-test/fixtures/ours-modifies.yaml@@ -0,0 +1,14 @@+databaseChangeLog:+ - objectQuotingStrategy: QUOTE_ALL_OBJECTS++ - changeSet:+ id: v58.2025-10-30T09:03:00+ author: alice+ comment: Modified by us+ changes:+ - addColumn:+ tableName: test_table+ columns:+ - column:+ name: col1+ type: varchar(100)diff --git a/mage/resources/merge-yaml-migrations-test/fixtures/theirs-adds-between.yaml b/mage/resources/merge-yaml-migrations-test/fixtures/theirs-adds-between.yamlnew file mode 100644index 00000000000..b5cf2bb63ff--- /dev/null+++ b/mage/resources/merge-yaml-migrations-test/fixtures/theirs-adds-between.yaml@@ -0,0 +1,14 @@+databaseChangeLog:+ - objectQuotingStrategy: QUOTE_ALL_OBJECTS++ - changeSet:+ id: v58.2025-10-30T09:00:00+ author: alice+ comment: First+ changes: []++ - changeSet:+ id: v58.2025-10-31T10:00:00+ author: charlie+ comment: Added between by them+ changes: []diff --git a/mage/resources/merge-yaml-migrations-test/fixtures/theirs-adds-one.yaml b/mage/resources/merge-yaml-migrations-test/fixtures/theirs-adds-one.yamlnew file mode 100644index 00000000000..3a48a583d77--- /dev/null+++ b/mage/resources/merge-yaml-migrations-test/fixtures/theirs-adds-one.yaml@@ -0,0 +1,22 @@+databaseChangeLog:+ - objectQuotingStrategy: QUOTE_ALL_OBJECTS++ - changeSet:+ id: v58.2025-10-30T09:03:00+ author: alice+ comment: Original changeset+ changes:+ - addColumn:+ tableName: test_table+ columns:+ - column:+ name: test_column+ type: varchar(50)++ - changeSet:+ id: v58.2025-11-02T11:00:00+ author: charlie+ comment: Added by them+ changes:+ - createTable:+ tableName: their_tablediff --git a/mage/resources/merge-yaml-migrations-test/fixtures/theirs-modifies.yaml b/mage/resources/merge-yaml-migrations-test/fixtures/theirs-modifies.yamlnew file mode 100644index 00000000000..67b112c92b9--- /dev/null+++ b/mage/resources/merge-yaml-migrations-test/fixtures/theirs-modifies.yaml@@ -0,0 +1,14 @@+databaseChangeLog:+ - objectQuotingStrategy: QUOTE_ALL_OBJECTS++ - changeSet:+ id: v58.2025-10-30T09:03:00+ author: alice+ comment: Modified by them+ changes:+ - addColumn:+ tableName: test_table+ columns:+ - column:+ name: col1+ type: varchar(200)diff --git a/mage/resources/merge-yaml-migrations-test/test_merge.bats b/mage/resources/merge-yaml-migrations-test/test_merge.batsnew file mode 100755index 00000000000..c59565892ee--- /dev/null+++ b/mage/resources/merge-yaml-migrations-test/test_merge.bats@@ -0,0 +1,130 @@+#!/usr/bin/env bats++# Test suite for merge-yaml-migrations script++setup() {+ SCRIPT_DIR="$(cd "$(dirname "$BATS_TEST_FILENAME")" && pwd)"+ MERGE_SCRIPT="$SCRIPT_DIR/../merge-yaml-migrations"+ FIXTURES_DIR="$SCRIPT_DIR/fixtures"+ TEMP_DIR="$(mktemp -d)"+}++teardown() {+ rm -rf "$TEMP_DIR"+}++@test "clean merge: both branches add different changesets" {+ run "$MERGE_SCRIPT" \+ "$FIXTURES_DIR/base-simple.yaml" \+ "$FIXTURES_DIR/ours-adds-one.yaml" \+ "$FIXTURES_DIR/theirs-adds-one.yaml" \+ 7 \+ "$TEMP_DIR/result.yaml"++ [ "$status" -eq 0 ]+ [[ "$output" == *"Clean merge"* ]]++ # Should have 3 changesets total+ changeset_count=$(grep -c "^ - changeSet:" "$TEMP_DIR/result.yaml")+ [ "$changeset_count" -eq 3 ]+}++@test "clean merge: only our branch adds a changeset" {+ run "$MERGE_SCRIPT" \+ "$FIXTURES_DIR/base-simple.yaml" \+ "$FIXTURES_DIR/ours-adds-one.yaml" \+ "$FIXTURES_DIR/base-simple.yaml" \+ 7 \+ "$TEMP_DIR/result.yaml"++ [ "$status" -eq 0 ]++ # Should have 2 changesets+ changeset_count=$(grep -c "^ - changeSet:" "$TEMP_DIR/result.yaml")+ [ "$changeset_count" -eq 2 ]+}++@test "conflict: both branches modify the same changeset differently" {+ run "$MERGE_SCRIPT" \+ "$FIXTURES_DIR/base-conflict.yaml" \+ "$FIXTURES_DIR/ours-modifies.yaml" \+ "$FIXTURES_DIR/theirs-modifies.yaml" \+ 7 \+ "$TEMP_DIR/result.yaml"++ [ "$status" -eq 1 ]+ [[ "$output" == *"Merge conflicts detected"* ]]++ grep -q "MERGE CONFLICT" "$TEMP_DIR/result.yaml"+}++@test "deletion: changeset deletion is respected" {+ run "$MERGE_SCRIPT" \+ "$FIXTURES_DIR/base-two-changesets.yaml" \+ "$FIXTURES_DIR/ours-deletes-second.yaml" \+ "$FIXTURES_DIR/base-two-changesets.yaml" \+ 7 \+ "$TEMP_DIR/result.yaml"++ [ "$status" -eq 0 ]++ # Should have only 1 changeset (deletion respected)+ changeset_count=$(grep -c "^ - changeSet:" "$TEMP_DIR/result.yaml")+ [ "$changeset_count" -eq 1 ]+}++@test "preserves footer warning" {+ run "$MERGE_SCRIPT" \+ "$FIXTURES_DIR/base-with-footer.yaml" \+ "$FIXTURES_DIR/ours-adds-with-footer.yaml" \+ "$FIXTURES_DIR/base-with-footer.yaml" \+ 7 \+ "$TEMP_DIR/result.yaml"++ [ "$status" -eq 0 ]++ # Footer should be preserved+ grep -q "# >>>>>>>>>> DO NOT ADD NEW MIGRATIONS BELOW THIS LINE!" "$TEMP_DIR/result.yaml"+ grep -q "# ADVICE:" "$TEMP_DIR/result.yaml"+}++@test "preserves formatting and blank lines" {+ run "$MERGE_SCRIPT" \+ "$FIXTURES_DIR/base-simple.yaml" \+ "$FIXTURES_DIR/ours-adds-one.yaml" \+ "$FIXTURES_DIR/theirs-adds-one.yaml" \+ 7 \+ "$TEMP_DIR/result.yaml"++ [ "$status" -eq 0 ]++ # Should preserve quoting strategy line+ grep -q " - objectQuotingStrategy: QUOTE_ALL_OBJECTS" "$TEMP_DIR/result.yaml"++ # Should not have consecutive blank lines+ ! grep -Pzo '\n\n\n' "$TEMP_DIR/result.yaml"+}++@test "sorts changesets by ID chronologically" {+ run "$MERGE_SCRIPT" \+ "$FIXTURES_DIR/base-sort.yaml" \+ "$FIXTURES_DIR/ours-adds-later.yaml" \+ "$FIXTURES_DIR/theirs-adds-between.yaml" \+ 7 \+ "$TEMP_DIR/result.yaml"++ [ "$status" -eq 0 ]++ # Extract changeset IDs in order+ ids=$(grep "id: v58" "$TEMP_DIR/result.yaml" | sed 's/.*id: //' | tr '\n' ' ')++ # Should be sorted chronologically+ [[ "$ids" == *"v58.2025-10-30T09:00:00"* ]]+ [[ "$ids" == *"v58.2025-10-31T10:00:00"* ]]+ [[ "$ids" == *"v58.2025-11-01T12:00:00"* ]]++ # Verify order by checking the "between" changeset comes before "later"+ between_line=$(grep -n "v58.2025-10-31T10:00:00" "$TEMP_DIR/result.yaml" | cut -d: -f1)+ later_line=$(grep -n "v58.2025-11-01T12:00:00" "$TEMP_DIR/result.yaml" | cut -d: -f1)+ [ "$between_line" -lt "$later_line" ]+}diff --git a/mage/src/mage/merge_yaml_migrations.clj b/mage/src/mage/merge_yaml_migrations.cljnew file mode 100644index 00000000000..436d7cbcac5--- /dev/null+++ b/mage/src/mage/merge_yaml_migrations.clj@@ -0,0 +1,206 @@+(ns mage.merge-yaml-migrations+ (:require+ [clj-yaml.core :as yaml]+ [clojure.string :as str]+ [mage.util :as u]+ [medley.core :as m]))++(set! *warn-on-reflection* true)++;; Parse YAML file to Clojure data structure+(defn- parse-yaml [file-path]+ (try+ (yaml/parse-string (slurp file-path))+ (catch Exception e+ (binding [*out* *err*]+ (println "Error parsing YAML file:" file-path)+ (println (.getMessage e)))+ (u/exit 2))))++(def ^:private FOOTER-PREFIX+ "# >>>>>>>>>> DO NOT ADD NEW MIGRATIONS BELOW THIS LINE! ADD THEM ABOVE <<<<<<<<<<")++(defn- extract-footer [content]+ (let [idx (some-> content (str/index-of FOOTER-PREFIX))]+ (if (and idx (pos? idx))+ (subs content idx)+ "")))++;; Extract raw text for each changeset from the file+(defn- extract-changeset-texts [file-content parsed-yaml]+ (let [lines (str/split-lines file-content)+ changelog (get parsed-yaml :databaseChangeLog)+ changesets (filterv #(contains? % :changeSet) changelog)]+ ;; For each changeset, find its text in the original file+ (->> (for [cs changesets]+ (let [cs-id (get-in cs [:changeSet :id])+ ;; Find the line with "id: <cs-id>"+ id-line-idx (first (keep-indexed+ (fn [idx line]+ (when (str/includes? line (str "id: " cs-id))+ idx))+ lines))+ ;; Find the changeset start (line before with " - changeSet:")+ cs-start-idx (when id-line-idx+ (loop [idx (dec id-line-idx)]+ (cond+ (< idx 0) 0+ (str/starts-with? (get lines idx) " - changeSet:") idx+ :else (recur (dec idx)))))+ ;; Find the changeset end (next line with " - changeSet:" or " - " at root level or "#" or end)+ cs-end-idx (when cs-start-idx+ (loop [idx (inc cs-start-idx)]+ (cond+ (>= idx (count lines)) (dec (count lines))+ (let [line (get lines idx)]+ (or (str/starts-with? line " - changeSet:")+ (and (str/starts-with? line " - ")+ (not (re-find #"^ - (changeSet|sql|addColumn|createTable|dropTable|addForeignKeyConstraint|dropForeignKeyConstraint|createIndex|dropIndex|addUniqueConstraint|dropUniqueConstraint|addPrimaryKey|dropPrimaryKey):" line)))+ (str/starts-with? line "# ")))+ (dec idx)+ :else (recur (inc idx)))))]+ (when (and cs-start-idx cs-end-idx)+ [cs-id (str/join "\n" (subvec (vec lines) cs-start-idx (inc cs-end-idx)))])))+ (into {}))))++(defn- extract-changesets [parsed-yaml]+ (->> (:databaseChangeLog parsed-yaml)+ (filterv :changeSet)))++(defn- cs-id [cs]+ (-> cs :changeSet :id))++(defn- merge-changesets [base-data ours-data theirs-data]+ (let [base-cs (m/index-by cs-id (extract-changesets base-data))+ ours-cs (m/index-by cs-id (extract-changesets ours-data))+ theirs-cs (m/index-by cs-id (extract-changesets theirs-data))+ all-ids (into #{}+ (comp (map keys) cat)+ [base-cs ours-cs theirs-cs])]+ (reduce+ (fn [acc id]+ (let [base (get base-cs id)+ ours (get ours-cs id)+ theirs (get theirs-cs id)]+ (cond+ (and ours (not theirs) (not base)) ; Added by us+ (conj acc {:id id :source :ours})+ (and theirs (not ours) (not base)) ; Added by them+ (conj acc {:id id :source :theirs})+ (and base ours (not theirs)) ; they have deleted it+ acc+ (and base theirs (not ours)) ; we have deleted it+ acc+ (and base (not ours) (not theirs)) ; only in base, so both deleted+ acc+ (and base ours theirs) ; check for modifications+ (cond+ (= ours theirs) ; identical+ (conj acc {:id id :source :ours})+ (and (= theirs base) (not= ours base)) ; our mod+ (conj acc {:id id :source :ours})+ (and (= ours base) (not= theirs base)) ; their mod+ (conj acc {:id id :source :theirs})+ :else ; CONFLICT+ (conj acc {:id id+ :source :conflict+ :ours ours+ :theirs theirs}))+ :else ; was I diligent enough?+ (do+ (binding [*out* *err*]+ (println "Unexpected case for changeset:" id))+ acc))))+ []+ all-ids)))++(defn- format-conflict-changeset [id ours theirs]+ (format+ (str "<<<<<<< MERGE CONFLICT for changeset: %s, our version:\n"+ "%s\n"+ "======= Their version:\n"+ "%s\n"+ ">>>>>>>\n")+ id ours theirs))++(defn- merge-files [base ours theirs {:keys [_marker-size]}]+ (let [ours-text (slurp ours)+ theirs-text (slurp theirs)+ ;; Parse yaml+ base-data (parse-yaml base)+ ours-data (parse-yaml ours)+ theirs-data (parse-yaml theirs)+ ;; Extract raw text for each changeset+ ours-cs-texts (extract-changeset-texts ours-text ours-data)+ theirs-cs-texts (extract-changeset-texts theirs-text theirs-data)+ ;; Extract footer from ours file+ footer (extract-footer ours-text)+ ;; Perform merge+ merged (merge-changesets base-data ours-data theirs-data)+ ;; Sort merged changesets by ID (chronological order)+ sorted-merged (sort-by :id merged)+ ;; Build result by concatenating raw text blocks+ header "databaseChangeLog:\n - objectQuotingStrategy: QUOTE_ALL_OBJECTS\n"+ changeset-texts+ (for [cs sorted-merged]+ (let [text (case (:source cs)+ :ours (get ours-cs-texts (:id cs))+ :theirs (get theirs-cs-texts (:id cs))+ :conflict (format-conflict-changeset ;; TODO: use marker-size here+ (:id cs)+ (get ours-cs-texts (:id cs))+ (get theirs-cs-texts (:id cs))))]+ ;; Remove trailing blank lines from each changeset+ (str/replace text #"\n+$" "")))]+ {:result (str header+ "\n"+ (str/join "\n\n" (filter some? changeset-texts))+ "\n"+ (when (seq footer) (str "\n" footer))+ (when-not (str/ends-with? footer "\n") "\n"))+ :conflicts (vec (keep #(when (= (:source %) :conflict) (:id %)) sorted-merged))+ :cnt (count sorted-merged)}))++;;++;;+;; Usage (called by git):+;; merge-yaml-migrations %O %A %B %L %P+;; %O = ancestor's version (base)+;; %A = current version (ours)+;; %B = other branch's version (theirs)+;; %L = conflict marker size+;; %P = file path+;;+;; Exit codes:+;; 0 = clean merge+;; 1 = conflicts detected (conflict markers added to file)+;; >1 = merge failed++(defn -main [{:keys [arguments]}]+ (when (< (count arguments) 3)+ (binding [*out* *err*]+ (println "Usage: merge-yaml-migrations <base> <ours> <theirs> [conflict-marker-size] [file-path]")+ (println "Will modify <ours>!"))+ (u/exit 2))++ (let [[base ours theirs marker-size target] arguments+ {:keys [result conflicts cnt]} (merge-files+ base+ ours+ theirs+ {:marker-size marker-size})]+ (spit (or target *out*) result)++ ;; Exit with appropriate code+ (when (seq conflicts)+ (binding [*out* *err*]+ (println "Merge conflicts detected in changesets:" (str/join ", " conflicts))+ (println "Conflict markers added to file. Please resolve manually."))+ (u/exit 1))++ (println "Clean merge of" cnt "changesets")+ (u/exit 0)))++(when (= *file* (System/getProperty "babashka.file"))+ (-main *command-line-args*))diff --git a/mage/src/mage/token_scan.clj b/mage/src/mage/token_scan.cljindex af506c18594..e24e09860e2 100644--- a/mage/src/mage/token_scan.clj+++ b/mage/src/mage/token_scan.clj@@ -89,29 +89,8 @@:matches []:error (str e)})))-(defn- git-ignored-files- "Returns a set of files that are ignored by git."- [files]- (println (c/yellow "Checking git ignore status for " (c/white (count files)) " files..."))- (let [{:keys [exit]- :as proc} (p/sh {:out :string- :err :string- :continue true- :dir u/project-root-directory- :in (str/join "\n" files)}- "git" "check-ignore" "--stdin")- output (:out proc)]- (when (= 128 exit)- (throw (ex-info "git check-ignore has failed with an exceptional status code: maybe git is not initialized in this directory or no gitignore file found."- {:babashka/exit exit :git-error (:err proc)})))- (->> output- str/split-lines- (remove str/blank?)- (map str/trim)- set)))-(defn remove-ignored [files]- (let [ignored (git-ignored-files files) ;; Returns a set of git-ignored files+ (let [ignored (u/git-ignored-files files) ;; Returns a set of git-ignored files_ (u/debug "Ignored files:" (count ignored))files' (remove ignored files)](u/debug "Files to scan:" (count files'))diff --git a/mage/src/mage/util.clj b/mage/src/mage/util.cljindex 3d283d0d392..088c808fbb2 100644--- a/mage/src/mage/util.clj+++ b/mage/src/mage/util.clj@@ -1,6 +1,7 @@(ns mage.util(:require[babashka.fs :as fs]+ [babashka.process :as p][babashka.tasks :refer [shell]][clojure.edn :as edn][clojure.java.io :as io]@@ -261,3 +262,24 @@([message exit-code](println message)(throw (ex-info "" {:mage/quiet true :babashka/exit exit-code}))))++(defn git-ignored-files+ "Returns a set of files that are ignored by git."+ [files]+ (println (c/yellow "Checking git ignore status for " (c/white (count files)) " files..."))+ (let [{:keys [exit]+ :as proc} (p/sh {:out :string+ :err :string+ :continue true+ :dir project-root-directory+ :in (str/join "\n" files)}+ "git" "check-ignore" "--stdin")+ output (:out proc)]+ (when (= 128 exit)+ (throw (ex-info "git check-ignore has failed with an exceptional status code: maybe git is not initialized in this directory or no gitignore file found."+ {:babashka/exit exit :git-error (:err proc)})))+ (->> output+ str/split-lines+ (remove str/blank?)+ (map str/trim)+ set)))diff --git a/mage/test/mage/core_test.clj b/mage/test/mage/core_test.cljindex c5cd923bed5..1dfb80f48ed 100644--- a/mage/test/mage/core_test.clj+++ b/mage/test/mage/core_test.clj@@ -3,11 +3,17 @@[babashka.tasks :as bt][clojure.string :as str][clojure.test :refer [deftest is testing]]+ [mage.merge-yaml-migrations-test :as merge-yaml-migrations-test][mage.alias-test][mage.token-scan-test][mage.util :as u][mage.util-test]))+(comment+ mage.util-test/keep-me+ merge-yaml-migrations-test/keep-me+ token-scan-test/keep-me)+(set! *warn-on-reflection* true)(deftest bin-mage-has-help-testdiff --git a/mage/test/mage/merge_yaml_migrations_test.clj b/mage/test/mage/merge_yaml_migrations_test.cljnew file mode 100644index 00000000000..1bee276306e--- /dev/null+++ b/mage/test/mage/merge_yaml_migrations_test.clj@@ -0,0 +1,100 @@+(ns mage.merge-yaml-migrations-test+ (:require+ [clojure.string :as str]+ [clojure.test :refer [deftest is testing]]+ [mage.merge-yaml-migrations :as merge-yaml]))++(def fixtures-dir "mage/resources/merge-yaml-migrations-test/fixtures")++(defn- fixture-path [filename]+ (str fixtures-dir "/" filename))++(defn- merge-files-test+ "Test helper that calls the private merge-files function"+ [base ours theirs]+ (#'merge-yaml/merge-files base ours theirs {:marker-size 7}))++(deftest clean-merge-both-add-different-changesets+ (testing "clean merge: both branches add different changesets"+ (let [{:keys [result conflicts cnt]} (merge-files-test+ (fixture-path "base-simple.yaml")+ (fixture-path "ours-adds-one.yaml")+ (fixture-path "theirs-adds-one.yaml"))]+ (is (empty? conflicts) "Should have no conflicts")+ (is (= 3 cnt) "Should have 3 changesets total")+ (is (= 3 (count (re-seq #"(?m)^ - changeSet:" result)))+ "Result should contain 3 changesets"))))++(deftest clean-merge-only-ours-adds+ (testing "clean merge: only our branch adds a changeset"+ (let [{:keys [result conflicts cnt]} (merge-files-test+ (fixture-path "base-simple.yaml")+ (fixture-path "ours-adds-one.yaml")+ (fixture-path "base-simple.yaml"))]+ (is (empty? conflicts) "Should have no conflicts")+ (is (= 2 cnt) "Should have 2 changesets")+ (is (= 2 (count (re-seq #"(?m)^ - changeSet:" result)))+ "Result should contain 2 changesets"))))++(deftest conflict-both-modify-same-changeset+ (testing "conflict: both branches modify the same changeset differently"+ (let [{:keys [result conflicts cnt]} (merge-files-test+ (fixture-path "base-conflict.yaml")+ (fixture-path "ours-modifies.yaml")+ (fixture-path "theirs-modifies.yaml"))]+ (is (= 1 cnt))+ (is (seq conflicts) "Should have conflicts")+ (is (str/includes? result "MERGE CONFLICT")+ "Result should contain conflict markers"))))++(deftest deletion-changeset-deleted+ (testing "deletion: changeset deletion is respected"+ (let [{:keys [result conflicts cnt]} (merge-files-test+ (fixture-path "base-two-changesets.yaml")+ (fixture-path "ours-deletes-second.yaml")+ (fixture-path "base-two-changesets.yaml"))]+ (is (empty? conflicts) "Should have no conflicts")+ (is (= 1 cnt) "Should have only 1 changeset (deletion respected)")+ (is (= 1 (count (re-seq #"(?m)^ - changeSet:" result)))+ "Result should contain 1 changeset"))))++(deftest preserves-footer-warning+ (testing "preserves footer warning"+ (let [{:keys [result conflicts]} (merge-files-test+ (fixture-path "base-with-footer.yaml")+ (fixture-path "ours-adds-with-footer.yaml")+ (fixture-path "base-with-footer.yaml"))]+ (is (empty? conflicts) "Should have no conflicts")+ (is (str/includes? result "# >>>>>>>>>> DO NOT ADD NEW MIGRATIONS BELOW THIS LINE!")+ "Footer warning should be preserved")+ (is (str/includes? result "# ADVICE:")+ "Footer advice should be preserved"))))++(deftest preserves-formatting+ (testing "preserves formatting and blank lines"+ (let [{:keys [result conflicts]} (merge-files-test+ (fixture-path "base-simple.yaml")+ (fixture-path "ours-adds-one.yaml")+ (fixture-path "theirs-adds-one.yaml"))]+ (is (empty? conflicts) "Should have no conflicts")+ (is (str/includes? result " - objectQuotingStrategy: QUOTE_ALL_OBJECTS")+ "Should preserve quoting strategy line")+ (is (not (str/includes? result "\n\n\n"))+ "Should not have consecutive blank lines"))))++(deftest sorts-changesets-chronologically+ (testing "sorts changesets by ID chronologically"+ (let [{:keys [result conflicts]} (merge-files-test+ (fixture-path "base-sort.yaml")+ (fixture-path "ours-adds-later.yaml")+ (fixture-path "theirs-adds-between.yaml"))]+ (is (empty? conflicts) "Should have no conflicts")++ ;; Extract changeset IDs in order+ (let [ids (re-seq #"id: v58\.2025-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}" result)+ ids-str (map #(subs % 4) ids)] ;; Remove "id: " prefix+ (is (= ["v58.2025-10-30T09:00:00"+ "v58.2025-10-31T10:00:00"+ "v58.2025-11-01T12:00:00"]+ ids-str)+ "Changesets should be sorted chronologically")))))diff --git a/mage/test/mage/util_test.clj b/mage/test/mage/util_test.cljindex 6b7bdeb081a..a7bffb68dad 100644--- a/mage/test/mage/util_test.clj+++ b/mage/test/mage/util_test.clj@@ -17,7 +17,10 @@(str/replace (re-pattern (str new-content "$")) ""))))(deftest updated-files-returns-all-kinds-of-files- (let [extension->path (-> (group-by fs/extension (fs/glob u/project-root-directory "**.*"))+ (let [all-files (map str (fs/glob u/project-root-directory "**.*"))+ git-ignored-files (u/git-ignored-files all-files)+ ok-files (remove git-ignored-files all-files)+ extension->path (-> (group-by fs/extension ok-files)(update-vals (comp str first))(select-keys ["clj" "cljc" "cljs" "edn" "js" "ts" "jsx" "json" "css" "html" "md"]))](doseq [[_ path] extension->path]diff --git a/package.json b/package.jsonindex 99406ac8b97..62392040945 100644--- a/package.json+++ b/package.json@@ -518,7 +518,8 @@"prettier --write"],"**/*": [- "./bin/mage -token-scan"+ "./bin/mage -token-scan",+ "./bin/mage -install-merge-drivers"]},"browserslist": [