The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
(ns metabase-enterprise.remote-sync.source.git-test
(:require
[clojure.java.io :as io]
[clojure.string :as str]
[clojure.test :refer :all]
[metabase-enterprise.remote-sync.source.git :as git]
[metabase-enterprise.remote-sync.source.protocol :as source.p]
[metabase-enterprise.serialization.v2.ingest :as ingest]
[metabase.test :as mt]
[metabase.util :as u])
(:import (java.io File)
(java.nio.file Files Paths)
(java.nio.file.attribute FileAttribute)
(org.apache.commons.io FileUtils)
(org.eclipse.jgit.api Git TransportCommand)
(org.eclipse.jgit.dircache DirCacheEditor DirCacheEditor$PathEdit DirCacheEntry)
(org.eclipse.jgit.lib AnyObjectId FileMode PersonIdent)
(org.eclipse.jgit.transport UsernamePasswordCredentialsProvider)))
(set! *warn-on-reflection* true)
(defn- write-files!
"Test seeding helper: wholesale-write `files` ({:path :content}) to `snapshot` via the commit builder
(clear the managed dirs, stage every file, push). Returns the new version."
[snapshot message files]
(let [c (source.p/open-commit snapshot)]
(source.p/replace-all! c)
(doseq [f files] (source.p/stage-upsert! c f))
(source.p/finish-commit! c message)))
(defn- apply-changes!
"Test helper: incremental patch via the commit builder — stage `upserts`, remove `delete-paths`, push.
Returns the new version."
[snapshot message upserts delete-paths]
(let [c (source.p/open-commit snapshot)]
(doseq [u upserts] (source.p/stage-upsert! c u))
(doseq [p delete-paths] (source.p/stage-delete! c p))
(source.p/finish-commit! c message)))
(defn- git-working-branch
"The working branch in the given repo"
[{:keys [^Git git]}]
(-> (.getRepository git)
(.getBranch)))
(defn- git-working-checkout!
"Checks out the given branch in the working directory"
[{:keys [^Git git]} ^String branch ^Boolean create]
(-> (.checkout git)
(.setName branch)
(.setCreateBranch create)
(.setForced true)
(.call)))
(defn- git-working-commit!
"Commits the current working directory"
[{:keys [^Git git]} message]
(-> (.commit git)
(.setMessage message)
(.setAuthor (PersonIdent. "Test Setup" "[email protected]"))
(.setCommitter (PersonIdent. "Test Setup" "[email protected]"))
(.call)))
(defn- git-working-add!
"Writes the given file in the working path and stages it for commit"
[{:keys [^Git git]} ^String path ^String content]
(let [repo (.getRepository git)
work-tree (.getWorkTree repo)
full-path (io/file work-tree path)]
(io/make-parents full-path)
(spit full-path content)
(-> (.add git)
(.addFilepattern path)
(.call))))
(defn- git-working-link!
"Writes a symlink in the working path and stages it, so a test can cover the mode-120000 entries a real
repo can hold. Git stores the link's *target text* as the blob, and never follows it."
[{:keys [^Git git]} ^String path ^String target]
(let [full-path (io/file (.getWorkTree (.getRepository git)) path)]
(io/make-parents full-path)
(Files/createSymbolicLink (.toPath full-path)
(Paths/get target (into-array String []))
(into-array FileAttribute []))
(-> (.add git)
(.addFilepattern path)
(.call))))
(defn- git-working-gitlink!
"Stages a submodule entry (mode 160000) pointing at `commit-id`. A gitlink has no working-tree file to
add, so this edits the index directly, as `git update-index --cacheinfo 160000,<sha>,<path>` does."
[{:keys [^Git git]} ^String path ^AnyObjectId commit-id]
(let [^DirCacheEditor editor (.editor (.lockDirCache (.getRepository git)))]
(.add editor (proxy [DirCacheEditor$PathEdit] [path]
(apply [^DirCacheEntry entry]
(.setFileMode entry FileMode/GITLINK)
(.setObjectId entry commit-id))))
(.commit editor)))
(defn- git-working-create-branch!
"Creates a branch with an initial commit and file using the working directory"
[source ^String branch]
(let [initial-branch (git-working-branch source)]
(git-working-checkout! source branch true)
(git-working-add! source (str "file-in-" branch ".txt") (str "File in " branch))
(git-working-commit! source (str "Init branch " branch))
(git-working-checkout! source initial-branch false)))
(defn- init-remote!
"Initializes a 'remote' git repo in the given directory"
[^String dir & {:keys [files branches]}]
(let [git (-> (Git/init)
(.setDirectory (File. dir))
(.setInitialBranch "master")
(.call))
remote {:git git}]
(doseq [[path content] files]
(git-working-add! remote path content))
(git-working-commit! remote "Initial commit")
(doseq [branch branches]
(git-working-create-branch! remote branch))
remote))
(defn- ->source!
"Creates a (local) 'remote' repo and initializes a git source that uses it"
[branch {:keys [^Git git] :as _remote-repo}]
(let [remote-url (-> (.getRepository git)
(.getDirectory)
(.toURI)
(.toURL)
(.toExternalForm))
local-repo (#'git/get-jgit (#'git/repo-path {:remote-url remote-url}) {:remote-url remote-url})]
(git/->GitSource local-repo remote-url branch nil ingest/legal-top-level-paths)))
(defn- init-source!
[branch dir & config]
(FileUtils/deleteDirectory (io/file dir))
(let [remote-repo (apply init-remote! dir config)]
[(->source! branch remote-repo) remote-repo]))
(defn- command-timeout
"Reads the protected `timeout` field (in seconds) that JGit applies to a TransportCommand's
network operations. 0 means no timeout (JGit's default), i.e. the operation can hang forever."
[^TransportCommand cmd]
(let [f (.getDeclaredField TransportCommand "timeout")]
(.setAccessible f true)
(.getInt f cmd)))
(deftest qualify-branch-test
(is (= "refs/heads/main" (#'git/qualify-branch "main")))
(is (= "refs/heads/main" (#'git/qualify-branch "refs/heads/main"))))
(deftest call-remote-command-applies-network-timeout-test
(testing "Remote git operations get a positive network timeout so a stalled connection can't hang
the sync thread forever (GHY-3727: pull/push gets stuck at progress 0 and 0.3)"
(mt/with-temp-dir [remote-dir nil]
(let [[source _remote] (init-source! "master" remote-dir :files {"master.txt" "File in master"})
^Git git (:git source)
cmd (.lsRemote git)]
(#'git/call-remote-command cmd source)
(is (pos? (command-timeout cmd))
"TransportCommand should have a positive (non-zero) timeout configured before .call")))))
(deftest call-remote-command-respects-timeout-setting-test
(testing "The network timeout applied to remote git operations is driven by remote-sync-git-timeout-seconds"
(mt/with-temp-dir [remote-dir nil]
(let [[source _remote] (init-source! "master" remote-dir :files {"master.txt" "File in master"})
^Git git (:git source)
cmd (.lsRemote git)]
(mt/with-temporary-setting-values [remote-sync-git-timeout-seconds 17]
(#'git/call-remote-command cmd source)
(is (= 17 (command-timeout cmd))))))))
(deftest log
(mt/with-temp-dir [remote-dir nil]
(let [[master remote] (init-source! "master" remote-dir :branches ["branch-1" "branch-2"])
branch-1 (->source! "branch-1" remote)
invalid (->source! "invalid" remote)]
(is (= ["Initial commit"] (map :message (git/log master))))
(is (= ["Init branch branch-1" "Initial commit"] (map :message (git/log branch-1))))
(is (nil? (git/log invalid))))))
;
(deftest branches
(mt/with-temp-dir [remote-dir nil]
(let [[source _remote] (init-source! "master" remote-dir :branches ["branch-1" "branch-2"])]
;; add extra branch to remote to check it is picked up
(git-working-create-branch! _remote "branch-3")
(is (= ["branch-1" "branch-2" "branch-3" "master"] (source.p/branches source))))))
(deftest snapshot
(mt/with-temp-dir [remote-dir nil]
(let [[master _remote] (init-source! "master" remote-dir
:files {"master.txt" "File in master"
"subdir/path.txt" "File in subdir"}
:branches ["branch-1" "branch-2"])
master-snapshot (source.p/snapshot master)]
(is (= (git/commit-sha master "master") (:version master-snapshot))))))
(deftest commit-sha-missing-object-is-nil-test
(testing "GHY-3917: a full SHA whose object isn't in the clone resolves to nil, not a phantom id"
(mt/with-temp-dir [remote-dir nil]
(let [[master _remote] (init-source! "master" remote-dir
:files {"master.txt" "File in master"})
;; JGit parses any complete 40-hex string into an ObjectId without a presence check; the
;; existence guard is what turns a base commit orphaned by an upstream force-push/rebase into
;; nil here instead of a later MissingObjectException when its tree is read.
absent-sha "0000000000000000000000000000000000000000"]
(is (some? (git/commit-sha master "master"))
"a real ref still resolves")
(is (nil? (git/commit-sha master absent-sha))
"a syntactically valid but absent full SHA resolves to nil")
(is (nil? (source.p/snapshot-at master absent-sha))
"snapshot-at returns nil for the orphaned base, so callers take the history-rewritten path")))))
(deftest list-files
(mt/with-temp-dir [remote-dir nil]
(let [[master remote] (init-source! "master" remote-dir
:files {"master.txt" "File in master"
"subdir/path.txt" "File in subdir"}
:branches ["branch-1" "branch-2"])
master-snap (source.p/snapshot master)
branch-1 (source.p/snapshot (->source! "branch-1" remote))
branch-2 (source.p/snapshot (->source! "branch-2" remote))]
(is (= ["master.txt" "subdir/path.txt"] (source.p/list-files master-snap)))
(is (= ["file-in-branch-1.txt" "master.txt" "subdir/path.txt"] (source.p/list-files branch-1)))
(is (= ["file-in-branch-2.txt" "master.txt" "subdir/path.txt"] (source.p/list-files branch-2))))))
(deftest list-dir
(mt/with-temp-dir [remote-dir nil]
(let [[master _remote] (init-source! "master" remote-dir
:files {"root.txt" "at the root"
"dir/readme.md" "a file, not a directory"
"dir/alpha/one.txt" "ONE"
"dir/alpha/two.txt" "TWO"
"dir/alpha/deep/leaf.txt" "LEAF"
"dir/beta/three.txt" "THREE"
;; \- sorts before \/, so git's own tree order here is
;; ("d/a-b" "d/a") — the case the sort normalizes
"d/a-b" "a sibling"
"d/a/x" "inside a directory"
"zzz/other.txt" "elsewhere"})
snap (source.p/snapshot master)]
(testing "immediate children only — no descendants, no siblings of the directory itself"
(is (= ["dir/alpha" "dir/beta" "dir/readme.md"] (source.p/list-dir snap "dir"))))
(testing "children are full repo-root relative paths, so they feed straight back into read-file"
(let [child (some #{"dir/alpha/one.txt"} (source.p/list-dir snap "dir/alpha"))]
(is (= "ONE" (source.p/read-file snap child)))))
(testing "files and directories are both listed, sorted — `deep` is a directory"
(is (= ["dir/alpha/deep" "dir/alpha/one.txt" "dir/alpha/two.txt"]
(source.p/list-dir snap "dir/alpha"))))
(testing "the repo root"
(is (= ["d" "dir" "root.txt" "zzz"] (source.p/list-dir snap ""))))
(testing "ordering is plain lexicographic, not git's own tree order"
;; git compares a directory as if it ended in "/", so its tree order here is ("d/a-b" "d/a").
;; We normalize to lexicographic — the one order a flat path list can produce too, so every
;; snapshot implementation agrees.
(is (= ["d/a" "d/a-b"] (source.p/list-dir snap "d"))))
(testing "nesting: each call steps down exactly one level, and paths stay rooted at the repo"
(is (= ["dir/alpha/deep/leaf.txt"] (source.p/list-dir snap "dir/alpha/deep"))))
(testing "a path that is a file, or absent, has no children rather than throwing"
(is (= [] (source.p/list-dir snap "dir/readme.md")))
(is (= [] (source.p/list-dir snap "nope")))
(is (= [] (source.p/list-dir snap "dir/alpha/nope")))))))
(deftest list-dir-dotfiles-and-symlinks
(testing "the entry kinds a real repo can hold, beyond plain files and directories"
(mt/with-temp-dir [remote-dir nil]
(let [remote (init-remote! remote-dir
:files {"dir/plain.txt" "a file"
"dir/.dotfile" "a dotfile"
"dir/.hidden/file.txt" "inside a dot-directory"
"outside/target.txt" "outside dir"})
_ (git-working-link! remote "dir/linkdir" "../outside")
_ (git-working-link! remote "dir/linkfile" "plain.txt")
head (git-working-commit! remote "add symlinks")
_ (git-working-gitlink! remote "dir/sub" head)
_ (git-working-commit! remote "add submodule")
snap (source.p/snapshot (->source! "master" remote))]
(testing "git has no notion of a hidden file, so neither do we — dotted names are listed as-is"
(is (= ["dir/.dotfile" "dir/.hidden" "dir/linkdir" "dir/linkfile" "dir/plain.txt" "dir/sub"]
(source.p/list-dir snap "dir")))
(is (= ["dir/.hidden/file.txt"] (source.p/list-dir snap "dir/.hidden"))
"a dot-directory is an ordinary tree we descend into"))
(testing "only a tree has children: everything else lists nothing, whatever it points at"
(is (= [] (source.p/list-dir snap "dir/linkdir")) "a symlink to a directory")
(is (= [] (source.p/list-dir snap "dir/linkfile")) "a symlink to a file")
(is (= [] (source.p/list-dir snap "dir/sub")) "a submodule (gitlink) — its tree isn't in this repo")
(is (= [] (source.p/list-dir snap "dir/plain.txt")) "a plain file"))
(testing "reading a symlink yields its target text, never the target's content: git does not follow
links, so a link can neither escape the repo nor pull in a file from outside the directory"
(is (= "../outside" (source.p/read-file snap "dir/linkdir")))
(is (= "plain.txt" (source.p/read-file snap "dir/linkfile"))))))))
(deftest read-file
(mt/with-temp-dir [remote-dir nil]
(let [[master _remote] (init-source! "master" remote-dir
:files {"master.txt" "File in master"
"subdir/path.txt" "File in subdir"}
:branches ["branch-1" "branch-2"])
master-snap (source.p/snapshot master)
branch-1 (source.p/snapshot (->source! "branch-1" _remote))]
(testing "Reading master"
(is (= "File in master" (source.p/read-file master-snap "master.txt")))
(is (= "File in subdir" (source.p/read-file master-snap "subdir/path.txt")))
(is (nil? (source.p/read-file master-snap "file-in-branch-1.txt"))))
(testing "Reading branch-1"
(is (= "File in master" (source.p/read-file branch-1 "master.txt")))
(is (= "File in branch-1" (source.p/read-file branch-1 "file-in-branch-1.txt")))
(is (nil? (source.p/read-file branch-1 "file-in-branch-2.txt")))))))
(deftest write-files
(let [subdir-path (str "collections/" "r" (subs (u/generate-nano-id "a") 1) "_subdir/")
thirddir-path (str "collections/" "s" (subs (u/generate-nano-id "c") 1) "_thirddir/")]
(mt/with-temp-dir [remote-dir nil]
(let [[master remote] (init-source! "master" remote-dir
:files {"master.txt" "File in master"
"master2.txt" "File 2 in master"
(str subdir-path "path.txt") "File in subdir"
(str subdir-path "path2.txt") "File 2 in subdir"
(str thirddir-path "path.txt") "File in third dir"
(str thirddir-path "path2.txt") "File 2 in third dir"}
:branches ["branch-1" "branch-2"])]
(testing "All files in managed dirs not in write set are removed; root files outside managed dirs are preserved"
(write-files! (source.p/snapshot master) "Update 1" [{:path "master.txt" :content "Updated master content"}
{:path (str subdir-path "path.txt") :content "Updated subdir content"}
{:path (str subdir-path "path3.txt") :content "Updated subdir content 3"}
{:path (str thirddir-path "path.txt") :content "Updated third dir content"}
{:path (str thirddir-path "path3.txt") :content "Updated third dir content 3"}])
(is (= ["Update 1" "Initial commit"] (map :message (git/log master))))
(let [master-snap (source.p/snapshot master)]
;; otherdir files are removed because collections/ is a managed dir and those files weren't in the write set
(is (= [(str subdir-path "path.txt")
(str subdir-path "path3.txt")
(str thirddir-path "path.txt")
(str thirddir-path "path3.txt")
"master.txt"
"master2.txt"]
(source.p/list-files master-snap)))
(is (= "Updated master content" (source.p/read-file master-snap "master.txt")))
(is (= "File 2 in master" (source.p/read-file master-snap "master2.txt")))
(is (= "Updated subdir content" (source.p/read-file master-snap (str subdir-path "path.txt"))))
(is (= "Updated subdir content 3" (source.p/read-file master-snap (str subdir-path "path3.txt")))))
(testing "Check remote repo directly"
(is (= "Updated master content" (git/read-file (assoc remote :version "master") "master.txt")))
(is (= [(str subdir-path "path.txt")
(str subdir-path "path3.txt")
(str thirddir-path "path.txt")
(str thirddir-path "path3.txt")
"master.txt"
"master2.txt"]
(git/list-files (assoc remote :version "master"))))
(is (= ["Update 1" "Initial commit"] (map :message (git/log (assoc remote :branch "master")))))))
(testing "Writing only to collections/ removes all other collection files"
(write-files! (source.p/snapshot master) "Update 2" [{:path (str thirddir-path "path.txt") :content "Only third dir content"}])
(is (= [(str thirddir-path "path.txt")
"master.txt"
"master2.txt"]
(git/list-files (assoc remote :version "master")))))))))
(deftest apply-changes
(let [subdir (str "collections/" "r" (subs (u/generate-nano-id "a") 1) "_subdir/")]
(mt/with-temp-dir [remote-dir nil]
(let [[master remote] (init-source! "master" remote-dir
:files {"master.txt" "root file"
(str subdir "keep.yaml") "keep me"
(str subdir "edit.yaml") "old content"
(str subdir "remove.yaml") "delete me"})]
(testing "apply-changes! overwrites/adds upserts, removes delete-paths, and PRESERVES every other file"
(apply-changes! (source.p/snapshot master) "Incremental"
[{:path (str subdir "edit.yaml") :content "new content"}
{:path (str subdir "new.yaml") :content "brand new"}]
[(str subdir "remove.yaml")])
(is (= ["Incremental" "Initial commit"] (map :message (git/log master))))
(let [snap (source.p/snapshot master)]
(is (= [(str subdir "edit.yaml")
(str subdir "keep.yaml")
(str subdir "new.yaml")
"master.txt"]
(source.p/list-files snap))
"edit overwritten + new added, remove deleted; keep.yaml (managed, untouched) and master.txt preserved")
(is (= "new content" (source.p/read-file snap (str subdir "edit.yaml"))))
(is (= "brand new" (source.p/read-file snap (str subdir "new.yaml"))))
(is (= "keep me" (source.p/read-file snap (str subdir "keep.yaml")))
"a managed-dir file not in the write set is preserved (unlike write-files!)")
(is (= "root file" (source.p/read-file snap "master.txt")))
(is (nil? (source.p/read-file snap (str subdir "remove.yaml")))))
(testing "the commit was pushed to the remote"
(is (= ["Incremental" "Initial commit"]
(map :message (git/log (assoc remote :branch "master")))))))))))
(deftest empty-commit?-detects-no-op-tree-test
(testing "empty-commit? is true exactly when the staged tree matches the parent commit's tree"
(mt/with-temp-dir [remote-dir nil]
(let [[master _remote] (init-source! "master" remote-dir
:files {"collections/a.yaml" "content A"
"collections/b.yaml" "content B"})]
(testing "re-staging identical content is empty"
(let [c (source.p/open-commit (source.p/snapshot master))]
(source.p/stage-upsert! c {:path "collections/a.yaml" :content "content A"})
(is (true? (source.p/empty-commit? c)))
(source.p/abort-commit! c)))
(testing "staging a real content change is not empty"
(let [c (source.p/open-commit (source.p/snapshot master))]
(source.p/stage-upsert! c {:path "collections/a.yaml" :content "changed A"})
(is (false? (source.p/empty-commit? c)))
(source.p/abort-commit! c)))
(testing "adding a new file is not empty"
(let [c (source.p/open-commit (source.p/snapshot master))]
(source.p/stage-upsert! c {:path "collections/c.yaml" :content "content C"})
(is (false? (source.p/empty-commit? c)))
(source.p/abort-commit! c)))
(testing "deleting a path that isn't in the tree is empty"
(let [c (source.p/open-commit (source.p/snapshot master))]
(source.p/stage-delete! c "collections/does-not-exist.yaml")
(is (true? (source.p/empty-commit? c)))
(source.p/abort-commit! c)))
(testing "none of the aborted no-op checks moved the branch"
(is (= ["Initial commit"] (map :message (git/log master)))))
(testing "empty-commit? then finish-commit! still commits a real change (tree written once, memoized)"
(let [c (source.p/open-commit (source.p/snapshot master))]
(source.p/stage-upsert! c {:path "collections/a.yaml" :content "changed A"})
(is (false? (source.p/empty-commit? c)))
(source.p/finish-commit! c "Edit A"))
(is (= ["Edit A" "Initial commit"] (map :message (git/log master))))
(is (= "changed A" (source.p/read-file (source.p/snapshot master) "collections/a.yaml"))))))))
(deftest apply-changes-preserves-deep-unchanged-subtree-test
(testing "an incremental upsert leaves deeply-nested, unrelated subtrees untouched (carried forward by id)"
(mt/with-temp-dir [remote-dir nil]
(let [[master _remote] (init-source! "master" remote-dir
:files {"collections/a/deep/nested/keep.yaml" "deep keep"
"collections/a/deep/sibling.yaml" "deep sibling"
"collections/b/edit.yaml" "old"
"notes.txt" "root note"})]
(apply-changes! (source.p/snapshot master) "Incremental deep"
[{:path "collections/b/edit.yaml" :content "new"}]
[])
(let [snap (source.p/snapshot master)]
(is (= "new" (source.p/read-file snap "collections/b/edit.yaml")))
(is (= "deep keep" (source.p/read-file snap "collections/a/deep/nested/keep.yaml"))
"a deeply-nested file in an unrelated subtree is carried forward unchanged")
(is (= "deep sibling" (source.p/read-file snap "collections/a/deep/sibling.yaml")))
(is (= "root note" (source.p/read-file snap "notes.txt"))))))))
(deftest write-files-reconciles-every-managed-dir-test
(testing "a full export wipes every managed dir not covered by the write set, keeps non-managed files, and re-adds an upsert inside a managed dir"
(mt/with-temp-dir [remote-dir nil]
(let [[master _remote] (init-source! "master" remote-dir
:files {"collections/old/old.yaml" "old collection"
"transforms/t1/t.yaml" "a transform"
"transforms/t2/t.yaml" "another transform"
"notes.txt" "root note"})]
(write-files! (source.p/snapshot master) "Full"
[{:path "collections/new/new.yaml" :content "new collection"}])
(let [snap (source.p/snapshot master)]
(is (= ["collections/new/new.yaml" "notes.txt"] (source.p/list-files snap))
"transforms/ (managed, no upserts) fully removed; collections/ reconciled to the write set; non-managed notes.txt preserved")
(is (= "new collection" (source.p/read-file snap "collections/new/new.yaml")))
(is (nil? (source.p/read-file snap "collections/old/old.yaml")))
(is (nil? (source.p/read-file snap "transforms/t1/t.yaml")))
(is (= "root note" (source.p/read-file snap "notes.txt"))))))))
(deftest apply-changes-tolerates-missing-delete-path-test
(testing "apply-changes! tolerates a delete-path that doesn't exist — the upsert still applies, no error"
(mt/with-temp-dir [remote-dir nil]
(let [[master _remote] (init-source! "master" remote-dir
:files {"collections/a/keep.yaml" "keep"})]
(apply-changes! (source.p/snapshot master) "Delete missing + add"
[{:path "collections/a/new.yaml" :content "new"}]
["collections/a/gone.yaml"])
(let [snap (source.p/snapshot master)]
(is (= ["collections/a/keep.yaml" "collections/a/new.yaml"] (source.p/list-files snap)))
(is (= "new" (source.p/read-file snap "collections/a/new.yaml")))
(is (= "keep" (source.p/read-file snap "collections/a/keep.yaml"))))))))
(deftest changed-files-test
(testing "changed-files classifies the paths whose blob differs between two commits"
(mt/with-temp-dir [remote-dir nil]
(let [[master _remote] (init-source! "master" remote-dir
:files {"keep.txt" "unchanged"
"edit.txt" "old content"
"remove.txt" "delete me"
"deep/nested/keep.txt" "deep unchanged"})
from-version (:version (source.p/snapshot master))]
(apply-changes! (source.p/snapshot master) "Change set"
[{:path "edit.txt" :content "new content"}
{:path "add.txt" :content "brand new"}]
["remove.txt"])
(let [snap (source.p/snapshot master)]
(testing "added / modified / deleted are reported in their own buckets"
(is (= {:added #{"add.txt"}
:modified #{"edit.txt"}
:deleted #{"remove.txt"}}
(git/changed-files snap from-version))))
(testing "unchanged files — including deep untouched subtrees — are not reported"
(let [{:keys [added modified deleted]} (git/changed-files snap from-version)
touched (reduce into #{} [added modified deleted])]
(is (not (contains? touched "keep.txt")))
(is (not (contains? touched "deep/nested/keep.txt")))))
(testing "comparing a version against itself reports no changes"
(is (= {:added #{} :modified #{} :deleted #{}}
(git/changed-files snap (:version snap)))))
(testing "an unresolvable from-version returns nil, signalling a full import"
(is (nil? (git/changed-files snap "no-such-ref-or-sha")))))))))
(deftest write-special-collections
(let [subdir-path (str "collections/" "r" (subs (u/generate-nano-id "a") 1) "_subdir/")]
(mt/with-temp-dir [remote-dir nil]
(init-source! "master" remote-dir
:files {"master.txt" "File in master"
(str subdir-path "path.txt") "File in subdir"}))))
(deftest concurrent-access
(mt/with-temp-dir [remote-dir nil]
(let [[master remote] (init-source! "master" remote-dir
:files {"master.txt" "File in master"
"subdir/path.txt" "File in subdir"}
:branches ["branch-1" "branch-2"])
new-branch (->source! "new-branch" remote)]
(testing "Initial clone is the same"
(is (= ["Initial commit"] (map :message (git/log master))))
(is (= ["Initial commit"] (map :message (git/log (assoc remote :branch "master")))))
;; Add an extra commit to remote
(git-working-add! remote "additional-file.txt" "Additional file content")
(git-working-commit! remote "Added additional file")
(testing "Source is behind remote"
(is (= ["Initial commit"] (map :message (git/log master))))
(is (= ["Added additional file" "Initial commit"] (map :message (git/log (assoc remote :branch "master"))))))
(testing "After fetch, source is up to date"
(git/fetch! master)
(is (= ["Added additional file" "Initial commit"] (map :message (git/log master)))))
(testing "Writing a file to source and pushing back to remote when there is new content on remote"
;; Make source be behind again
(git-working-add! remote "only-on-remote.txt" "Initially on remote")
(git-working-commit! remote "Only on remote")
(write-files! (source.p/snapshot master) "Added to source" [{:path "initially-source.txt" :content "Initially on source"}])
(testing "Remote has the new commit with just the files committed, but only version is in history"
(is (= ["Added to source" "Only on remote" "Added additional file" "Initial commit"] (map :message (git/log (assoc remote :branch "master")))))
(is (= ["additional-file.txt" "initially-source.txt" "master.txt" "only-on-remote.txt" "subdir/path.txt"] (git/list-files (assoc remote :version "master"))))
(is (= "Initially on source" (git/read-file (assoc remote :version "master") "initially-source.txt"))))
(testing "Source has the same history"
(is (= (map :message (git/log (assoc remote :branch "master"))) (map :message (git/log master))))))
(testing "Writing to a branch local has not seen (but remote has) adds it to the history on remote"
(git-working-checkout! remote "new-branch" true)
(git-working-add! remote "new-branch-file.txt" "Initially on remote")
(git-working-add! remote "new-branch-remote.txt" "Initially on remote")
(git-working-commit! remote "New-branch on remote")
(is (= ["New-branch on remote" "Added to source" "Only on remote" "Added additional file" "Initial commit"] (map :message (git/log (assoc remote :branch "new-branch")))))
(is (nil? (git/log new-branch)))
(write-files! (source.p/snapshot new-branch) "New-branch on source" [{:path "new-branch-source.txt" :content "Initially on source"}
{:path "new-branch-file.txt" :content "Updated on source"}])
(is (= ["New-branch on source" "New-branch on remote" "Added to source" "Only on remote" "Added additional file" "Initial commit"] (map :message (git/log (assoc remote :branch "new-branch"))))))))))
(deftest git-source-using-commit-ref
(mt/with-temp-dir [remote-dir nil]
(let [[master _remote] (init-source! "master" remote-dir
:files {"master.txt" "File in master"
"subdir/path.txt" "File in subdir"})
old-master (source.p/snapshot master)]
(write-files! (source.p/snapshot master) "Update file" [{:path "master.txt" :content "Updated file in master"}
{:path "new-file.txt" :content "New file in master"}])
(is (= "File in master" (source.p/read-file old-master "master.txt")))
(is (= "Updated file in master" (source.p/read-file (source.p/snapshot master) "master.txt")))
(is (= ["master.txt" "subdir/path.txt"] (source.p/list-files old-master))))))
(deftest version-test
(testing "version returns the commit id for the current state"
(mt/with-temp-dir [remote-dir nil]
(let [[master remote] (init-source! "master" remote-dir
:files {"master.txt" "File in master"})
initial-version (source.p/version (source.p/snapshot master))]
(is (string? initial-version) "version should return a string")
(is (= 40 (count initial-version)) "version should be a full SHA-1 hash (40 characters)")
(is (= (git/commit-sha master "master") initial-version)
"version should match the commit id for the branch")
(testing "version changes after writing files"
(write-files! (source.p/snapshot master) "Update file" [{:path "master.txt" :content "Updated content"}])
(let [new-version (source.p/version (source.p/snapshot master))]
(is (not= initial-version new-version) "version should change after commit")
(is (= 40 (count new-version)) "new version should also be a full SHA-1 hash")
(is (= (git/commit-sha master "master") new-version)
"new version should match the new commit id")))
(testing "version is consistent across multiple calls"
(let [version-1 (source.p/version (source.p/snapshot master))
version-2 (source.p/version (source.p/snapshot master))]
(is (= version-1 version-2) "version should be consistent without changes")))
(testing "version differs for different branches"
(git-working-create-branch! remote "branch-1")
(let [branch-1 (->source! "branch-1" remote)
master-version (source.p/version (source.p/snapshot master))
branch-version (source.p/version (source.p/snapshot branch-1))]
(is (not= master-version branch-version)
"different branches should have different versions")))
(testing "version matches specific commit ref"
(let [commit-ref (git/commit-sha master "master")
source-with-ref (->source! commit-ref remote)]
(is (= commit-ref (source.p/version (source.p/snapshot source-with-ref)))
"version should work with explicit commit refs")))))))
(deftest default-branch
(mt/with-temp-dir [remote-dir nil]
(let [[master _remote] (init-source! "master" remote-dir)]
(is (= "master" (git/default-branch master))))))
(deftest write-files-top-level-exports-replaced-test
(let [old-col-path (str "collections/" "r" (subs (u/generate-nano-id "a") 1) "_mycol/")
new-col-path (str "collections/" "s" (subs (u/generate-nano-id "b") 1) "_othercol/")
kept-col-path (str "collections/" "t" (subs (u/generate-nano-id "c") 1) "_keptcol/")]
(mt/with-temp-dir [remote-dir nil]
(let [[master _remote] (init-source! "master" remote-dir
:files {"databases/old_db/old_db.yaml" "Old database"
"databases/old_db/schemas/public.yaml" "Old schema"
"snippets/old_snippet.yaml" "Old snippet"
"unmanaged/keep_me.txt" "Unmanaged file"
(str old-col-path "cards/card1.yaml") "Card in old col"
(str old-col-path "cards/card2.yaml") "Card 2 in old col"
(str kept-col-path "dashboards/dash1.yaml") "Dashboard in kept col"})]
(testing "Writing to a managed dir removes all stale files in ALL managed dirs"
(write-files! (source.p/snapshot master) "Rename database"
[{:path "databases/new_db/new_db.yaml" :content "Renamed database"}
{:path "databases/new_db/schemas/public.yaml" :content "Same schema"}
{:path (str old-col-path "cards/card1.yaml") :content "Card in old col"}
{:path (str old-col-path "cards/card2.yaml") :content "Card 2 in old col"}
{:path (str kept-col-path "dashboards/dash1.yaml") :content "Dashboard in kept col"}
{:path "snippets/old_snippet.yaml" :content "Old snippet"}])
(let [files (set (source.p/list-files (source.p/snapshot master)))]
(is (contains? files "databases/new_db/new_db.yaml") "New database file should exist")
(is (contains? files "databases/new_db/schemas/public.yaml") "New schema file should exist")
(is (not (contains? files "databases/old_db/old_db.yaml")) "Old database file should be removed")
(is (not (contains? files "databases/old_db/schemas/public.yaml")) "Old schema file should be removed")
(is (contains? files (str old-col-path "cards/card1.yaml")) "Written collection files should remain")
(is (contains? files "snippets/old_snippet.yaml") "Written snippet file should remain")
(is (contains? files "unmanaged/keep_me.txt") "Unmanaged files should be untouched")))
(testing "Entity moved between collections removes files from old collection"
(write-files! (source.p/snapshot master) "Move card to new collection"
[{:path (str new-col-path "cards/card1.yaml") :content "Card moved to new col"}
{:path (str kept-col-path "dashboards/dash1.yaml") :content "Dashboard still here"}
{:path "databases/new_db/new_db.yaml" :content "Renamed database"}
{:path "databases/new_db/schemas/public.yaml" :content "Same schema"}])
(let [files (set (source.p/list-files (source.p/snapshot master)))]
(is (contains? files (str new-col-path "cards/card1.yaml")) "Moved card should exist in new collection")
(is (contains? files (str kept-col-path "dashboards/dash1.yaml")) "Kept collection files should remain")
(is (not (contains? files (str old-col-path "cards/card1.yaml"))) "Old collection card should be removed")
(is (not (contains? files (str old-col-path "cards/card2.yaml"))) "Other files in old collection should also be removed")
(is (not (contains? files "snippets/old_snippet.yaml")) "Snippets cleaned up when not in write set")
(is (contains? files "unmanaged/keep_me.txt") "Unmanaged files still untouched")))))))
(deftest write-files-entity-rename-within-collection-test
(let [col-path (str "collections/" "u" (subs (u/generate-nano-id "d") 1) "_col/")]
(mt/with-temp-dir [remote-dir nil]
(let [[master _remote] (init-source! "master" remote-dir
:files {(str col-path "cards/eid123_OldCardName.yaml") "Card with old name"
(str col-path "cards/eid456_OtherCard.yaml") "Other card"})]
(testing "Entity renamed within a collection removes the old-named file"
(write-files! (source.p/snapshot master) "Rename card"
[{:path (str col-path "cards/eid123_NewCardName.yaml") :content "Card with new name"}
{:path (str col-path "cards/eid456_OtherCard.yaml") :content "Other card"}])
(let [files (set (source.p/list-files (source.p/snapshot master)))]
(is (contains? files (str col-path "cards/eid123_NewCardName.yaml")) "Renamed card should exist")
(is (contains? files (str col-path "cards/eid456_OtherCard.yaml")) "Other card should still exist")
(is (not (contains? files (str col-path "cards/eid123_OldCardName.yaml"))) "Old card name file should be removed")))))))
(deftest ensure-origin-configured-sets-origin-after-clone-test
(mt/with-temp-dir [remote-dir nil]
(let [[source _remote] (init-source! "master" remote-dir
:files {"master.txt" "File in master"})
^Git local-git (:git source)
config (.getConfig (.getRepository local-git))
origin-url (.getString config "remote" "origin" "url")]
(is (some? origin-url) "Origin URL should be set after clone"))))
(deftest ensure-origin-configured-repairs-corrupted-url-test
(mt/with-temp-dir [remote-dir nil]
(let [[source remote] (init-source! "master" remote-dir
:files {"master.txt" "File in master"})
^Git local-git (:git source)
config (.getConfig (.getRepository local-git))
corrupted-url "https://wrong-url.example.com/repo.git"]
(.setString config "remote" "origin" "url" corrupted-url)
(.save config)
(is (= corrupted-url (.getString config "remote" "origin" "url"))
"Origin URL should be corrupted")
(reset! @#'git/jgit {})
(let [repaired-source (->source! "master" remote)
^Git repaired-git (:git repaired-source)
repaired-config (.getConfig (.getRepository repaired-git))
repaired-url (.getString repaired-config "remote" "origin" "url")]
(is (not= corrupted-url repaired-url)
"Origin URL should no longer be the corrupted URL")
(is (str/includes? repaired-url remote-dir)
"Origin URL should point to the remote directory")))))
(deftest ensure-origin-configured-sets-fetch-refspec-test
(mt/with-temp-dir [remote-dir nil]
(let [[source remote] (init-source! "master" remote-dir
:files {"master.txt" "File in master"})
^Git local-git (:git source)
config (.getConfig (.getRepository local-git))]
(.setString config "remote" "origin" "url" "https://wrong-url.example.com/repo.git")
(.unset config "remote" "origin" "fetch")
(.save config)
(reset! @#'git/jgit {})
(let [repaired-source (->source! "master" remote)
^Git repaired-git (:git repaired-source)
repaired-config (.getConfig (.getRepository repaired-git))]
(is (= "+refs/heads/*:refs/heads/*"
(.getString repaired-config "remote" "origin" "fetch"))
"Origin fetch refspec should be set after repair")))))
(deftest ensure-origin-configured-allows-fetch-after-repair-test
(mt/with-temp-dir [remote-dir nil]
(let [[source remote] (init-source! "master" remote-dir
:files {"master.txt" "File in master"})
^Git local-git (:git source)
config (.getConfig (.getRepository local-git))]
(.setString config "remote" "origin" "url" "https://wrong-url.example.com/repo.git")
(.save config)
(reset! @#'git/jgit {})
(let [repaired-source (->source! "master" remote)]
(git-working-add! remote "new-file.txt" "New content")
(git-working-commit! remote "Add new file")
(git/fetch! repaired-source)
(is (= ["Add new file" "Initial commit"]
(map :message (git/log repaired-source)))
"Should be able to fetch after origin repair")))))
(deftest get-jgit-reclones-after-local-repo-deleted-test
(testing "GHY-3815: if the cached local clone dir is deleted out from under us, the next
operation re-clones instead of returning a stale cached Git instance (which fails
permanently with 'origin: not found' until an instance restart)"
(mt/with-temp-dir [remote-dir nil]
(let [[source remote] (init-source! "master" remote-dir :branches ["branch-1"])
remote-url (:remote-url source)
^File local-path (#'git/repo-path {:remote-url remote-url})
^Git cached-git (:git source)]
(is (.exists local-path) "Precondition: local clone dir exists after the initial clone")
(is (= ["branch-1" "master"] (source.p/branches source))
"Precondition: branches works before the dir is deleted")
(FileUtils/deleteDirectory local-path)
(is (not (.exists local-path)) "Local clone dir is gone")
(let [fresh-source (->source! "master" remote)]
(is (.exists local-path) "Local clone dir was re-created (re-cloned)")
(is (not (identical? cached-git (:git fresh-source)))
"A fresh Git instance is returned, not the stale cached one")
(is (= ["branch-1" "master"] (source.p/branches fresh-source))
"branches works again after the dir was deleted, without an instance restart"))))))
(deftest ^:parallel credentials-provider-test
(testing "GitHub URL uses x-access-token"
(let [provider (git/credentials-provider "https://github.com/org/repo.git" "my-token")]
(is (instance? UsernamePasswordCredentialsProvider provider))))
(testing "Bitbucket URL uses x-token-auth"
(let [provider (#'git/credentials-provider "https://bitbucket.org/org/repo" "my-token")]
(is (instance? UsernamePasswordCredentialsProvider provider)))))
;; ---------------------------------------------------------------------------
;; Missing remote branch tests (issue #72778)
;; ---------------------------------------------------------------------------
(defn- delete-remote-branch!
"Deletes a branch on the 'remote' repo used by a test."
[{:keys [^Git git]} ^String branch]
(-> (.branchDelete git)
(.setBranchNames ^"[Ljava.lang.String;" (into-array String [branch]))
(.setForce true)
(.call)))
(deftest fetch!-prunes-deleted-remote-branches-test
(mt/with-temp-dir [remote-dir nil]
(let [[source remote] (init-source! "master" remote-dir :branches ["branch-1"])]
(is (some? (git/commit-sha source "branch-1"))
"Precondition: branch-1 is resolvable locally after initial clone")
(delete-remote-branch! remote "branch-1")
(git/fetch! source)
(is (nil? (git/commit-sha source "branch-1"))
"branch-1 ref is pruned locally after the remote branch is deleted")
(is (some? (git/commit-sha source "master"))
"other refs are unaffected"))))
(deftest snapshot-throws-missing-branch-ex-data-test
(mt/with-temp-dir [remote-dir nil]
(let [[_master remote] (init-source! "master" remote-dir
:files {"master.txt" "x"})
bad-source (->source! "does-not-exist" remote)]
(try
(source.p/snapshot bad-source)
(is false "snapshot should have thrown")
(catch clojure.lang.ExceptionInfo e
(is (= "Invalid branch: does-not-exist" (ex-message e)))
(is (= :missing-branch (:error-type (ex-data e))))
(is (= "does-not-exist" (:branch (ex-data e)))))))))
(deftest snapshot-throws-missing-branch-after-remote-delete-test
(mt/with-temp-dir [remote-dir nil]
(let [[_master remote] (init-source! "master" remote-dir :branches ["branch-1"])
source-on-branch-1 (->source! "branch-1" remote)]
(is (some? (source.p/snapshot source-on-branch-1))
"Precondition: snapshot works before the branch is deleted")
(delete-remote-branch! remote "branch-1")
(try
(source.p/snapshot source-on-branch-1)
(is false "snapshot should have thrown after the remote branch was deleted")
(catch clojure.lang.ExceptionInfo e
(is (= :missing-branch (:error-type (ex-data e))))
(is (= "branch-1" (:branch (ex-data e)))))))))