Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Enumerate data apps by reading git tree objects (#78114)

Timofey Kachalova2d673e0fe1c
diff --git a/enterprise/backend/src/metabase_enterprise/data_apps/sync.clj b/enterprise/backend/src/metabase_enterprise/data_apps/sync.clj
index 0d83759ba64..af5897142ab 100644
--- a/enterprise/backend/src/metabase_enterprise/data_apps/sync.clj
+++ b/enterprise/backend/src/metabase_enterprise/data_apps/sync.clj
@@ -8,7 +8,9 @@
failure-isolation guarantees this namespace implements.
This namespace does no Git access of its own — the snapshot's `read-file` /
- `list-files` come from remote-sync."
+ `list-dir` come from remote-sync. The cached `bundle` blob is what serving
+ reads, so a failed sync never takes a working app offline, and the admin
+ `enabled` toggle is preserved across syncs."
(:require
[clojure.string :as str]
[metabase-enterprise.data-apps.config :as data-app.config]
@@ -18,7 +20,6 @@
[toucan2.core :as t2])
(:import
(java.security MessageDigest)
- (java.util.regex Pattern)
(org.apache.commons.codec.binary Hex)))
(set! *warn-on-reflection* true)
@@ -46,34 +47,34 @@
;;; ----------------------------------------------------- Discovery -----------------------------------------------------
-(def ^:private app-dir-regex
- ;; the data_apps/<dir> folder a repo path sits in (dir = a single path segment)
- (re-pattern (format "(%s/[^/]+)/.*" (Pattern/quote data-app.config/apps-dir))))
-
(defn- discover-app-configs
- "Given the snapshot's `list-files` and `read-file` fns (where `read-file`
+ "Given the snapshot's `list-dir` and `read-file` fns (where `list-dir` returns a
+ directory's immediate children as repo-root relative paths, and `read-file`
returns file text or nil), iterate the folders under `data_apps/` and return one
entry per folder that is an app — it holds a `data_app.yaml`; a folder without
- one is skipped. Each entry carries `:slug` (the directory name) plus either the
- parsed app `{:display_name :bundle :allowed_hosts}` (with `:bundle` the
- repo-root relative bundle path) or `{:config-error <message>}`. Parse/read
- failures are isolated per app so one bad config can't abort the sync — and the
- `:slug` is present either way, so pruning can tell a directory that still exists
- (transiently broken config) from one that was removed."
- [list-files read-file]
- (let [files (set (list-files))
- app-dirs (distinct (keep #(second (re-matches app-dir-regex %)) files))]
- (for [dir app-dirs
- :let [config-path (files (str dir "/data_app.yaml"))]
- :when config-path]
- (let [slug (data-app.config/dir-slug dir)]
- (try
- (if-let [content (read-file config-path)]
- (let [{:keys [display_name path allowed_hosts]} (data-app.config/parse-app-config (->bytes content) dir)]
- {:slug slug, :display_name display_name, :bundle (str dir "/" path), :allowed_hosts allowed_hosts})
- {:slug slug, :config-error (tru "Could not read {0}." config-path)})
- (catch Throwable e
- {:slug slug, :config-error (ex-message e)}))))))
+ one is skipped. Each entry is a parsed app `{:slug :display_name :bundle
+ :allowed_hosts}` (with `:bundle` the repo-root relative bundle path) or
+ `{:config-error <message>}`. Parse/read failures are isolated per app so one bad
+ config can't abort the sync.
+
+ Reads only `data_apps/` and its app folders, never the whole tree. That matters
+ beyond this fn: `data_apps/` is not a serdes path, so a pull that changes only
+ data apps builds no ingestable at all (see remote-sync's
+ `incremental-import-plan`) — discovery here is the only thing that would touch
+ the tree, and it now stays proportional to the number of apps rather than to
+ the size of the repo."
+ [list-dir read-file]
+ (for [dir (list-dir data-app.config/apps-dir)
+ :let [config-path (str dir "/" data-app.config/config-file-name)]
+ ;; a plain file under `data_apps/` lists no children, so it drops out here
+ :when (some #{config-path} (list-dir dir))]
+ (try
+ (if-let [content (read-file config-path)]
+ (let [{:keys [slug display_name path allowed_hosts]} (data-app.config/parse-app-config (->bytes content) dir)]
+ {:slug slug, :display_name display_name, :bundle (str dir "/" path), :allowed_hosts allowed_hosts})
+ {:slug (data-app.config/dir-slug dir), :config-error (tru "Could not read {0}." config-path)})
+ (catch Throwable e
+ {:slug (data-app.config/dir-slug dir), :config-error (ex-message e)}))))
;;; ----------------------------------------------------- Materialize -----------------------------------------------------
@@ -149,9 +150,9 @@
(defn import-from-snapshot!
"Materialize data apps from a synced repo `snapshot`:
- {:read-file (fn [path] -> <file-text-string> | nil)
- :list-files (fn [] -> [<path-string> ...])
- :sha <commit-sha-string>}
+ {:read-file <(fn [path] -> <file-text-string> | nil)>
+ :list-dir <(fn [path] -> [<child-path-string> ...])>
+ :sha <commit-sha-string>}
Discovers every `data_apps/<dir>/data_app.yaml`, upserts a row per app, and prunes
rows whose directory is gone from the snapshot — all in one transaction. Returns
@@ -160,21 +161,22 @@
unchanged content does not count) and `:removed` counts apps dropped for no longer
being in the repo.
- See `README.md` in this directory for the source-of-truth rules this implements,
- and for why a broken config marks its app failed instead of pruning it."
- [{:keys [read-file list-files sha]}]
+ Two apps can't collide on a slug here: a slug *is* an app's directory name (see
+ the config namespace), a repo can't hold two `data_apps/<slug>` directories, and
+ discovery takes one config per directory (see [[discover-app-configs]])."
+ [{:keys [read-file list-dir sha]}]
;; realize discovery once (it calls read-file/parse per config); `results` is
;; then walked several times below
- (let [results (vec (discover-app-configs list-files read-file))
- good (remove :config-error results)
- errors (vec (keep :config-error results))
+ (let [results (vec (discover-app-configs list-dir read-file))
+ good (remove :config-error results)
+ errors (vec (keep :config-error results))
;; every app directory present in the repo (parsed or not) — the set of
;; slugs that should survive this sync
present-slugs (into #{} (map :slug) results)
;; pre-sync rows, so we can tell a real change from a sha/timestamp bump
- existing (into {} (map (juxt :name identity))
- (t2/select [:model/DataApp :name :display_name :allowed_hosts
- :bundle_path :bundle_hash :sync_error]))
+ existing (into {} (map (juxt :name identity))
+ (t2/select [:model/DataApp :name :display_name :allowed_hosts
+ :bundle_path :bundle_hash :sync_error]))
{:keys [changed removed]}
(t2/with-transaction [_conn]
(let [changed (reduce (fn [n {:keys [slug config-error] :as cfg}]
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
index eb82e819d0d..f1c568d9abd 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
@@ -321,9 +321,9 @@
;; `read-file` returns file text (a string) or nil; data-apps.sync converts to
;; bytes on its side, keeping all Java interop out of this namespace.
(data-apps.sync/sync-from-snapshot!
- {:read-file (fn [path] (source.p/read-file snapshot path))
- :list-files (fn [] (source.p/list-files snapshot))
- :sha (source.p/version snapshot)}))
+ {:read-file (fn [path] (source.p/read-file snapshot path))
+ :list-dir (fn [path] (source.p/list-dir snapshot path))
+ :sha (source.p/version snapshot)}))
(defn load-snapshot!
"Loads a snapshot's serialized entities into the app DB and reconciles local state to match it:
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/source.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/source.clj
index 93652085417..6d8d7bd2d3b 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/source.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/source.clj
@@ -19,6 +19,24 @@
(set! *warn-on-reflection* true)
+(defn paths->children
+ "The immediate children of directory `path` implied by a flat collection of file `paths`. For snapshots
+ that hold their whole file list in memory and so have no tree objects to walk; the git source overrides
+ this with a real subtree read."
+ [paths ^String path]
+ ;; the root is the empty prefix — snapshot paths are repo-relative, so they never start with a slash
+ (let [prefix (if (str/blank? path) "" (str path "/"))]
+ (->> paths
+ (keep (fn [^String p]
+ (when (str/starts-with? p prefix)
+ (let [child (subs p (count prefix))]
+ (str prefix (if-let [idx (str/index-of child "/")]
+ (subs child 0 idx)
+ child))))))
+ distinct
+ sort
+ vec)))
+
;; A read-only, path-filtered view over a snapshot, used to scope ingestion to a set of path regexes:
;; files (and reads) outside the filters are omitted. It is not a write target — exports write to the
;; unfiltered source snapshot — so the write methods throw rather than silently filtering writes.
@@ -30,6 +48,11 @@
(some (fn [path-filter] (re-matches path-filter file-path)) path-filters))
(source.p/list-files original-snapshot)))
+ ;; Derived from this view's own (filtered) file list rather than delegated, so a child filtered out of
+ ;; `list-files` can't reappear here.
+ (list-dir [this path]
+ (paths->children (source.p/list-files this) path))
+
(read-file [_ path]
(when (some (fn [path-filter] (re-matches path-filter path)) path-filters)
(source.p/read-file original-snapshot path)))
@@ -138,6 +161,7 @@
(let [by-path (into {} (map (juxt :path :content)) specs)]
(reify source.p/SourceSnapshot
(list-files [_] (vec (keys by-path)))
+ (list-dir [_ path] (paths->children (keys by-path) path))
(read-file [_ path] (get by-path path))
(open-commit [_] (throw (ex-info "in-memory merge snapshot is read-only" {})))
(version [_] nil))))
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj
index 05b151d0351..f48080af164 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj
@@ -206,6 +206,55 @@
(recur (conj files (.getPathString tree-walk)))
files)))))
+(defn- tree-children
+ "Paths of the entries at the walk's current depth, consuming the walk. `next` climbs back out of a
+ subtree once it's exhausted, so a drop in depth is what marks the end of the children.
+
+ Sorted rather than left in tree order: git orders a tree's entries as if directories ended in `/`, so
+ raw order puts a sibling `a-b` before the directory `a` (`\\-` < `\\/`). Sorting gives plain
+ lexicographic order instead — the one rule the flat snapshots can honour too, since they have no tree
+ order to inherit."
+ [^TreeWalk tree-walk]
+ (let [depth (.getDepth tree-walk)]
+ (loop [paths []]
+ (if (and (.next tree-walk) (= depth (.getDepth tree-walk)))
+ (recur (conj paths (.getPathString tree-walk)))
+ (vec (sort paths))))))
+
+(defn list-dir
+ "Lists the immediate children of one directory in the git repository at the snapshot.
+
+ Takes a GitSnapshot containing a :git Git instance and :version specifying which commit to read, and a
+ repo-root relative directory path (no trailing slash).
+
+ Resolves the commit's root tree, looks `path` up in it and reads that single tree object, iterating its
+ entries non-recursively — so the cost is proportional to the depth of `path` plus the number of entries
+ it holds, not to the size of the repository. The clone is bare (git objects, no working tree), which is
+ exactly what this walks.
+
+ `.isSubtree` is what makes a non-directory return `[]`: a symlink and a submodule are entries with
+ their own modes, not trees, so neither can be descended into (and a submodule's tree isn't in this
+ repository at all).
+
+ See [[metabase-enterprise.remote-sync.source.protocol/list-dir]] for the contract this implements and
+ why it takes the shape it does."
+ [{:keys [^Git git ^String version]} ^String path]
+ (let [repo (.getRepository git)]
+ (with-open [rev-walk (RevWalk. repo)]
+ (or (when-let [commit-id (.resolve repo version)]
+ (let [tree (.getTree (.parseCommit rev-walk commit-id))]
+ (if (str/blank? path)
+ (with-open [^TreeWalk tree-walk (TreeWalk. repo)]
+ (.addTree tree-walk tree)
+ (tree-children tree-walk))
+ ;; one binary search per path segment down to the entry, then a single tree object read
+ (when-let [found (TreeWalk/forPath repo path tree)]
+ (with-open [^TreeWalk tree-walk found]
+ (when (.isSubtree tree-walk)
+ (.enterSubtree tree-walk)
+ (tree-children tree-walk)))))))
+ []))))
+
(defn read-file
"Reads the contents of a specific file from the git snapshot.
@@ -500,6 +549,9 @@
(list-files [this]
(list-files this))
+ (list-dir [this path]
+ (list-dir this path))
+
(read-file [this path]
(read-file this path))
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/source/protocol.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/source/protocol.clj
index 342e3de2b43..cc96f86c5b2 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/source/protocol.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/source/protocol.clj
@@ -49,6 +49,50 @@
Returns a collection of file path strings.")
+ (list-dir [snapshot path]
+ "Lists the immediate children of the directory `path` (repo-root relative, no trailing slash; the
+ empty string is the root).
+
+ Takes a SourceSnapshot instance implementing this protocol.
+
+ Returns a lexicographically sorted vector of repo-root relative paths, or an empty vector. Given
+
+ .
+ ├── bar
+ └── baz
+ └── burp
+
+ (list-files snapshot) ;; => [\"bar\" \"baz/burp\"]
+
+ (list-dir snapshot \"\") ;; => [\"bar\" \"baz\"]
+ (list-dir snapshot \"baz\") ;; => [\"baz/burp\"]
+ (list-dir snapshot \"bar\") ;; => []
+ (list-dir snapshot \"nope\") ;; => []
+
+ Every choice that example shows is deliberate, and hard to revisit once callers depend on it:
+
+ - *Full paths, not bare names* — `baz/burp`, never `burp`. `read-file` takes a repo-root relative
+ path and `list-files` returns them, so a child can be fed straight back in without reassembling
+ it, and one rule covers everything this protocol hands out.
+
+ - *One level only.* \"Every file under `path`\" is derivable by prefix-filtering `list-files`; a
+ single directory's contents are not derivable from anything cheaper, and for a git source cost
+ one tree object rather than one per subtree. So this is the primitive; the recursive form is the
+ derived one.
+
+ - *Directories are listed alongside files* — and are the reason this exists. `list-files` yields
+ only leaves, so a directory never appears there and can be discovered only here.
+
+ - *Only a directory has children.* A file, a symlink (even one pointing at a directory), a
+ submodule, and a path that doesn't exist all return `[]` rather than throwing. Dotted names are
+ ordinary entries: git has no notion of a hidden file, so neither does this.
+
+ - *Sorted lexicographically*, not in git's own tree order — git compares a directory as if it
+ ended in `/`, which would put `d/a-b` before `d/a`. Lexicographic is the one order the snapshots
+ that hold a flat file list can produce too, so every implementation agrees.
+
+ Reads only the one directory, so its cost is independent of the size of the rest of the repo.")
+
(read-file [snapshot path]
"Reads the contents of a file from the snapshot.
diff --git a/enterprise/backend/test/metabase_enterprise/data_apps/api_test.clj b/enterprise/backend/test/metabase_enterprise/data_apps/api_test.clj
index bbebf30a38d..3e8dea64471 100644
--- a/enterprise/backend/test/metabase_enterprise/data_apps/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/data_apps/api_test.clj
@@ -3,6 +3,7 @@
[clojure.string :as str]
[clojure.test :refer :all]
[metabase-enterprise.data-apps.sync :as data-app.sync]
+ [metabase-enterprise.remote-sync.source :as source]
[metabase.test :as mt]
[toucan2.core :as t2]))
@@ -22,11 +23,12 @@
(defn- snapshot
"Build a snapshot (as the remote-sync import passes one) from a path->content
- map. `read-file` returns file text (a string) or nil."
+ map. `read-file` returns file text (a string) or nil; `list-dir` reuses the
+ derivation the real non-git snapshots use, so the fake can't drift from it."
[path->content & {:keys [sha] :or {sha fake-sha}}]
- {:sha sha
- :list-files (fn [] (vec (keys path->content)))
- :read-file (fn [p] (get path->content p))})
+ {:sha sha
+ :list-dir (fn [dir] (source/paths->children (keys path->content) dir))
+ :read-file (fn [p] (get path->content p))})
(defn- app-config
"Render a per-app data_app.yaml from `{:name :path :allowed_hosts}`. No slug: an
diff --git a/enterprise/backend/test/metabase_enterprise/data_apps/sync_test.clj b/enterprise/backend/test/metabase_enterprise/data_apps/sync_test.clj
index 8f8d17b34d1..3a6b426b8f8 100644
--- a/enterprise/backend/test/metabase_enterprise/data_apps/sync_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/data_apps/sync_test.clj
@@ -7,6 +7,7 @@
[clojure.string :as str]
[clojure.test :refer :all]
[metabase-enterprise.data-apps.sync :as data-app.sync]
+ [metabase-enterprise.remote-sync.source :as source]
[metabase.test :as mt]
[toucan2.core :as t2]))
@@ -16,9 +17,9 @@
(defn- snapshot
[path->content & {:keys [sha] :or {sha fake-sha}}]
- {:sha sha
- :list-files (fn [] (vec (keys path->content)))
- :read-file (fn [p] (get path->content p))})
+ {:sha sha
+ :list-dir (fn [dir] (source/paths->children (keys path->content) dir))
+ :read-file (fn [p] (get path->content p))})
(defn- app-files
"The repo files for one app in `data_apps/<dir>`. `dir` is the app's slug — the
@@ -149,10 +150,9 @@
(deftest an-unreadable-config-is-a-config-error-test
(testing "a data_app.yaml the snapshot lists but can't read is isolated as a config-error, not a crash"
(mt/with-model-cleanup [:model/DataApp]
+ ;; the config is listed in the tree, but reading its blob yields nothing
(let [result (data-app.sync/import-from-snapshot!
- {:sha fake-sha
- :list-files (fn [] ["data_apps/ghost/data_app.yaml"])
- :read-file (fn [_] nil)})]
+ (snapshot {"data_apps/ghost/data_app.yaml" nil}))]
(is (= 1 (count (:config-errors result))))
(is (str/includes? (first (:config-errors result)) "data_apps/ghost/data_app.yaml"))
(is (empty? (t2/select-fn-set :name :model/DataApp)))))))
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj
index bdb1bfcd10c..3e878968fda 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj
@@ -1785,6 +1785,7 @@ serdes/meta:
(reify source.p/SourceSnapshot
(version [_] version)
(list-files [_] [])
+ (list-dir [_ _] [])
(read-file [_ _] nil)
(open-commit [_]
(reify source.p/CommitBuilder
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj
index d346abb3cb2..91b7de837ff 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj
@@ -9,9 +9,12 @@
[metabase.test :as mt]
[metabase.util :as u])
(:import (java.io File)
+ (java.nio.file Files Paths)
+ (java.nio.file.attribute FileAttribute)
(org.apache.commons.io FileUtils)
(org.eclipse.jgit.api Git TransportCommand)
- (org.eclipse.jgit.lib PersonIdent)
+ (org.eclipse.jgit.dircache DirCacheEditor DirCacheEditor$PathEdit DirCacheEntry)
+ (org.eclipse.jgit.lib AnyObjectId FileMode PersonIdent)
(org.eclipse.jgit.transport UsernamePasswordCredentialsProvider)))
(set! *warn-on-reflection* true)
@@ -70,6 +73,30 @@
(.addFilepattern path)
(.call))))
+(defn- git-working-link!
+ "Writes a symlink in the working path and stages it, so a test can cover the mode-120000 entries a real
+ repo can hold. Git stores the link's *target text* as the blob, and never follows it."
+ [{:keys [^Git git]} ^String path ^String target]
+ (let [full-path (io/file (.getWorkTree (.getRepository git)) path)]
+ (io/make-parents full-path)
+ (Files/createSymbolicLink (.toPath full-path)
+ (Paths/get target (into-array String []))
+ (into-array FileAttribute []))
+ (-> (.add git)
+ (.addFilepattern path)
+ (.call))))
+
+(defn- git-working-gitlink!
+ "Stages a submodule entry (mode 160000) pointing at `commit-id`. A gitlink has no working-tree file to
+ add, so this edits the index directly, as `git update-index --cacheinfo 160000,<sha>,<path>` does."
+ [{:keys [^Git git]} ^String path ^AnyObjectId commit-id]
+ (let [^DirCacheEditor editor (.editor (.lockDirCache (.getRepository git)))]
+ (.add editor (proxy [DirCacheEditor$PathEdit] [path]
+ (apply [^DirCacheEntry entry]
+ (.setFileMode entry FileMode/GITLINK)
+ (.setObjectId entry commit-id))))
+ (.commit editor)))
+
(defn- git-working-create-branch!
"Creates a branch with an initial commit and file using the working directory"
[source ^String branch]
@@ -198,6 +225,72 @@
(is (= ["file-in-branch-1.txt" "master.txt" "subdir/path.txt"] (source.p/list-files branch-1)))
(is (= ["file-in-branch-2.txt" "master.txt" "subdir/path.txt"] (source.p/list-files branch-2))))))
+(deftest list-dir
+ (mt/with-temp-dir [remote-dir nil]
+ (let [[master _remote] (init-source! "master" remote-dir
+ :files {"root.txt" "at the root"
+ "dir/readme.md" "a file, not a directory"
+ "dir/alpha/one.txt" "ONE"
+ "dir/alpha/two.txt" "TWO"
+ "dir/alpha/deep/leaf.txt" "LEAF"
+ "dir/beta/three.txt" "THREE"
+ ;; \- sorts before \/, so git's own tree order here is
+ ;; ("d/a-b" "d/a") — the case the sort normalizes
+ "d/a-b" "a sibling"
+ "d/a/x" "inside a directory"
+ "zzz/other.txt" "elsewhere"})
+ snap (source.p/snapshot master)]
+ (testing "immediate children only — no descendants, no siblings of the directory itself"
+ (is (= ["dir/alpha" "dir/beta" "dir/readme.md"] (source.p/list-dir snap "dir"))))
+ (testing "children are full repo-root relative paths, so they feed straight back into read-file"
+ (let [child (some #{"dir/alpha/one.txt"} (source.p/list-dir snap "dir/alpha"))]
+ (is (= "ONE" (source.p/read-file snap child)))))
+ (testing "files and directories are both listed, sorted — `deep` is a directory"
+ (is (= ["dir/alpha/deep" "dir/alpha/one.txt" "dir/alpha/two.txt"]
+ (source.p/list-dir snap "dir/alpha"))))
+ (testing "the repo root"
+ (is (= ["d" "dir" "root.txt" "zzz"] (source.p/list-dir snap ""))))
+ (testing "ordering is plain lexicographic, not git's own tree order"
+ ;; git compares a directory as if it ended in "/", so its tree order here is ("d/a-b" "d/a").
+ ;; We normalize to lexicographic — the one order a flat path list can produce too, so every
+ ;; snapshot implementation agrees.
+ (is (= ["d/a" "d/a-b"] (source.p/list-dir snap "d"))))
+ (testing "nesting: each call steps down exactly one level, and paths stay rooted at the repo"
+ (is (= ["dir/alpha/deep/leaf.txt"] (source.p/list-dir snap "dir/alpha/deep"))))
+ (testing "a path that is a file, or absent, has no children rather than throwing"
+ (is (= [] (source.p/list-dir snap "dir/readme.md")))
+ (is (= [] (source.p/list-dir snap "nope")))
+ (is (= [] (source.p/list-dir snap "dir/alpha/nope")))))))
+
+(deftest list-dir-dotfiles-and-symlinks
+ (testing "the entry kinds a real repo can hold, beyond plain files and directories"
+ (mt/with-temp-dir [remote-dir nil]
+ (let [remote (init-remote! remote-dir
+ :files {"dir/plain.txt" "a file"
+ "dir/.dotfile" "a dotfile"
+ "dir/.hidden/file.txt" "inside a dot-directory"
+ "outside/target.txt" "outside dir"})
+ _ (git-working-link! remote "dir/linkdir" "../outside")
+ _ (git-working-link! remote "dir/linkfile" "plain.txt")
+ head (git-working-commit! remote "add symlinks")
+ _ (git-working-gitlink! remote "dir/sub" head)
+ _ (git-working-commit! remote "add submodule")
+ snap (source.p/snapshot (->source! "master" remote))]
+ (testing "git has no notion of a hidden file, so neither do we — dotted names are listed as-is"
+ (is (= ["dir/.dotfile" "dir/.hidden" "dir/linkdir" "dir/linkfile" "dir/plain.txt" "dir/sub"]
+ (source.p/list-dir snap "dir")))
+ (is (= ["dir/.hidden/file.txt"] (source.p/list-dir snap "dir/.hidden"))
+ "a dot-directory is an ordinary tree we descend into"))
+ (testing "only a tree has children: everything else lists nothing, whatever it points at"
+ (is (= [] (source.p/list-dir snap "dir/linkdir")) "a symlink to a directory")
+ (is (= [] (source.p/list-dir snap "dir/linkfile")) "a symlink to a file")
+ (is (= [] (source.p/list-dir snap "dir/sub")) "a submodule (gitlink) — its tree isn't in this repo")
+ (is (= [] (source.p/list-dir snap "dir/plain.txt")) "a plain file"))
+ (testing "reading a symlink yields its target text, never the target's content: git does not follow
+ links, so a link can neither escape the repo nor pull in a file from outside the directory"
+ (is (= "../outside" (source.p/read-file snap "dir/linkdir")))
+ (is (= "plain.txt" (source.p/read-file snap "dir/linkfile"))))))))
+
(deftest read-file
(mt/with-temp-dir [remote-dir nil]
(let [[master _remote] (init-source! "master" remote-dir
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/source/wrapping_source_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/source/wrapping_source_test.clj
index 31895f28fe7..301c063ad24 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/source/wrapping_source_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/source/wrapping_source_test.clj
@@ -9,6 +9,9 @@
(list-files [_]
(keys files))
+ (list-dir [_ path]
+ (source/paths->children (keys files) path))
+
(read-file [_ path]
(get files path))
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/source_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/source_test.clj
index 587ba275e5d..38f33c0b753 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/source_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/source_test.clj
@@ -31,6 +31,7 @@
(let [by-path (into {} (map (juxt :path :content)) (source/serialize-specs entities task-id))]
(reify source.p/SourceSnapshot
(list-files [_] (vec (keys by-path)))
+ (list-dir [_ p] (source/paths->children (keys by-path) p))
(read-file [_ p] (get by-path p))
(open-commit [_]
(let [staged (atom [])]
@@ -49,6 +50,28 @@
(abort-commit! [_] nil))))
(version [_] "remote-tip"))))
+(deftest paths->children-test
+ (testing "the flat-path stand-in matches the contract git's list-dir implements"
+ (let [paths ["root.txt"
+ "data_apps/README.md"
+ "data_apps/beta/data_app.yaml"
+ "data_apps/alpha/data_app.yaml"
+ "data_apps/alpha/deep/nested.js"]]
+ (testing "immediate children only, as full paths, sorted and deduped"
+ (is (= ["data_apps/README.md" "data_apps/alpha" "data_apps/beta"]
+ (source/paths->children paths "data_apps")))
+ (is (= ["data_apps/alpha/data_app.yaml" "data_apps/alpha/deep"]
+ (source/paths->children paths "data_apps/alpha"))))
+ (testing "the repo root — paths are repo-relative, so the root prefix is empty, not \"/\""
+ (is (= ["data_apps" "root.txt"] (source/paths->children paths ""))))
+ (testing "a file, or an absent directory, has no children"
+ (is (= [] (source/paths->children paths "root.txt")))
+ (is (= [] (source/paths->children paths "nope"))))
+ (testing "plain lexicographic order, which a single level of git's own tree order is not"
+ ;; git orders entries as if directories ended in "/", and \- sorts before \/, so git would hand
+ ;; back ("d/a-b" "d/a") here. Lexicographic is the rule a flat path list can honour too.
+ (is (= ["d/a" "d/a-b"] (source/paths->children ["d/a-b" "d/a/x"] "d")))))))
+
(deftest preview-merge-clean-test
(testing "preview-merge reports a clean merge and summary without writing"
(mt/with-temp [:model/RemoteSyncTask {task-id :id} {:sync_task_type "export"}]
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj
index 1b1e3325c39..170f96d33ce 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj
@@ -3,6 +3,7 @@
(:require
[clojure.string :as str]
[clojure.test :as t]
+ [metabase-enterprise.remote-sync.source :as source]
[metabase-enterprise.remote-sync.source.protocol :as source.p]
[metabase-enterprise.serialization.v2.ingest :as ingest]
[metabase-enterprise.transforms-python.core :as transforms-python]
@@ -182,6 +183,10 @@ width: fixed
;; Default success case - return files from atom
(keys (get @files-atom branch {}))))
+ ;; Derived from `list-files` (which propagates this mock's failure modes), as the non-git snapshots do.
+ (list-dir [this path]
+ (source/paths->children (source.p/list-files this) path))
+
(read-file [_this path]
(case fail-mode
:read-file-error (throw (Exception. "Failed to read file"))
@@ -309,6 +314,8 @@ width: fixed
source.p/SourceSnapshot
(list-files [_] (vec (keys (get-in @state [:trees version] {}))))
+ (list-dir [_ path]
+ (source/paths->children (keys (get-in @state [:trees version] {})) path))
(read-file [_ path] (get-in @state [:trees version path]))
(open-commit [_]
(let [staged-upserts (atom [])