Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

7.7 KB
Raw Blame
(ns metabuild-common.files
  (:require
   [clojure.java.io :as io]
   [clojure.string :as str]
   [metabuild-common.misc :as misc]
   [metabuild-common.output :as out]
   [metabuild-common.shell :as shell]
   [metabuild-common.steps :as steps])
  (:import
   (java.io File FileInputStream FileOutputStream)
   (java.net URI)
   (java.nio.file Files FileSystem FileSystems FileVisitOption Path Paths)
   (java.util Collections)
   (java.util.function BiPredicate)
   (java.util.zip ZipEntry ZipOutputStream)
   (org.apache.commons.io FileUtils)))

(set! *warn-on-reflection* true)

(defn file-exists?
  "Does a file or directory with `filename` exist?"
  [^String filename]
  (when filename
    (.exists (File. filename))))

(defn assert-file-exists
  "If file with `filename` exists, return `filename` as is; otherwise, throw Exception."
  ^String [filename & [message]]
  (when-not (file-exists? filename)
    (throw (ex-info (format "File %s does not exist. %s" (pr-str filename) (or message "")) {:filename filename})))
  (str filename))

(defn create-directory-unless-exists!
  "Create a directory if it does not already exist. Returns `dir`."
  ^String [^String dir]
  (steps/step (format "Create directory %s if it does not exist" dir)
    (if (file-exists? dir)
      (out/announce "%s already exists." dir)
      (steps/step (format "Create directory %s" dir)
        (.mkdirs (File. dir)))))
  dir)

(defn delete-file-if-exists!
  "Delete a file or directory (recursively) if it exists."
  ([^String filename]
   (steps/step (format "Delete %s if exists" filename)
     (if (file-exists? filename)
       (let [file (File. filename)]
         (if (.isDirectory file)
           (FileUtils/deleteDirectory file)
           (.delete file))
         (out/safe-println (format "Deleted %s." filename)))
       (out/safe-println (format "Don't need to delete %s, file does not exist." filename)))
     (assert (not (file-exists? filename)))))

  ([file & more]
   (dorun (map delete-file-if-exists! (cons file more)))))

(defn copy-file!
  "Copy a `source` file (or directory, recursively) to `dest`."
  [^String source ^String dest]
  (let [source-file (File. (assert-file-exists source))]
    ;; Use native `cp` rather than FileUtils or the like because codesigning is broken when you use those because they
    ;; don't preserve symlinks or something like that.
    (if (.isDirectory source-file)
      (steps/step (format "Copying directory %s -> %s" source dest)
        (shell/sh "cp" "-R" source dest))
      (steps/step (format "Copying file %s -> %s" source dest)
        (shell/sh "cp" source dest))))
  (assert-file-exists dest))

(defn- ->URI ^java.net.URI [filename]
  (java.net.URI. (str "file://" filename)))

(defn- ->Path ^Path [filename]
  (Paths/get (->URI filename)))

(defn find-files
  "Pure Java version of `find`. Recursively find files in `dir-path` that satisfy `pred`, which has the signature

    (pred filename-string) -> Boolean"
  [^String dir-path pred]
  (->> (Files/find (->Path dir-path)
                   Integer/MAX_VALUE
                   (reify BiPredicate
                     (test [_ path _]
                       (boolean (pred (str path)))))
                   ^FileVisitOption (make-array FileVisitOption 0))
       .toArray
       (map str)
       sort))

(defn filename
  "Create a filename path String by joining path components:

    (filename \"usr\" \"cam\" \".emacs.d\" \"init.el\")
    ;; -> \"usr/cam/.emacs.d/init.el\""
  [& path-components]
  (str/join File/separatorChar path-components))

(def ^String project-root-directory
  "Root directory of the Metabase repo, e.g. `/users/cam/metabase`. Determined based on its location relative to this
  source file."
  (.. (Paths/get (.toURI (io/resource "metabuild_common/files.clj")))
      toFile
      getParentFile   ; /home/cam/metabase/bin/build/src/metabuild_common/
      getParentFile   ; /home/cam/metabase/bin/build/src/
      getParentFile   ; /home/cam/metabase/bin/build/
      getParentFile   ; /home/cam/metabase/bin/
      getParentFile   ; /home/cam/metabase/
      getCanonicalPath))

(defn download-file!
  "Download a file from `url` to `dest-path` using `wget`."
  [url dest-path]
  {:pre [(string? url) (string? dest-path) (str/starts-with? url "http")]}
  (steps/step (format "Download %s -> %s" url dest-path)
    (delete-file-if-exists! dest-path)
    (shell/sh {:quiet? true} "wget" "--quiet" "--no-cache" "--output-document" dest-path url)
    (assert-file-exists dest-path)))

(defn nio-path
  "Convert a String `path` to a `java.nio.file.Path`, for use with NIO methods."
  ^Path [^String path]
  (.getPath (FileSystems/getDefault) path (misc/varargs String)))

(defn file-size
  "Get the size, in bytes, of the file at `path`."
  ^Long [^String path]
  (Files/size (nio-path path)))

(defn temporary-file
  "Create a temporary file with prefix and suffix. Default to delete on exit."
  ([prefix suffix] (temporary-file prefix suffix true))
  ([prefix suffix delete-on-exit?]
   (let [file (File/createTempFile prefix suffix)]
     (when delete-on-exit?
       (.deleteOnExit file))
     file)))

(defn absolute?
  "Whether `file` is an absolute path."
  [file]
  (.isAbsolute (io/file file)))

(defn directory?
  "Whether `file` is a directory."
  [file]
  (.isDirectory (io/file file)))

(defn exists?
  "Whether `file` exists or not."
  [file]
  (.exists (io/file file)))

(defn zip-directory->file
  "Given a source directory and a destination zip file path,
   zip the directory and writes it to the destination"
  ([source-dir zip-file]
   (zip-directory->file source-dir zip-file {}))
  ([^String source-dir ^String zip-file {:keys [_verbose]}]
   (let [verbose true
         ^File source-path (File. source-dir)
         entry-count (atom 0)]
     (when-not (exists? source-path)
       (throw (ex-info "Directory to zip must exist!" {:source-path source-path})))
     (with-open [fos (FileOutputStream. ^String zip-file)
                 zos (ZipOutputStream. fos)]
       (doseq [^File file (file-seq source-path)
               :when (not (directory? file))
               :when
               (or (str/ends-with? (str/lower-case file) "yaml")
                   (str/ends-with? (str/lower-case file) "yml"))]
         (when verbose (out/safe-println "Zipping file:" file))
         (let [file-path (.getAbsolutePath file)
               buffer (byte-array 1024)
               fis (FileInputStream. file)]
           (swap! entry-count inc)
           (.putNextEntry zos (ZipEntry. file-path))
           (loop [len (.read fis buffer)]
             (when (pos? len)
               (.write zos buffer 0 len)
               (recur (.read fis buffer))))
           (.closeEntry zos))))
     (out/announce "%d Entries zipped to '%s'!" @entry-count zip-file))))

(defn do-with-open-jar-file-system
  "IMPL for `with-open-jar-file-system`."
  [path-to-jar f]
  (let [file (io/file path-to-jar)]
    (assert (.exists file)
            (format "Cannot find jar at path %s" path-to-jar))
    (with-open [filesystem (FileSystems/newFileSystem (URI. (str "jar:" (.toURI file)))
                                                      Collections/EMPTY_MAP)]
      (f filesystem))))

(defmacro with-open-jar-file-system
  "Execute `body` with an open NIO JAR filesystem for the JAR at `path-to-jar`."
  [[filesystem-binding path-to-jar & more] & body]
  (let [body (if (seq more)
               `((with-open-jar-file-system ~(vec more) ~@body))
               body)]
    `(do-with-open-jar-file-system ~path-to-jar (fn [~(vary-meta filesystem-binding assoc :tag `FileSystem)] ~@body))))

(defn get-path-in-filesystem
  "Get an NIO Path in a given NIO `FileSystem`."
  ^Path [^FileSystem filesystem ^String path-component & more-components]
  (.getPath filesystem path-component (misc/varargs String more-components)))