Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Token scanner: skip image/archive/binary files (#75141)

bryan3831a896ca84
diff --git a/mage/src/mage/token_scan.clj b/mage/src/mage/token_scan.clj
index e24e09860e2..ac9bf6658ae 100644
--- a/mage/src/mage/token_scan.clj
+++ b/mage/src/mage/token_scan.clj
@@ -1,7 +1,6 @@
(ns mage.token-scan
(:require
[babashka.fs :as fs]
- [babashka.process :as p]
[clojure.java.io :as io]
[clojure.string :as str]
[mage.color :as c]
@@ -40,6 +39,17 @@
(defn- whitelisted? [line]
(boolean (some #(str/includes? line %) token-like-whitelist)))
+(def ^:private blacklisted-extensions
+ "File extensions to skip when scanning. These are binary/image/archive formats that
+ cannot contain meaningful plaintext tokens and only generate false positives or errors."
+ #{"png" "jpg" "jpeg" "gif" "bmp" ;; images (svg excluded: it's text and can hide tokens)
+ "zip" "tar" "gz" "7z" "rar" ;; archives
+ "exe" "dll" "so" "bin" ;; binaries
+ })
+
+(defn- blacklisted-file? [file-path]
+ (contains? blacklisted-extensions (str/lower-case (str (fs/extension file-path)))))
+
(defn- truncate-around-match
"Truncate text around a regex match, showing ~100 chars before and after"
[text pattern]
@@ -149,6 +159,7 @@
:babashka/exit 1})))]
(->> files
(filter fs/regular-file?)
+ (remove blacklisted-file?)
(map str)
distinct)))
@@ -164,7 +175,6 @@
:as full-info} (scan-files token-patterns files')
info (dissoc full-info :scanned :duration-ms)
duration-ms (/ (- (System/nanoTime) start-time) 1e6)]
-
;; Print results
(doseq [{:keys [file matches error]} scanned]
(when error
@@ -176,7 +186,6 @@
(when column-number (str ":" column-number))
(c/yellow "[" pattern-name "]")
(c/magenta (str/trim line-text)))))))
-
(println "Scan completed in: " (format "%.0f" duration-ms) "ms")
(when verbose
(println "--------------------")
@@ -185,7 +194,6 @@
files-with-matches))
(println "Total matches: " ((if (zero? total-matches) c/green c/yellow)
total-matches)))
-
(if (> total-matches 0)
(throw (ex-info nil
{:outcome info
diff --git a/mage/test/mage/token_scan_test.clj b/mage/test/mage/token_scan_test.clj
index a72bbafcf8e..6d2b95be99d 100644
--- a/mage/test/mage/token_scan_test.clj
+++ b/mage/test/mage/token_scan_test.clj
@@ -41,6 +41,18 @@
(finally
(fs/delete sample-file-path))))
+(deftest blacklisted-extensions-skipped-test
+ (let [png-path (str u/project-root-directory "/" "test-token-scan.png")]
+ (try
+ ;; Write a token-like string into a .png file; it must be skipped, not scanned.
+ (spit png-path token-like-comment)
+ (is (= {:total-files 0, :files-with-matches 0, :total-matches 0}
+ (-> (token-scan/run-scan {:arguments [png-path] :options {}})
+ (select-keys [:total-files :files-with-matches :total-matches])))
+ "blacklisted file extensions (e.g. .png) should be skipped entirely")
+ (finally
+ (fs/delete png-path)))))
+
(deftest missing-file-test
(try
(token-scan/run-scan {:arguments ["some-missing-file.txt"]})