The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Token scanner: skip image/archive/binary files (#75141)
bryan3831a896ca84
diff --git a/mage/src/mage/token_scan.clj b/mage/src/mage/token_scan.cljindex e24e09860e2..ac9bf6658ae 100644--- a/mage/src/mage/token_scan.clj+++ b/mage/src/mage/token_scan.clj@@ -1,7 +1,6 @@(ns mage.token-scan(:require[babashka.fs :as fs]- [babashka.process :as p][clojure.java.io :as io][clojure.string :as str][mage.color :as c]@@ -40,6 +39,17 @@(defn- whitelisted? [line](boolean (some #(str/includes? line %) token-like-whitelist)))+(def ^:private blacklisted-extensions+ "File extensions to skip when scanning. These are binary/image/archive formats that+ cannot contain meaningful plaintext tokens and only generate false positives or errors."+ #{"png" "jpg" "jpeg" "gif" "bmp" ;; images (svg excluded: it's text and can hide tokens)+ "zip" "tar" "gz" "7z" "rar" ;; archives+ "exe" "dll" "so" "bin" ;; binaries+ })++(defn- blacklisted-file? [file-path]+ (contains? blacklisted-extensions (str/lower-case (str (fs/extension file-path)))))+(defn- truncate-around-match"Truncate text around a regex match, showing ~100 chars before and after"[text pattern]@@ -149,6 +159,7 @@:babashka/exit 1})))](->> files(filter fs/regular-file?)+ (remove blacklisted-file?)(map str)distinct)))@@ -164,7 +175,6 @@:as full-info} (scan-files token-patterns files')info (dissoc full-info :scanned :duration-ms)duration-ms (/ (- (System/nanoTime) start-time) 1e6)]-;; Print results(doseq [{:keys [file matches error]} scanned](when error@@ -176,7 +186,6 @@(when column-number (str ":" column-number))(c/yellow "[" pattern-name "]")(c/magenta (str/trim line-text)))))))-(println "Scan completed in: " (format "%.0f" duration-ms) "ms")(when verbose(println "--------------------")@@ -185,7 +194,6 @@files-with-matches))(println "Total matches: " ((if (zero? total-matches) c/green c/yellow)total-matches)))-(if (> total-matches 0)(throw (ex-info nil{:outcome infodiff --git a/mage/test/mage/token_scan_test.clj b/mage/test/mage/token_scan_test.cljindex a72bbafcf8e..6d2b95be99d 100644--- a/mage/test/mage/token_scan_test.clj+++ b/mage/test/mage/token_scan_test.clj@@ -41,6 +41,18 @@(finally(fs/delete sample-file-path))))+(deftest blacklisted-extensions-skipped-test+ (let [png-path (str u/project-root-directory "/" "test-token-scan.png")]+ (try+ ;; Write a token-like string into a .png file; it must be skipped, not scanned.+ (spit png-path token-like-comment)+ (is (= {:total-files 0, :files-with-matches 0, :total-matches 0}+ (-> (token-scan/run-scan {:arguments [png-path] :options {}})+ (select-keys [:total-files :files-with-matches :total-matches])))+ "blacklisted file extensions (e.g. .png) should be skipped entirely")+ (finally+ (fs/delete png-path)))))+(deftest missing-file-test(try(token-scan/run-scan {:arguments ["some-missing-file.txt"]})