The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Add kondo ignore ratchets (#77773)
Chris Truter57e0ad66ea59
diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.ednindex 2bb588e578f..216b43e77e2 100644--- a/.clj-kondo/config.edn+++ b/.clj-kondo/config.edn@@ -179,6 +179,8 @@clojure.core.async/take!clojure.core.async/to-chan!clojure.core.async/to-chan!!+ ;; only writes .clj-kondo/ratchets.edn, from a :once fixture that finishes before the parallel tests+ dev.kondo-ratchet/fix!metabase.ai-tracing.core/record!metabase.api.macros.defendpoint.tools-manifest/assert-optional-fields-nullable!metabase.channel.core/send!diff --git a/.clj-kondo/ratchets.edn b/.clj-kondo/ratchets.ednnew file mode 100644index 00000000000..a538fad0015--- /dev/null+++ b/.clj-kondo/ratchets.edn@@ -0,0 +1,51 @@+;; Per-linter budgets for inline `:clj-kondo/ignore` forms.+;; metabase.core.kondo-ratchet-test fails when the budgets drift from the actual counts.+;; `./bin/mage fix-kondo-ratchets` lowers budgets to match the tree; local test runs do it+;; automatically. Raising a budget, or adding one for a new linter (`--seed`), is a hand edit+;; to defend in your PR.+;; :all is the vector-less ignore form, which suppresses every linter on the next form.+{:ignore-counts {:aliased-namespace-symbol 3+ :all 61+ :case-symbol-test 2+ :clojure-lsp/unused-public-var 9+ :consistent-alias 1+ :def-fn 1+ :deprecated-namespace 97+ :deprecated-var 91+ :discouraged-java-method 4+ :discouraged-namespace 81+ :discouraged-var 143+ :equals-true 28+ :invalid-arity 1+ :main-without-gen-class 1+ :metabase/defsetting-namespace 7+ :metabase/disallow-hardcoded-driver-names-in-tests 34+ :metabase/discourage-millis-duration 1+ :metabase/i-like-making-cams-eyes-bleed-with-horrifically-long-tests 19+ :metabase/modules 12+ :metabase/namespace-name 4+ :metabase/prefer-with-dynamic-fn-redefs 35+ :metabase/test-helpers-use-non-thread-safe-functions 40+ :metabase/validate-defendpoint-has-response-schema 462+ :metabase/validate-defendpoint-query-params-use-kebab-case 40+ :metabase/validate-defendpoint-route-uses-kebab-case 44+ :metabase/validate-deftest 19+ :missing-docstring 10+ :missing-protocol-method 4+ :non-arg-vec-return-type-hint 1+ :reduce-without-init 3+ :redundant-fn-wrapper 1+ :redundant-nested-call 1+ :syntax 2+ :unquote-not-syntax-quoted 1+ :unresolved-namespace 5+ :unresolved-require 1+ :unresolved-symbol 4+ :unresolved-var 1+ :unsorted-required-namespaces 1+ :unused-alias 1+ :unused-binding 9+ :unused-excluded-var 2+ :unused-namespace 1+ :unused-private-var 9+ :unused-value 1}}diff --git a/.clj-kondo/src/hooks/clojure/test.clj b/.clj-kondo/src/hooks/clojure/test.cljindex b925e6acdab..b94def3b1da 100644--- a/.clj-kondo/src/hooks/clojure/test.clj+++ b/.clj-kondo/src/hooks/clojure/test.clj@@ -153,6 +153,7 @@"build.""i18n." ; bin/i18n"lint-migrations-file-test"+ "mage.""main-test" ; bin/release-list"metabase.deps-edn-test""metabase.driver."diff --git a/.github/workflows/kondo-ratchets-self-heal.yml b/.github/workflows/kondo-ratchets-self-heal.ymlnew file mode 100644index 00000000000..b59ac10941d--- /dev/null+++ b/.github/workflows/kondo-ratchets-self-heal.yml@@ -0,0 +1,85 @@+name: Kondo Ratchets Self-Heal++# Stale budgets in .clj-kondo/ratchets.edn (above the actual ignore counts) fail CI. On PRs+# labelled kondo-ratchets-self-healing, this commits the lowered budgets to the branch;+# unlabelled PRs rely on local test runs to tighten the file.+#+# Two jobs so the automation token never shares an environment with PR-controlled code: the+# ratchet-down job runs ./bin/mage from the PR branch and only exports the fixed file as an+# artifact; the push job starts from a clean runner, executes nothing from the PR, and pushes.+permissions:+ contents: read++on:+ pull_request:+ types: [opened, synchronize, reopened, labeled, unlabeled]+ paths:+ - "**.clj"+ - "**.cljc"+ - "**.cljs"++concurrency:+ group: ${{ github.workflow }}-${{ github.event.pull_request.number }}+ cancel-in-progress: true++jobs:+ ratchet-down:+ runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}+ timeout-minutes: 10+ # fork PRs don't get the automation token, so we couldn't push to them anyway+ if: >-+ contains(github.event.pull_request.labels.*.name, 'kondo-ratchets-self-healing')+ && github.event.pull_request.head.repo.full_name == github.repository+ outputs:+ changed: ${{ steps.diff.outputs.changed }}+ steps:+ - uses: actions/checkout@v6+ with:+ ref: ${{ github.event.pull_request.head.ref }}+ persist-credentials: false+ - name: Lower ignore budgets to match the source tree+ run: ./bin/mage fix-kondo-ratchets+ - name: Check for changes+ id: diff+ run: |+ if git diff --quiet .clj-kondo/ratchets.edn; then+ echo "budgets already tight"+ echo "changed=false" >> "$GITHUB_OUTPUT"+ else+ echo "changed=true" >> "$GITHUB_OUTPUT"+ fi+ - uses: actions/upload-artifact@v7+ if: steps.diff.outputs.changed == 'true'+ with:+ name: ratchets-edn+ path: .clj-kondo/ratchets.edn+ # dot-directories are excluded by default, which would ship an empty artifact+ include-hidden-files: true++ push:+ needs: ratchet-down+ if: needs.ratchet-down.outputs.changed == 'true'+ runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}+ timeout-minutes: 5+ steps:+ - uses: actions/checkout@v6+ with:+ ref: ${{ github.event.pull_request.head.ref }}+ persist-credentials: false+ - uses: actions/download-artifact@v8+ with:+ name: ratchets-edn+ path: .clj-kondo/+ - name: Commit to the PR branch+ env:+ PR_BRANCH: ${{ github.event.pull_request.head.ref }}+ REPO: ${{ github.repository }}+ # METABASE_AUTOMATION_USER_TOKEN so the self-healing commit triggers CI+ # (a GITHUB_TOKEN push would not)+ AUTOMATION_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}+ run: |+ git config --global user.name "Metabase Automation"+ git config --global user.email "[email protected]"+ git add .clj-kondo/ratchets.edn+ git commit -m "Ratchet down kondo ignore budgets" --no-verify+ git push "https://x-access-token:${AUTOMATION_TOKEN}@github.com/${REPO}.git" "HEAD:$PR_BRANCH"diff --git a/.gitignore b/.gitignoreindex a61caada9ec..fa38fb324c4 100644--- a/.gitignore+++ b/.gitignore@@ -151,6 +151,7 @@ mise.local.toml**/.clj-kondo/*!**/.clj-kondo/README.md!**/.clj-kondo/config.edn+!**/.clj-kondo/ratchets.edn!**/.clj-kondo/config/!**/.clj-kondo/src/!**/.clj-kondo/test/diff --git a/CLAUDE.md b/CLAUDE.mdindex ca3924fc0a2..1e1fd8fd7c1 100644--- a/CLAUDE.md+++ b/CLAUDE.md@@ -77,6 +77,29 @@ It piggybacks on a running dev nREPL (~5s) and auto-spawns a JVM if none is runnthe four generated keys; structural changes it can't safely make (a new module needs a human `:team`, ormodules need reordering) are printed as `WARNING:` lines for you to resolve by hand.+## Kondo Ignore Ratchets++`.clj-kondo/ratchets.edn` records, per linter, how many inline `:clj-kondo/ignore` forms the backend source+tree may contain. `metabase.core.kondo-ratchet-test` fails when the budgets drift from the actual counts,+in either direction. Prefer fixing the underlying warning over adding an ignore.++Budget too high (you removed ignores): a local run of the test tightens the file for you — commit the+change. PRs labelled `kondo-ratchets-self-healing` get the lowered budgets committed to the branch by CI.+To tighten by hand (babashka, no JVM; a no-op prints `unchanged`):++```bash+./bin/mage fix-kondo-ratchets+```++Budget too low (you added an ignore): the task only raises a budget when told to. If the ignore is+genuinely required, run `./bin/mage fix-kondo-ratchets --seed :the-linter` and defend the increase in the+PR.++Introducing a new linter: `./bin/mage kondo-insert-ignores :the-linter` inserts an ignore at every site it+flags, then `./bin/mage fix-kondo-ratchets --seed :the-linter` records the budget — no big-bang cleanup.+To burn debt down, `./bin/mage kondo-redundant-ignores` lists ignores that are no longer needed (slow:+full kondo run).+## Tool PreferencesIf `clojure-mcp` tools are available, prefer them over shell-based alternatives for Clojure development.\ No newline at end of filediff --git a/bb.edn b/bb.ednindex b97953d6076..8b7845fb816 100644--- a/bb.edn+++ b/bb.edn@@ -1,7 +1,8 @@{:min-bb-version "1.12.212";; we put path as bin, and everything is in the ./mage subdirectory,;; so the namespaces are mage.cli, mage.format, etc.- :paths ["mage/src" "mage/test" "bin/lint-migrations-file/src"]+ ;; dev/src is here for tasks whose logic is shared with JVM tests (e.g. dev.kondo-ratchet)+ :paths ["mage/src" "mage/test" "bin/lint-migrations-file/src" "dev/src"];; use Git version of dep temporarily until a new release > 0.16.4 is cut:deps {dev.weavejester/cljfmt {:git/sha "075eae26e3c8dad433b88f05927bcd50e3571328":git/url "https://github.com/weavejester/cljfmt"} ;; run cljfmt from bb@@ -274,6 +275,30 @@:requires [[mage.modules]]:task (task! (mage.modules/cli-fix-config (cli/parse! (current-task))))}+ fix-kondo-ratchets+ {:doc "Lower the per-linter ignore budgets in .clj-kondo/ratchets.edn to match the source tree."+ :examples [["./bin/mage fix-kondo-ratchets" "Ratchet budgets down (never raises one; over-budget linters are only warned about)"]+ ["./bin/mage fix-kondo-ratchets --seed :some-linter" "Also set :some-linter's budget to its actual count, for landing a new linter"]]+ :options [["-s" "--seed LINTER" "Set LINTER's budget to the actual count (may add or raise it)"]]+ :requires [[dev.kondo-ratchet]]+ :task (task! (dev.kondo-ratchet/fix! (:options parsed)))}++ kondo-redundant-ignores+ {:doc "Report inline kondo ignores that are no longer necessary (slow: full kondo run)."+ :examples [["./bin/mage kondo-redundant-ignores" "Scan the backend tree for removable ignores"]+ ["./bin/mage kondo-redundant-ignores --fix" "Also remove them from source"]]+ :options [["-f" "--fix" "Remove the candidate ignores from source"]]+ :requires [[mage.kondo-ratchet]]+ :task (task! (mage.kondo-ratchet/redundant-ignores parsed))}++ kondo-insert-ignores+ {:doc "Insert inline ignores at every site a linter flags -- for landing a new linter without a big-bang fix."+ :examples [["./bin/mage kondo-insert-ignores :metabase/new-linter" "Insert ignores everywhere the linter fires, then seed its budget"]+ ["./bin/mage kondo-insert-ignores :equals-true test/metabase/pulse" "Only under the given paths"]]+ :arg-schema [:sequential [:string {:name "linter-then-paths"}]]+ :requires [[mage.kondo-ratchet]]+ :task (task! (mage.kondo-ratchet/insert-ignores (:arguments parsed)))}+ls{:doc "List all mage public and private tasks":examples [["./bin/mage ls" "List all tasks"]]diff --git a/dev/src/dev/kondo_ratchet.clj b/dev/src/dev/kondo_ratchet.cljnew file mode 100644index 00000000000..4cd372082da--- /dev/null+++ b/dev/src/dev/kondo_ratchet.clj@@ -0,0 +1,252 @@+(ns dev.kondo-ratchet+ "Ratchet on inline kondo ignore forms.++ Per-linter budgets live in `.clj-kondo/ratchets.edn`.+ `metabase.core.kondo-ratchet-test` fails when they drift from the tree;+ `./bin/mage fix-kondo-ratchets` lowers budgets, never raises them.+ Loaded by both the bb task and the JVM test, so keep it dependency-free."+ {:clj-kondo/config '{:linters {:discouraged-var {clojure.core/println {:level :off}}}}}+ (:require+ [clojure.edn :as edn]+ [clojure.java.io :as io]+ [clojure.string :as str]))++(set! *warn-on-reflection* true)++(def ratchets-file+ "The budgets file, relative to the repo root."+ ".clj-kondo/ratchets.edn")++(def ^:private source-roots+ ["src" "test" "enterprise" "modules/drivers" "dev" "bin" "mage"])++(def ^:private source-extensions+ [".clj" ".cljc" ".cljs"])++;; Concatenated so this file never contains a literal ignore marker.+(def ^:private ignore-marker+ (str ":clj-kondo" "/ignore"))++;; `#_{... [:some-linter]}`, `^{... [:some-linter]}`, and the prefix-less attr-map form+;; `(ns foo {... [:some-linter]})`; the vector may span lines. The lazy tail after the vector runs to the+;; map's own closing brace, so extra keys still count and removal spans the whole form; a nested-brace+;; value stops the match at the vector instead.+(def ^:private vector-form-re+ (re-pattern (str "(?:(?:#_|\\^)\\s*)?\\{\\s*" ignore-marker "\\s*\\[([^\\]]*)\\](?:[^{}]*?\\})?")))++;; Bare `#_kw` / `^kw` with no linter vector: suppresses every linter on the next form.+(def ^:private bare-form-re+ (re-pattern (str "(?:#_\\s*|\\^)" ignore-marker "(?![\\w./-])")))++(defn mask-strings-and-comments+ "`content` with string-literal and line-comment interiors replaced by spaces, newlines kept.+ Same length as the input, so offsets and line numbers carry over.+ Ignore forms inside strings (test fixtures) or commented-out code must not count."+ [content]+ (let [sb (StringBuilder. ^String content)+ n (count content)]+ (loop [i 0, state :code]+ (if (>= i n)+ (str sb)+ (let [c (.charAt sb i)]+ (case state+ :code (case c+ \" (recur (inc i) :string)+ \; (do (.setCharAt sb i \space)+ (recur (inc i) :comment))+ ;; char literal: never treat the next char as a delimiter+ \\ (recur (+ i 2) :code)+ (recur (inc i) :code))+ :string (case c+ \" (recur (inc i) :code)+ \\ (do (.setCharAt sb i \space)+ (when (< (inc i) n)+ (when-not (= (.charAt sb (inc i)) \newline)+ (.setCharAt sb (inc i) \space)))+ (recur (+ i 2) :string))+ \newline (recur (inc i) :string)+ (do (.setCharAt sb i \space)+ (recur (inc i) :string)))+ :comment (if (= c \newline)+ (recur (inc i) :code)+ (do (.setCharAt sb i \space)+ (recur (inc i) :comment)))))))))++(defn- linter-keywords+ [vector-contents]+ (map (comp keyword #(subs % 1))+ (re-seq #":[A-Za-z][A-Za-z0-9*+!?<>=._/-]*" vector-contents)))++(defn line-linters+ "Linter keywords suppressed by inline ignore forms on `line`.+ The bare vector-less form counts as `:all`.+ Like [[scan]], ignore forms inside string literals or line comments don't count."+ [line]+ (let [masked (mask-strings-and-comments line)]+ (concat (mapcat (comp linter-keywords second) (re-seq vector-form-re masked))+ (repeat (count (re-seq bare-form-re masked)) :all))))++(defn- offset->line+ "1-based line number of character offset `i` in `content`."+ [content i]+ (inc (count (filter #(= % \newline) (subs content 0 i)))))++(defn- matches-with-offsets+ "Like re-seq, but returns `{:start _, :end _, :linters [...]}` for each match of `re` in `masked`."+ [re masked bare?]+ (let [m (re-matcher re masked)]+ (loop [acc []]+ (if (.find m)+ (recur (conj acc {:start (.start m)+ :end (.end m)+ :linters (if bare? [:all] (vec (linter-keywords (.group m 1))))}))+ acc))))++(defn ignore-matches+ "Inline ignore matches in `content`, in file order:+ `{:start _, :end _, :line _, :linters [...]}` with character offsets and a 1-based line.+ Matches inside string literals or line comments are excluded."+ [content]+ (let [masked (mask-strings-and-comments content)]+ (->> (concat (matches-with-offsets vector-form-re masked false)+ (matches-with-offsets bare-form-re masked true))+ (sort-by :start)+ (map #(assoc % :line (offset->line masked (:start %)))))))++(defn scan+ "Occurrences of inline ignore forms under `roots` (relative to the repo root).+ Returns `{:file \"src/...\", :line 42, :linters [...]}` maps.+ Forms inside string literals or line comments don't count."+ ([]+ (scan source-roots))+ ([roots]+ (for [root roots+ ^java.io.File f (file-seq (io/file root))+ :when (and (.isFile f)+ (some #(str/ends-with? (.getPath f) %) source-extensions))+ :let [content (slurp f)]+ :when (str/includes? content ignore-marker)+ m (ignore-matches content)]+ {:file (.getPath f)+ :line (:line m)+ :linters (:linters m)})))++(defn actual-counts+ "Per-linter occurrence counts for `occurrences`, as returned by [[scan]]."+ [occurrences]+ (frequencies (mapcat :linters occurrences)))++(defn- sorted-by-str+ [kvs]+ (into (sorted-map-by #(compare (str %1) (str %2))) kvs))++(defn drift+ "Linters whose count in `occurrences` differs from its budget in `recorded` (absent = 0, either side).+ Returns `{linter {:recorded _, :actual _}}`, plus `:examples` (up to 5 `file:line`) when over budget."+ [recorded occurrences]+ (let [actual (actual-counts occurrences)]+ (sorted-by-str+ (for [linter (into (set (keys actual)) (keys recorded))+ :let [budget (get recorded linter 0)+ n (get actual linter 0)]+ :when (not= budget n)]+ [linter (cond-> {:recorded budget, :actual n}+ (> n budget)+ (assoc :examples (->> occurrences+ (filter #(some #{linter} (:linters %)))+ (map #(str (:file %) ":" (:line %)))+ (take 5)+ vec)))]))))++(defn read-ratchets+ "Parsed contents of [[ratchets-file]], with empty defaults when the file doesn't exist."+ []+ (merge {:ignore-counts {}}+ (when (.exists (io/file ratchets-file))+ (edn/read-string (slurp ratchets-file)))))++(def ^:private header+ (str ";; Per-linter budgets for inline `" ignore-marker "` forms.\n"+ ";; metabase.core.kondo-ratchet-test fails when the budgets drift from the actual counts.\n"+ ";; `./bin/mage fix-kondo-ratchets` lowers budgets to match the tree; local test runs do it\n"+ ";; automatically. Raising a budget, or adding one for a new linter (`--seed`), is a hand edit\n"+ ";; to defend in your PR.\n"+ ";; :all is the vector-less ignore form, which suppresses every linter on the next form.\n"))++(defn render+ "Text of the ratchets file for the `{:ignore-counts _}` map `ratchets`.+ Byte-stable: [[fix!]] idempotency and the file-hygiene test depend on it."+ [{:keys [ignore-counts]}]+ (let [counts-indent (apply str (repeat (count "{:ignore-counts {") \space))]+ (str header+ "{:ignore-counts "+ (if (empty? ignore-counts)+ "{}"+ (let [entries (sort-by (comp str first) ignore-counts)+ width (apply max (map (comp count str first) entries))]+ (str "{"+ (str/join (str "\n" counts-indent)+ (for [[linter n] entries]+ (format (str "%-" width "s %d") (str linter) n)))+ "}")))+ "}\n")))++(defn lowered-counts+ "`recorded` with each budget lowered to its actual count; entries with no ignores left are dropped.+ Linters in `seeded` get their budget set to the actual count outright — the explicit escape hatch for+ landing a new linter. Otherwise never raises a budget, never adds one."+ [recorded actual seeded]+ (into (sorted-by-str+ (for [linter seeded+ :when (pos? (get actual linter 0))]+ [linter (get actual linter)]))+ (keep (fn [[linter budget]]+ (let [n (get actual linter 0)]+ (cond+ (contains? (set seeded) linter) nil+ (zero? n) nil+ (< n budget) [linter n]+ :else [linter budget]))))+ recorded))++(defn change-report+ "The lines [[fix!]] prints: lowered/dropped/seeded budgets, plus warnings for anything over budget."+ [{:keys [ignore-counts]} occurrences seeded]+ (let [actual (actual-counts occurrences)]+ (concat+ (for [linter seeded+ :let [n (get actual linter 0)]]+ (if (pos? n)+ (format "seeded %s at %d" linter n)+ (format "WARNING: %s has no inline ignores -- nothing to seed" linter)))+ (for [[linter budget] (sort-by (comp str first) (apply dissoc ignore-counts seeded))+ :let [n (get actual linter 0)]+ :when (not= n budget)]+ (cond+ (zero? n) (format "dropped %s (no ignores left)" linter)+ (< n budget) (format "lowered %s %d -> %d" linter budget n)+ :else (format "WARNING: %s is over budget (%d recorded, %d actual) -- remove ignores, or accept them all with `--seed %s`"+ linter budget n linter)))+ (for [[linter n] (sort-by (comp str first) (apply dissoc actual (concat seeded (keys ignore-counts))))]+ (format "WARNING: %s has %d ignores but no budget entry -- seed one with `./bin/mage fix-kondo-ratchets --seed %s`"+ linter n linter)))))++(defn fix!+ "Rewrite [[ratchets-file]]: lower budgets and normalize formatting.+ `--seed LINTER` (`{:seed \"...\"}` here) sets that budget to the actual count, adding or raising it.+ Prints the [[change-report]], or `unchanged` on a no-op."+ ([]+ (fix! nil))+ ([{:keys [seed]}]+ (let [{:keys [ignore-counts] :as ratchets} (read-ratchets)+ occurrences (scan)+ seeded (if seed [(keyword (str/replace-first seed #"^:" ""))] [])+ actual (actual-counts occurrences)+ text (render {:ignore-counts (lowered-counts ignore-counts actual seeded)})+ file (io/file ratchets-file)+ old (when (.exists file) (slurp file))]+ (run! println (change-report ratchets occurrences seeded))+ (if (= old text)+ (println "unchanged")+ (do (spit file text)+ (println (str "wrote " ratchets-file)))))))diff --git a/mage/src/mage/kondo_ratchet.clj b/mage/src/mage/kondo_ratchet.cljnew file mode 100644index 00000000000..71ae9323d21--- /dev/null+++ b/mage/src/mage/kondo_ratchet.clj@@ -0,0 +1,198 @@+(ns mage.kondo-ratchet+ "Tooling around the inline ignore ratchets that needs a real kondo run (slow: a minute or two):+ report ignores kondo considers redundant, and bulk-insert ignores when landing a new linter."+ (:require+ [clojure.edn :as edn]+ [clojure.string :as str]+ [dev.kondo-ratchet :as kondo-ratchet]+ [mage.shell :as shell]))++(set! *warn-on-reflection* true)++(def ^:private lint-roots+ ["src" "test" "enterprise/backend" "modules/drivers" "dev" "bin" "mage"])++(defn- kondo-findings!+ "Kondo run over `roots`, optionally with `linter` forced to `:warning`; returns findings as EDN maps+ (all of them when `linter` is nil, else just that type)."+ [linter roots]+ (let [config (pr-str (cond-> {:output {:format :edn}}+ linter (assoc :linters {linter {:level :warning}})))+ {:keys [out]} (apply shell/sh* {:quiet? true}+ "clojure" "-M:kondo" "--config" config "--lint" roots)+ findings (:findings (edn/read-string (str/join "\n" out)))]+ (cond->> findings+ linter (filter #(= (:type %) linter)))))++(defn- insert-ignore-lines+ "`text` with an ignore inserted above each 1-based row of `row->linters`, at matching indentation."+ [text row->linters]+ (let [ending (if (str/ends-with? text "\n") "\n" "")]+ (str (str/join "\n"+ (reduce (fn [lines [row linters]]+ (let [indent (re-find #"^\s*" (nth lines (dec row)))]+ (into (conj (subvec lines 0 (dec row))+ (str indent "#_{:clj-kondo/ignore ["+ (str/join " " (sort-by str (distinct linters)))+ "]}"))+ (subvec lines (dec row)))))+ (vec (str/split-lines text))+ (sort-by key > row->linters)))+ ending)))++;;;; ---------------------------------------------------------------------------+;;;; kondo-redundant-ignores+;;;; ---------------------------------------------------------------------------++(defn- ignored-linters-at+ "Linter keywords named by the ignore form at `row` of `file`; reads a couple of extra lines since the+ vector may wrap."+ [file row]+ (let [lines (vec (str/split-lines (slurp file)))]+ (kondo-ratchet/line-linters+ (str/join "\n" (subvec lines (dec row) (min (count lines) (+ row 2)))))))++(defn- lsp-only?+ "Does the ignore form suppress only linters kondo doesn't run, so a re-lint could never restore it?"+ [linters]+ (boolean (and (seq linters)+ (every? #(= (namespace %) "clojure-lsp") linters))))++(defn- names-lsp?+ "Does the ignore form suppress any linter kondo doesn't run, so a re-lint could not fully restore it?"+ [linters]+ (boolean (some #(= (namespace %) "clojure-lsp") linters)))++(defn- remove-ignores-at+ "The inline ignore forms that start on the 1-based `rows` of `text`, removed.+ Forms that name any clojure-lsp/* linter survive ([[names-lsp?]]): the verify pass could never restore+ that half of the suppression. Forms whose matched span has unbalanced braces (nested maps the regex+ can't span) are skipped and reported under `:skipped` rather than corrupted.+ A line left whitespace-only by a removal is deleted; an inline removal also swallows one space.+ Returns `{:text _, :sites [{:row _, :linters _} ...], :skipped [rows...]}` — `:sites` carries the+ post-removal row of the form each ignore covered and the linters it named, so a verifying re-lint can+ tell exposed findings apart from unrelated pre-existing ones."+ [text rows]+ (let [rowset (set rows)+ masked (kondo-ratchet/mask-strings-and-comments text)+ balanced? (fn [{:keys [start end]}]+ (let [span (subs masked start end)]+ (= (count (filter #{\{} span))+ (count (filter #{\}} span)))))+ {removable true, unbalanced false}+ (group-by balanced?+ (->> (kondo-ratchet/ignore-matches text)+ (filter (comp rowset :line))+ (remove (comp names-lsp? :linters))))+ result (reduce (fn [{:keys [text] :as acc} {:keys [start end line linters]}]+ (let [before (subs text 0 start)+ after (subs text end)+ after (if (and (str/ends-with? before " ") (str/starts-with? after " "))+ (str/replace-first after #" +" "")+ after)+ line-start (inc (or (str/last-index-of before "\n") -1))+ line-end (or (str/index-of after "\n") (count after))+ remnant (str (subs before line-start) (subs after 0 line-end))+ blank? (str/blank? remnant)]+ (-> acc+ (assoc :text (if blank?+ (str (subs before 0 line-start)+ (subs after (min (count after) (inc line-end))))+ (str before after)))+ (update :deletions conj [line+ (if blank?+ (inc (count (filter #(= % \newline) (subs text start end))))+ 0)+ linters]))))+ {:text text, :deletions []}+ ;; bottom-up so earlier offsets stay valid+ (sort-by :start > removable))+ post-removal-row (fn [line]+ (- line (transduce (keep (fn [[l k]] (when (< l line) k))) + (:deletions result))))]+ {:text (:text result)+ :sites (for [[line _ linters] (:deletions result)]+ {:row (post-removal-row line)+ :linters linters})+ ;; adjusted like :sites, so warnings point at the rewritten file+ :skipped (map (comp post-removal-row :line) unbalanced)}))++(defn redundant-ignores+ "Report inline ignores kondo flags as redundant, dropping its two known false-positive classes:+ findings with no location, and ignores that only name linters kondo doesn't run (clojure-lsp/*).+ With `--fix`, remove the candidates from source."+ [parsed]+ (println "Running kondo with :redundant-ignore enabled (full lint, takes a minute or two)...")+ (let [findings (kondo-findings! :redundant-ignore lint-roots)+ {located true, homeless false} (group-by #(some? (:row %)) findings)+ {lsp-only true, candidates false}+ (group-by (fn [{:keys [filename row]}]+ (lsp-only? (ignored-linters-at filename row)))+ located)]+ (doseq [{:keys [filename row]} (sort-by (juxt :filename :row) candidates)]+ (println (format "%s:%d %s" filename row (str/join " " (ignored-linters-at filename row)))))+ (println)+ (println (format "%d candidate redundant ignores (dropped as false positives: %d location-less, %d lsp-only)"+ (count candidates) (count homeless) (count lsp-only)))+ (when (and (get-in parsed [:options :fix]) (seq candidates))+ (let [files (vec (sort (distinct (map :filename candidates))))+ sites (into {}+ (for [[file file-candidates] (group-by :filename candidates)]+ (let [{:keys [text sites skipped]} (remove-ignores-at (slurp file)+ (map :row file-candidates))]+ (spit file text)+ (doseq [row skipped]+ (println (format "WARNING: %s:%d skipped -- the ignore form's braces don't balance within the match; remove it by hand"+ file row)))+ [file sites])))+ near-sites (fn [{:keys [filename row]}]+ (filter #(<= (abs (- row (:row %))) 5) (sites filename)))+ ;; a finding near a removed site, of a type the removed ignore named, was exposed by the+ ;; removal; anything further away is a pre-existing finding in a file the usual+ ;; `mage kondo` roots don't cover+ exposed? (fn [{:keys [type] :as finding}]+ (boolean (some #(some #{:all type} (:linters %)) (near-sites finding))))]+ (println (format "Removed them across %d files; re-linting those files to verify..." (count files)))+ ;; redundant-ignore can't see findings from hook-based linters, so some ignores it called+ ;; redundant were still doing work; restore those.+ (let [{regressions true, others false} (group-by exposed? (filter :row (kondo-findings! nil files)))+ mismatched (filter (comp seq near-sites) others)]+ (doseq [[file file-regressions] (sort-by key (group-by :filename regressions))]+ (spit file (insert-ignore-lines (slurp file)+ (-> (group-by :row file-regressions)+ (update-vals #(map :type %))))))+ (doseq [{:keys [filename row type]} (sort-by (juxt :filename :row) mismatched)]+ (println (format "WARNING: %s:%d has a new %s finding near a removed ignore that named other linters -- fix or re-ignore by hand"+ filename row type)))+ (if (empty? regressions)+ (println "Verified: the removals exposed no findings.")+ (println (format "Restored ignores at %d sites where a finding reappeared (%s)."+ (count (distinct (map (juxt :filename :row) regressions)))+ (str/join ", " (sort (distinct (map :type regressions))))))))+ (println "Now run `./bin/mage fix-kondo-ratchets` to update the budgets, and `./bin/mage kondo`+for the final word (a finding deep inside a large ignored form can escape the nearby-site check).")))))++;;;; ---------------------------------------------------------------------------+;;;; kondo-insert-ignores+;;;; ---------------------------------------------------------------------------++(defn insert-ignores+ "Insert an inline ignore above every site `linter` flags, so a new linter can land without a big-bang+ fix. Args: `LINTER [PATHS...]`; paths default to the usual lint roots."+ [[linter-arg & paths]]+ (when (str/blank? (str linter-arg))+ (throw (ex-info "Usage: ./bin/mage kondo-insert-ignores LINTER [PATHS...]" {:exit-code 1})))+ (let [linter (keyword (str/replace-first linter-arg #"^:" ""))+ roots (or (seq paths) lint-roots)+ _ (println (format "Running kondo with %s enabled over %s..." linter (str/join " " roots)))+ findings (kondo-findings! linter roots)+ by-file (group-by :filename findings)]+ (doseq [[file fs] (sort-by key by-file)]+ (spit file (insert-ignore-lines (slurp file) (zipmap (map :row fs) (repeat [linter]))))+ (println (format "%s: %d inserted" file (count (distinct (map :row fs))))))+ (println)+ (if (empty? by-file)+ (println "No findings; nothing inserted.")+ (println (format "Inserted %d ignores across %d files. Now seed the budget:\n ./bin/mage fix-kondo-ratchets --seed %s"+ (count (distinct (map (juxt :filename :row) findings)))+ (count by-file)+ linter)))))diff --git a/mage/test/mage/core_test.clj b/mage/test/mage/core_test.cljindex 912d6a04223..937d0f912fd 100644--- a/mage/test/mage/core_test.clj+++ b/mage/test/mage/core_test.clj@@ -12,6 +12,7 @@[mage.bot.prompt-test][mage.doctor-test][mage.fix-unused-requires-test]+ [mage.kondo-ratchet-test][mage.merge-yaml-migrations-test :as merge-yaml-migrations-test][mage.modules-test][mage.token-scan-test]@@ -22,6 +23,7 @@;; Load test namespaces to ensure code coveragemage.doctor-test/keep-memage.fix-unused-requires-test/keep-me+ mage.kondo-ratchet-test/keep-memage.util-test/keep-memage.modules-test/keep-memerge-yaml-migrations-test/keep-mediff --git a/mage/test/mage/kondo_ratchet_test.clj b/mage/test/mage/kondo_ratchet_test.cljnew file mode 100644index 00000000000..f3a563d0897--- /dev/null+++ b/mage/test/mage/kondo_ratchet_test.clj@@ -0,0 +1,65 @@+(ns mage.kondo-ratchet-test+ (:require+ [clojure.test :refer [deftest is testing]]+ [mage.kondo-ratchet :as kondo-ratchet]))++;; Referenced by core_test.clj to ensure namespace is loaded+(def keep-me :loaded)++(deftest insert-ignore-lines-test+ (testing "inserts at the flagged line's indentation"+ (is (= "(defn f [x]\n #_{:clj-kondo/ignore [:equals-true]}\n (= true x))\n"+ (#'kondo-ratchet/insert-ignore-lines "(defn f [x]\n (= true x))\n" {2 [:equals-true]}))))+ (testing "multiple rows insert bottom-up so earlier rows stay valid; same-row linters share one form"+ (is (= (str "#_{:clj-kondo/ignore [:x]}\n"+ "(a)\n"+ "#_{:clj-kondo/ignore [:x :y]}\n"+ "(b)\n")+ (#'kondo-ratchet/insert-ignore-lines "(a)\n(b)\n" {1 [:x], 2 [:y :x :y]}))))+ (testing "a file without a trailing newline doesn't gain one"+ (is (= "#_{:clj-kondo/ignore [:x]}\n(a)"+ (#'kondo-ratchet/insert-ignore-lines "(a)" {1 [:x]})))))++(deftest remove-ignores-at-test+ (testing "a standalone ignore line disappears entirely; :sites points at the uncovered form"+ (is (= {:text "(defn f [x]\n (= true x))\n", :sites [{:row 2, :linters [:equals-true]}], :skipped []}+ (#'kondo-ratchet/remove-ignores-at+ "(defn f [x]\n #_{:clj-kondo/ignore [:equals-true]}\n (= true x))\n"+ [2]))))+ (testing "an inline ignore is cut out of its line, swallowing a doubled space"+ (is (= {:text "(do (foo))\n", :sites [{:row 1, :linters [:x]}], :skipped []}+ (#'kondo-ratchet/remove-ignores-at "(do #_{:clj-kondo/ignore [:x]} (foo))\n" [1]))))+ (testing "a multi-line ignore vector goes too"+ (is (= {:text "(a)\n(b)\n", :sites [{:row 2, :linters [:x :y]}], :skipped []}+ (#'kondo-ratchet/remove-ignores-at "(a)\n#_{:clj-kondo/ignore [:x\n :y]}\n(b)\n" [2]))))+ (testing "an ignore map with extra keys is removed whole, not truncated at the vector"+ (is (= {:text "(a)\n", :sites [{:row 1, :linters [:x]}], :skipped []}+ (#'kondo-ratchet/remove-ignores-at "#_{:clj-kondo/ignore [:x] :reason \"legacy\"}\n(a)\n" [1]))))+ (testing "an extra-key map with NESTED braces would be truncated by the regex, so it is skipped whole"+ (is (= {:text "#_{:clj-kondo/ignore [:x] :reason {:ticket \"ABC-1\"}}\n(a)\n", :sites [], :skipped [1]}+ (#'kondo-ratchet/remove-ignores-at+ "#_{:clj-kondo/ignore [:x] :reason {:ticket \"ABC-1\"}}\n(a)\n"+ [1]))))+ (testing "a skipped row is reported in post-removal coordinates when removals above it delete lines"+ (is (= {:text "(a)\n#_{:clj-kondo/ignore [:y] :reason {:nested 1}}\n(b)\n"+ :sites [{:row 1, :linters [:x]}]+ :skipped [2]}+ (#'kondo-ratchet/remove-ignores-at+ "#_{:clj-kondo/ignore [:x]}\n(a)\n#_{:clj-kondo/ignore [:y] :reason {:nested 1}}\n(b)\n"+ [1 3]))))+ (testing "any form naming a clojure-lsp/* linter survives — a re-lint could never restore that half"+ (is (= {:text (str "(do #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (foo))\n"+ "#_{:clj-kondo/ignore [:x :clojure-lsp/unused-public-var]}\n"+ "(bar)\n")+ :sites [{:row 1, :linters [:x]}]+ :skipped []}+ (#'kondo-ratchet/remove-ignores-at+ (str "(do #_{:clj-kondo/ignore [:x]} #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (foo))\n"+ "#_{:clj-kondo/ignore [:x :clojure-lsp/unused-public-var]}\n"+ "(bar)\n")+ [1 2]))))+ (testing "rows without an ignore are left alone; multiple removals shift later :sites rows up"+ (is (= {:text "(a)\n(b)\n", :sites [{:row 2, :linters [:y]} {:row 1, :linters [:x]}], :skipped []}+ (#'kondo-ratchet/remove-ignores-at+ "#_{:clj-kondo/ignore [:x]}\n(a)\n#_{:clj-kondo/ignore [:y]}\n(b)\n"+ [1 2 3])))))diff --git a/test/metabase/core/kondo_ratchet_test.clj b/test/metabase/core/kondo_ratchet_test.cljnew file mode 100644index 00000000000..cfd01409739--- /dev/null+++ b/test/metabase/core/kondo_ratchet_test.clj@@ -0,0 +1,175 @@+(ns metabase.core.kondo-ratchet-test+ "Ratchet on inline kondo ignore forms: budgets live in `.clj-kondo/ratchets.edn` and only move down.+ The ignore forms in this file are string fixtures — the scanner masks string literals, so they don't+ count as suppressions."+ (:require+ [clojure.edn :as edn]+ [clojure.java.io :as io]+ [clojure.string :as str]+ [clojure.test :refer :all]+ [dev.kondo-ratchet :as kondo-ratchet]))++(set! *warn-on-reflection* true)++;; Outside CI, tighten the ratchets before asserting — the fix rides along in your next commit.+;; The self-heal workflow does the same for labelled PRs.+(use-fixtures :once (fn [thunk]+ (when-not (System/getenv "CI")+ (kondo-ratchet/fix!))+ (thunk)))++;;;; ---------------------------------------------------------------------------+;;;; The ratchets themselves+;;;; ---------------------------------------------------------------------------++(deftest ^:parallel budgets-match-actual-counts-test+ (testing (str "\nBudgets in " kondo-ratchet/ratchets-file " must match the actual inline ignore counts.\n"+ "Budget too low: remove an ignore, or seed the budget with\n"+ "`./bin/mage fix-kondo-ratchets --seed <linter>` and defend it in the PR.\n"+ "Budget too high: run `./bin/mage fix-kondo-ratchets`, or label the PR\n"+ "kondo-ratchets-self-healing and CI commits the fix to your branch. Too high in a local\n"+ "run means `fix!` itself is broken, since the test fixture just ran it.")+ (is (= {}+ (kondo-ratchet/drift (:ignore-counts (kondo-ratchet/read-ratchets))+ (kondo-ratchet/scan))))))++(deftest ^:parallel ratchets-file-normalized-test+ (testing (str "\n" kondo-ratchet/ratchets-file " should be sorted and aligned exactly as the generator"+ " writes it.\nAfter a hand edit, run `./bin/mage fix-kondo-ratchets` to normalize the formatting.")+ (is (= (kondo-ratchet/render (kondo-ratchet/read-ratchets))+ (slurp kondo-ratchet/ratchets-file)))))++;;;; ---------------------------------------------------------------------------+;;;; Scanner unit tests+;;;; ---------------------------------------------------------------------------++(deftest ^:parallel mask-strings-and-comments-test+ (are [expected content] (= expected (kondo-ratchet/mask-strings-and-comments content))+ "(f \" \" x)" "(f \"a ; b\" x)"+ "(f) \n(g)" "(f) ; hey\n(g)"+ "(f \" \n \")" "(f \"a b\nc \")"+ ;; escaped quote stays inside the string; char literals never open one+ "(f \" \")" "(f \"a\\\"b c\")"+ "[\\\" \" \"]" "[\\\" \"abc\"]")+ (testing "masking preserves length and newline positions"+ (let [content "(f \"a\nb\") ; c\n(g)"+ masked (kondo-ratchet/mask-strings-and-comments content)]+ (is (= (count content) (count masked)))+ (is (= [5 13] (keep-indexed #(when (= %2 \newline) %1) masked))))))++(deftest ^:parallel line-linters-test+ (are [expected line] (= expected (kondo-ratchet/line-linters line))+ [:discouraged-var] " #_{:clj-kondo/ignore [:discouraged-var]}"+ [:a :b] "#_{:clj-kondo/ignore [:a :b]}"+ [:a :b :c] "#_{:clj-kondo/ignore [:a :b]} x #_{:clj-kondo/ignore [:c]}"+ [:metabase/modules] " ^{:clj-kondo/ignore [:metabase/modules]}"+ [:deprecated-var] "#_ {:clj-kondo/ignore [:deprecated-var]} (old-fn)"+ [:attr-map] "(ns b {:clj-kondo/ignore [:attr-map]})"+ [:extra] "#_{:clj-kondo/ignore [:extra] :reason 1}"+ ;; vector-less forms suppress everything -> :all+ [:all] " #_:clj-kondo/ignore"+ [:all] " #_ :clj-kondo/ignore (foo)"+ [:all] " ^:clj-kondo/ignore (foo)"+ ;; lookalikes that must NOT count+ [] "#_:clj-kondo/ignore-my-advice"+ [] "(defn foo [x] (inc x))"+ ;; ignore forms inside strings and comments don't count either+ [] "(def s \"#_{:clj-kondo/ignore [:in-a-string]}\")"+ [] ";; #_{:clj-kondo/ignore [:commented-out]}"))++(deftest ^:parallel scan-test+ (let [dir (.toFile (java.nio.file.Files/createTempDirectory+ "kondo-ratchet-test"+ (make-array java.nio.file.attribute.FileAttribute 0)))]+ (spit (io/file dir "a.clj")+ (str "(ns a)\n"+ "#_{:clj-kondo/ignore [:x :y]}\n"+ "(defn f [] 1)\n"+ ";; g is only used from the REPL\n"+ "#_:clj-kondo/ignore\n"+ "(defn g [] 2)\n"+ "(def h \"docstring with #_{:clj-kondo/ignore [:in-a-string]}\" 3)\n"+ "#_{:clj-kondo/ignore [:multi\n"+ " :line]}\n"+ "(defn i [] 4) #_{:clj-kondo/ignore [:trailing]} ;; trailing needs suppressing here\n"+ ";; #_{:clj-kondo/ignore [:commented-out]}\n"+ "(defn j [] 5)\n"))+ (spit (io/file dir "b.clj")+ (str "(ns b {:clj-kondo/ignore [:attr-map]})\n"+ "#_{:clj-kondo/ignore [:extra] :reason \"legacy\"}\n"+ "(defn k [] 6)\n"))+ (let [occurrences (sort-by (juxt :file :line) (kondo-ratchet/scan [(.getPath dir)]))]+ (is (= [{:file (.getPath (io/file dir "a.clj")), :line 2, :linters [:x :y]}+ {:file (.getPath (io/file dir "a.clj")), :line 5, :linters [:all]}+ {:file (.getPath (io/file dir "a.clj")), :line 8, :linters [:multi :line]}+ {:file (.getPath (io/file dir "a.clj")), :line 10, :linters [:trailing]}+ {:file (.getPath (io/file dir "b.clj")), :line 1, :linters [:attr-map]}+ {:file (.getPath (io/file dir "b.clj")), :line 2, :linters [:extra]}]+ occurrences)+ "strings and commented-out forms don't count; multi-line vectors, attr-maps, and extra keys do")+ (is (= {:x 1, :y 1, :all 1, :multi 1, :line 1, :trailing 1, :attr-map 1, :extra 1}+ (kondo-ratchet/actual-counts occurrences))))))++;;;; ---------------------------------------------------------------------------+;;;; Budget bookkeeping unit tests+;;;; ---------------------------------------------------------------------------++(deftest ^:parallel render-test+ (testing "keys come out sorted, values aligned, and the text round-trips losslessly"+ (let [ratchets {:ignore-counts {:discouraged-var 3, :all 1, :metabase/modules 2}}+ text (kondo-ratchet/render ratchets)]+ (is (str/ends-with? text (str "{:ignore-counts {:all 1\n"+ " :discouraged-var 3\n"+ " :metabase/modules 2}}\n")))+ (is (= ratchets (edn/read-string text)))+ (is (= text (kondo-ratchet/render (edn/read-string text))))))+ (testing "empty ratchets"+ (is (str/ends-with? (kondo-ratchet/render {:ignore-counts {}})+ "{:ignore-counts {}}\n"))))++(deftest ^:parallel lowered-counts-test+ (is (= {:lower 3, :over-budget 5}+ (kondo-ratchet/lowered-counts {:lower 5, :over-budget 5, :gone 5}+ {:lower 3, :over-budget 7, :new-linter 9}+ []))+ "budgets only ever move down: :lower shrinks to actual, :over-budget stays (the test's business),+ :gone is dropped, :new-linter is not added")+ (testing "seeding is the escape hatch: sets the budget to actual, adding or raising"+ (is (= {:new-linter 9, :raised 7}+ (kondo-ratchet/lowered-counts {:raised 5}+ {:new-linter 9, :raised 7}+ [:new-linter :raised])))+ (is (= {}+ (kondo-ratchet/lowered-counts {} {} [:nothing-to-seed]))+ "seeding a linter with no ignores adds nothing")))++(deftest ^:parallel drift-test+ (let [occurrences [{:file "f.clj", :line 1, :linters [:a]}+ {:file "f.clj", :line 2, :linters [:a :b]}+ {:file "g.clj", :line 3, :linters [:c]}]]+ (is (= {:a {:recorded 1, :actual 2, :examples ["f.clj:1" "f.clj:2"]}+ :b {:recorded 0, :actual 1, :examples ["f.clj:2"]}+ :c {:recorded 9, :actual 1}+ :gone {:recorded 3, :actual 0}}+ (kondo-ratchet/drift {:a 1, :c 9, :gone 3} occurrences))+ ":a and :b are over budget (with examples); :c and :gone are stale (without)"))+ (testing "a matching budget doesn't appear"+ (is (= {} (kondo-ratchet/drift {:a 1} [{:file "f.clj", :line 1, :linters [:a]}]))))+ (testing "examples are capped at 5"+ (let [occurrences (for [line (range 1 10)]+ {:file "f.clj", :line line, :linters [:a]})]+ (is (= 5 (count (:examples (:a (kondo-ratchet/drift {} occurrences)))))))))++(deftest ^:parallel change-report-test+ (let [occurrences (for [[linter n] {:lower 3, :over 7, :new 9, :same 4}+ i (range n)]+ {:file "f.clj", :line (inc i), :linters [linter]})]+ (is (= ["seeded :new at 9"+ "WARNING: :void has no inline ignores -- nothing to seed"+ "dropped :gone (no ignores left)"+ "lowered :lower 5 -> 3"+ "WARNING: :over is over budget (5 recorded, 7 actual) -- remove ignores, or accept them all with `--seed :over`"]+ (kondo-ratchet/change-report {:ignore-counts {:lower 5, :over 5, :gone 5, :same 4}}+ occurrences+ [:new :void]))+ "an untouched budget (:same) earns no line")))