The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Lint snowplow schemas in CI for additional smells (#73369)
Chris Truterda49ba56e601
diff --git a/.github/file-paths.yaml b/.github/file-paths.yamlindex c9df747803e..d53e9d23cd5 100644--- a/.github/file-paths.yaml+++ b/.github/file-paths.yaml@@ -156,6 +156,8 @@ embedding_sdk_host_sample_apps:snowplow: &snowplow- ".github/workflows/snowplow.yml"- "snowplow/**"+ - "bin/lint-snowplow-schemas.bb"+ - "bin/lint-snowplow-schemas-test.bb"e2e_all:- *defaultdiff --git a/.github/workflows/snowplow.yml b/.github/workflows/snowplow.ymlindex 5a76f9db78e..b91bc72dbaa 100644--- a/.github/workflows/snowplow.yml+++ b/.github/workflows/snowplow.yml@@ -28,5 +28,48 @@ jobs:steps:- name: Checkoutuses: actions/checkout@v4- - name: Lint- run: docker run -v $(pwd):/snowplow snowplow/igluctl:0.6.0 lint snowplow+ - name: Set up Java+ uses: actions/setup-java@v4+ with:+ distribution: temurin+ java-version: 21+ - name: Set up Babashka+ uses: turtlequeue/setup-babashka@2d4df498c7a578b4f3e906283f9af913590bdec7 # v1.7.0+ with:+ babashka-version: 1.12.197+ - name: Install igluctl+ run: |+ set -euo pipefail+ curl -fsSL -o "$RUNNER_TEMP/igluctl.zip" https://github.com/snowplow/igluctl/releases/download/0.13.0/igluctl_0.13.0.zip+ echo "0eac19121fca1431dec44738ada8eacc4242811a95d015ee71e00deec2b4d748 $RUNNER_TEMP/igluctl.zip" | sha256sum -c -+ unzip -q "$RUNNER_TEMP/igluctl.zip" -d "$RUNNER_TEMP/igluctl"+ chmod +x "$RUNNER_TEMP/igluctl/igluctl"+ echo "$RUNNER_TEMP/igluctl" >> "$GITHUB_PATH"+ - name: Lint (igluctl)+ # igluctl recursively parses every file as JSON, so it chokes on colocated+ # *.md docs and on two schemas whose internal `self.version` doesn't match+ # the filename version (pending Snowcat audit). Stage a filtered copy and+ # patch only the `self.version` mismatch so igluctl still lints the rest+ # of those schemas.+ run: |+ set -euo pipefail+ LINT_ROOT="$RUNNER_TEMP/iglu-lint"+ rm -rf "$LINT_ROOT"+ mkdir -p "$LINT_ROOT"+ cp -r snowplow/iglu-client-embedded/schemas "$LINT_ROOT/"+ find "$LINT_ROOT" -name '*.md' -delete+ # TODO(BOT-1403): drop these patches once the Snowcat audit fixes the+ # internal `self.version` mismatch on these two schemas. We rewrite+ # only `self.version` in the staged copy so the file's path version+ # matches what igluctl expects; every other lint rule still runs.+ # Each `sed` invocation uses the schema's wrong/right pair, so it+ # silently no-ops once the underlying schema is fixed.+ sed -i 's/"version": "1-0-1"/"version": "1-0-2"/' \+ "$LINT_ROOT/schemas/com.metabase/embed_share/jsonschema/1-0-2"+ sed -i 's/"version": "1-0-0"/"version": "1-0-1"/' \+ "$LINT_ROOT/schemas/com.metabase/serialization/jsonschema/1-0-1"+ igluctl lint "$LINT_ROOT/schemas"+ - name: Lint (Metabase schema rules)+ run: ./bin/lint-snowplow-schemas.bb+ - name: Test linter rules+ run: ./bin/lint-snowplow-schemas-test.bbdiff --git a/bin/lint-snowplow-schemas-test.bb b/bin/lint-snowplow-schemas-test.bbnew file mode 100755index 00000000000..3b37c14e2e7--- /dev/null+++ b/bin/lint-snowplow-schemas-test.bb@@ -0,0 +1,111 @@+#!/usr/bin/env bb++(ns lint-snowplow-schemas-test+ "Unit tests for `bin/lint-snowplow-schemas.bb`. Run with `./bin/lint-snowplow-schemas-test.bb`."+ (:require+ [babashka.fs :as fs]+ [clojure.test :as t :refer [deftest is testing]]))++;; Load the script under test as a sibling file. The script's `(when (= *file* (System/getProperty+;; "babashka.file")) ...)` guard prevents `-main` from running on load.+(load-file (str (fs/parent (fs/absolutize *file*)) "/lint-snowplow-schemas.bb"))+(require '[lint-snowplow-schemas :as l])++(deftest nullable-type?-test+ (testing "scalar 'null' is nullable"+ (is (l/nullable-type? "null")))+ (testing "vector containing 'null' is nullable"+ (is (l/nullable-type? ["string" "null"])))+ (testing "scalar non-null is not nullable"+ (is (not (l/nullable-type? "string"))))+ (testing "vector without 'null' is not nullable"+ (is (not (l/nullable-type? ["string" "integer"]))))+ (testing "missing/empty type is not nullable"+ (is (not (l/nullable-type? nil)))+ (is (not (l/nullable-type? [])))))++(deftest object-like?-test+ (testing "explicit object type"+ (is (l/object-like? {:type "object"})))+ (testing "vector type containing 'object'"+ (is (l/object-like? {:type ["object" "null"]})))+ (testing "properties present without explicit type"+ (is (l/object-like? {:properties {:x {:type "string"}}})))+ (testing "scalar non-object schema"+ (is (not (l/object-like? {:type "string"}))))+ (testing "empty schema is not object-like"+ (is (not (l/object-like? {})))))++(deftest check-schema-test+ (testing "scalar nullable required property is flagged"+ (is (= [{:path "/p"+ :pointer "$.properties.x"+ :msg "required property 'x' has nullable type \"null\""}]+ (l/check-schema "/p" {:type "object"+ :additionalProperties false+ :required ["x"]+ :properties {:x {:type "null"}}}))))+ (testing "vector-with-null required property is flagged"+ (is (= [{:path "/p"+ :pointer "$.properties.x"+ :msg "required property 'x' has nullable type [\"string\" \"null\"]"}]+ (l/check-schema "/p" {:type "object"+ :additionalProperties false+ :required ["x"]+ :properties {:x {:type ["string" "null"]}}}))))+ (testing "non-required nullable property is allowed"+ (is (empty? (l/check-schema "/p" {:type "object"+ :additionalProperties false+ :required []+ :properties {:x {:type ["string" "null"]}}}))))+ (testing "missing additionalProperties on type:object is flagged"+ (is (= [{:path "/p"+ :pointer "$"+ :msg "object schema must explicitly set 'additionalProperties' (true or false)"}]+ (l/check-schema "/p" {:type "object" :properties {}}))))+ (testing "missing additionalProperties on type [object null] is flagged"+ (is (= [{:path "/p"+ :pointer "$"+ :msg "object schema must explicitly set 'additionalProperties' (true or false)"}]+ (l/check-schema "/p" {:type ["object" "null"] :properties {}}))))+ (testing "missing additionalProperties on schema with :properties but no :type is flagged"+ (is (= [{:path "/p"+ :pointer "$"+ :msg "object schema must explicitly set 'additionalProperties' (true or false)"}]+ (l/check-schema "/p" {:properties {:x {:type "string"}}}))))+ (testing "additionalProperties: false (or true) satisfies the rule"+ (is (empty? (l/check-schema "/p" {:type "object" :additionalProperties false :properties {}})))+ (is (empty? (l/check-schema "/p" {:type "object" :additionalProperties true :properties {}}))))+ (testing "non-object schema doesn't trigger the additionalProperties rule"+ (is (empty? (l/check-schema "/p" {:type "string"}))))+ (testing "non-map property values (e.g. boolean schemas) don't crash the check"+ (is (= [] (l/check-schema "/p" {:type "object"+ :additionalProperties false+ :required ["x"]+ :properties {:x true}})))))++(deftest classify-violations-test+ (let [v1 {:path "/a" :pointer "$" :msg "x"}+ v2 {:path "/b" :pointer "$.properties.y" :msg "y"}+ baseline #{{:path "/a" :pointer "$"}+ {:path "/c" :pointer "$"}}]+ (testing "matching entries are grandfathered"+ (let [{:keys [new-problems grandfathered]} (l/classify-violations [v1] baseline)]+ (is (empty? new-problems))+ (is (= [v1] grandfathered))))+ (testing "non-matching entries are new problems"+ (let [{:keys [new-problems grandfathered]} (l/classify-violations [v2] baseline)]+ (is (= [v2] new-problems))+ (is (empty? grandfathered))))+ (testing "baseline entries with no matching violation are stale"+ (let [{:keys [stale]} (l/classify-violations [v1] baseline)]+ (is (= [{:path "/c" :pointer "$"}] stale))))+ (testing ":fatal violations bypass the baseline even if path/pointer matches"+ (let [fatal {:path "/a" :pointer "$" :fatal true :msg "unparseable"}+ {:keys [new-problems grandfathered]} (l/classify-violations [fatal] baseline)]+ (is (= [fatal] new-problems))+ (is (empty? grandfathered))))))++(when (= *file* (System/getProperty "babashka.file"))+ (let [{:keys [fail error]} (t/run-tests 'lint-snowplow-schemas-test)]+ (System/exit (if (zero? (+ fail error)) 0 1))))diff --git a/bin/lint-snowplow-schemas.bb b/bin/lint-snowplow-schemas.bbnew file mode 100755index 00000000000..d235ae9639b--- /dev/null+++ b/bin/lint-snowplow-schemas.bb@@ -0,0 +1,119 @@+#!/usr/bin/env bb++(ns lint-snowplow-schemas+ "Lint snowplow JSON schemas for two rules igluctl 0.6.0 misses: required properties whose type allows+ `null`, and object schemas with no explicit `additionalProperties`.++ Checks the root schema only — nested object schemas under `properties`, `oneOf`, etc. aren't recursed.+ Use Spectral if you need full coverage; this is a tight gate for the two pitfalls we've actually hit.++ Pre-existing violations are baselined (see `baseline`) so the gate enforces new schemas without forcing+ in-place edits to deployed ones."+ (:require+ [babashka.fs :as fs]+ [cheshire.core :as json]))++(def schemas-root "snowplow/iglu-client-embedded/schemas")++;; Iglu schema files match `<vendor>/<name>/jsonschema/<MAJOR>-<MINOR>-<PATCH>` — filter strictly so+;; stray docs (*.md), editor backups, or .DS_Store can't crash the linter.+(def iglu-path-re #".*/[^/]+/[^/]+/jsonschema/\d+-\d+-\d+$")++;; Known violations grandfathered in. Each entry is {:path :pointer}, so unrelated regressions in+;; the same file still fail. Drop an entry once its schema is bumped to a new version.+(def baseline+ #{{:path "snowplow/iglu-client-embedded/schemas/com.metabase/action/jsonschema/1-0-0"+ :pointer "$.properties.action_id"}+ {:path "snowplow/iglu-client-embedded/schemas/com.metabase/csvupload/jsonschema/1-0-3"+ :pointer "$.properties.generated_columns"}+ {:path "snowplow/iglu-client-embedded/schemas/com.metabase/model/jsonschema/1-0-0"+ :pointer "$.properties.model_id"}+ {:path "snowplow/iglu-client-embedded/schemas/com.metabase/embedded_analytics_js/jsonschema/1-0-0"+ :pointer "$"}+ {:path "snowplow/iglu-client-embedded/schemas/com.metabase/serialization/jsonschema/1-0-0"+ :pointer "$"}+ {:path "snowplow/iglu-client-embedded/schemas/com.metabase/serialization/jsonschema/1-0-1"+ :pointer "$"}})++(defn schema-files []+ (->> (fs/glob schemas-root "**")+ (filter fs/regular-file?)+ (map str)+ (filter #(re-matches iglu-path-re %))+ sort))++(defn has-type? [schema kind]+ (let [t (:type schema)]+ (or (= t kind)+ (and (sequential? t) (some #{kind} t)))))++(defn object-like?+ "True for schemas that describe an object: explicit `type: object`, vector type+ containing `\"object\"` (e.g. `[\"object\",\"null\"]`), or `properties` present+ even without an explicit type."+ [schema]+ (or (has-type? schema "object")+ (contains? schema :properties)))++(defn nullable-type? [t]+ (or (= t "null")+ (and (sequential? t) (some #{"null"} t))))++(defn check-schema [path schema]+ (let [missing-additional (when (and (object-like? schema)+ (not (contains? schema :additionalProperties)))+ [{:pointer "$"+ :msg "object schema must explicitly set 'additionalProperties' (true or false)"}])+ required (set (:required schema))+ ;; JSON Schema permits boolean property schemas (e.g. `"foo": true`); skip non-map+ ;; values rather than letting the destructure throw and surface a misleading+ ;; "unparseable" error for the whole file.+ nullable-required (for [[prop prop-schema] (:properties schema)+ :when (map? prop-schema)+ :let [t (:type prop-schema)]+ :when (and (contains? required (name prop))+ (nullable-type? t))]+ {:pointer (str "$.properties." (name prop))+ :msg (format "required property '%s' has nullable type %s"+ (name prop) (pr-str t))})]+ (mapv #(assoc % :path path) (concat missing-additional nullable-required))))++(defn classify-violations+ "Partition `violations` against `baseline` (set of `{:path :pointer}`) into+ `:new-problems` (must fail), `:grandfathered` (matched by baseline) and+ `:stale` (baseline entries with no matching violation, must also fail so the+ baseline shrinks as schemas get fixed). `:fatal` violations bypass baselining."+ [violations baseline]+ (let [baselined? #(and (not (:fatal %))+ (contains? baseline (select-keys % [:path :pointer])))+ {grandfathered true new-problems false} (group-by baselined? violations)+ violation-keys (into #{} (map #(select-keys % [:path :pointer])) violations)+ stale (remove #(contains? violation-keys %) baseline)]+ {:new-problems (vec new-problems)+ :grandfathered (vec grandfathered)+ :stale (vec stale)}))++(defn -main [& _]+ (let [files (schema-files)+ all (mapcat (fn [path]+ (try+ (check-schema path (json/parse-string (slurp path) true))+ (catch Exception e+ ;; :fatal forces the entry past the baseline — a parse failure+ ;; should never be silently grandfathered.+ [{:path path :pointer "$" :fatal true+ :msg (str "unparseable: " (ex-message e))}])))+ files)+ {:keys [new-problems grandfathered stale]} (classify-violations all baseline)]+ (doseq [{:keys [path pointer msg]} new-problems]+ (println (format "%s %s %s" path pointer msg)))+ (doseq [{:keys [path pointer]} stale]+ (println (format "stale baseline entry — no longer violating: %s %s" path pointer))+ (println " ↳ remove this entry from `baseline` in bin/lint-snowplow-schemas.bb"))+ (println)+ (println (format "Linted %d schemas, %d new problem(s), %d grandfathered, %d stale baseline entries"+ (count files) (count new-problems) (count grandfathered) (count stale)))+ (System/exit (if (or (seq new-problems) (seq stale)) 1 0))))++(when (= *file* (System/getProperty "babashka.file"))+ (apply -main *command-line-args*))