Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Make register-listeners! idempotent, mirroring the queue-declaration fix (#80887)

John Swansonbb2b9786f394
diff --git a/src/metabase/mq/listener.clj b/src/metabase/mq/listener.clj
index 1432e191658..1338433a0c9 100644
--- a/src/metabase/mq/listener.clj
+++ b/src/metabase/mq/listener.clj
@@ -27,19 +27,22 @@
[channel]
(get @*listeners* channel))
-(defn- register-listener!
- "Atomically registers a listener for the given channel.
-
- Throws if the queue has not been declared via [[q.registry/def-queue!]] — catching missing-queue
- typos at startup rather than at first publish. Also throws if a listener is already
- registered for the channel."
- [channel listener-map]
+(defn- assert-queue-declared!
+ "Throws if `channel` is a queue channel whose queue has not been declared via
+ [[q.registry/def-queue!]] — catching missing-queue typos at startup rather than at first publish."
+ [channel]
(when (and (= "queue" (namespace channel))
(nil? (q.registry/get-queue channel)))
(throw (ex-info (str "No queue declared for " channel
" — declare it with `def-queue!` before registering a listener.")
{:channel channel
- :known-queues (set (keys @q.registry/*queues*))})))
+ :known-queues (set (keys @q.registry/*queues*))}))))
+
+(defn- register-listener!
+ "Atomically registers a listener for the given channel. Throws if a listener is already
+ registered for the channel."
+ [channel listener-map]
+ (assert-queue-declared! channel)
(let [[old _] (swap-vals! *listeners*
(fn [m]
(if (contains? m channel)
@@ -49,6 +52,14 @@
(throw (ex-info (str "Listener already registered for " (namespace channel) " " (name channel))
{:channel channel})))))
+(defn register-declared-listener!
+ "Registers `listener-map` for `channel`, replacing any existing registration. An implementation
+ detail of [[def-listener!]] — public only so the macro expansion can call it."
+ [channel listener-map]
+ (assert-queue-declared! channel)
+ (swap! *listeners* assoc channel listener-map)
+ nil)
+
(defn batch-listen!
"Low-level listener registration. Prefer [[def-listener!]] for production code — the macro
is the supported way to wire a handler so it gets activated through `register-listeners!`
@@ -74,6 +85,39 @@
{:arglists '([channel])}
identity)
+(defonce ^{:doc "channel → symbol of the namespace whose `def-listener!` declaration owns it.
+ The load-time ownership ledger behind [[claim-listener-declaration!]]. `defonce` so reloading
+ this namespace doesn't orphan existing claims. Deliberately not test-rebound like [[*listeners*]]:
+ a claim is a fact about the source code, not about a running mq instance."}
+ listener-declaration-sites
+ (atom {}))
+
+(defn claim-listener-declaration!
+ "Records `ns-symb` as the declaration site that owns the listener for `channel`. An implementation
+ detail of [[def-listener!]] — like [[register-declared-listener!]], public only so the macro
+ expansion can call it.
+
+ Re-claiming from the same namespace is a no-op, so reloading a declaring namespace is always
+ legal. A *different* namespace claiming an already-claimed channel throws, at load time: each
+ declaration installs a [[def-listener*]] method for its channel, so duplicates would silently
+ clobber each other and which handler won would depend on load order.
+
+ If a declaration has genuinely moved between namespaces, evict the stale claim from the REPL
+ with `(swap! metabase.mq.listener/listener-declaration-sites dissoc <channel>)`, or restart."
+ [channel ns-symb]
+ (let [[old _] (swap-vals! listener-declaration-sites
+ (fn [m] (if (contains? m channel)
+ m
+ (assoc m channel ns-symb))))]
+ (when-let [existing (get old channel)]
+ (when (not= existing ns-symb)
+ ;; not i18n'ed because this is developer-facing only.
+ (throw (ex-info (format (str "A listener for %s is already declared in %s. A listener has exactly one "
+ "declaration site; if you've moved it, remove the old claim with "
+ "(swap! metabase.mq.listener/listener-declaration-sites dissoc %s) or restart.")
+ channel existing channel)
+ {:channel channel :existing-site existing :new-site ns-symb}))))))
+
(defmacro def-listener!
"Declares a listener for a queue.
@@ -90,15 +134,25 @@
(mq/def-queue! :queue/search-reindex {:transactional :require :exclusive true :max-batch-messages 50})
(mq/def-listener! :queue/search-reindex [messages]
- (process-batch messages))"
+ (process-batch messages))
+
+ A listener has exactly one declaration site: reloading the declaring namespace is fine (and
+ its latest handler wins on the next `register-listeners!`), but a second namespace declaring
+ a listener for the same channel throws at load time — see [[claim-listener-declaration!]]."
{:arglists '([channel bindings & body])}
[channel bindings & body]
- `(defmethod def-listener* ~channel [~'_]
- (batch-listen! ~channel (fn [~@bindings] ~@body))))
+ `(do
+ (claim-listener-declaration! ~channel '~(ns-name *ns*))
+ (defmethod def-listener* ~channel [~'_]
+ (register-declared-listener! ~channel {:listener (fn [~@bindings] ~@body)}))))
(defn register-listeners!
"Call all [[def-listener!]] implementations to register their listeners.
Called at startup and in test setup (from `with-test-mq`).
+ Idempotent: each run replaces any listeners already registered for the declared channels, so
+ a second `mq.init/start!` against the same root [[*listeners*]] atom (a REPL where a dev
+ server is already up, or a namespace refresh that re-runs test initialization) succeeds and
+ installs the latest declared handlers.
Throws on the first registration failure so broken listeners are caught early."
[]
(doseq [[k f] (methods def-listener*)]
diff --git a/test/metabase/mq/listener_test.clj b/test/metabase/mq/listener_test.clj
index 522aae9f2d0..56f37d1d657 100644
--- a/test/metabase/mq/listener_test.clj
+++ b/test/metabase/mq/listener_test.clj
@@ -53,6 +53,34 @@
(is (= ["m1"] @multi-received))
(is (pos? @multi-count) "the side-effecting first statement also ran"))))
+(deftest register-listeners!-is-idempotent-test
+ (testing "a second register-listeners! run replaces registrations instead of throwing"
+ ;; This is the REPL scenario: `mq.init/start!` runs once (dev-server startup, or a prior
+ ;; `test.initialize :mq`), then runs again against the same root `*listeners*` atom — e.g. a
+ ;; tools.namespace refresh unloads `metabase.test.initialize` (resetting its `defonce` of
+ ;; initialized steps) without reloading `metabase.mq.listener`. The second run must win
+ ;; quietly, like `register-queues!` does, rather than dying with "Listener already registered".
+ (binding [listener/*listeners* (atom {})
+ q.registry/*queues* (atom {})]
+ (q.registry/register-queues!)
+ (listener/register-listeners!)
+ (let [first-registration (:listener (listener/get-listener :queue/listener-macro-test))]
+ (is (fn? first-registration))
+ (listener/register-listeners!)
+ (let [second-registration (:listener (listener/get-listener :queue/listener-macro-test))]
+ (is (fn? second-registration))
+ (testing "the re-run replaced the registration (latest declaration wins)"
+ (is (not (identical? first-registration second-registration)))))))))
+
+(deftest claim-listener-declaration-test
+ (testing "re-claiming a channel from its own declaring namespace is a no-op (namespace reload)"
+ (is (nil? (listener/claim-listener-declaration! :queue/listener-macro-test
+ 'metabase.mq.listener-test))))
+ (testing "claiming an already-claimed channel from a different namespace throws at load time"
+ (is (thrown-with-msg? ExceptionInfo #"already declared"
+ (listener/claim-listener-declaration! :queue/listener-macro-test
+ 'metabase.some.other.ns)))))
+
(deftest listener-requires-declared-queue-test
(binding [listener/*listeners* (atom {})
q.registry/*queues* (atom {})]