Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(java): the Router's load-and-evict lifecycle

Adeshina Lasisif86dc03f8c99
diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml
index f054bdcb..c3d34467 100644
--- a/.github/workflows/java.yml
+++ b/.github/workflows/java.yml
@@ -76,7 +76,7 @@ jobs:
if: ${{ !cancelled() }}
run: >
python laya-java/scripts/check_test_results.py
- laya-java/laya-java/build/test-results/test --min-tests 220 --allow-aborted
+ laya-java/laya-java/build/test-results/test --min-tests 390 --allow-aborted
# LAST, deliberately. This gate and the tests are independent signals -- the tests compare
# against the COMMITTED fixture, this compares the committed fixture against a fresh
@@ -178,7 +178,7 @@ jobs:
if: ${{ !cancelled() }}
run: >
python laya-java/scripts/check_test_results.py
- laya-java/laya-java/build/test-results/test --min-tests 370
+ laya-java/laya-java/build/test-results/test --min-tests 540
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
if: ${{ failure() }}
diff --git a/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/Router.java b/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/Router.java
index be4c4cb9..bf96792a 100644
--- a/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/Router.java
+++ b/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/Router.java
@@ -2,6 +2,9 @@ package com.convaiinnovations.laya;
import com.convaiinnovations.laya.json.PythonJson;
import com.convaiinnovations.laya.lang.LanguageDetection;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -12,6 +15,8 @@ import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.locks.ReentrantLock;
/**
* Decides which laya checkpoint a request should go to. A port of {@code laya.router}.
@@ -34,7 +39,7 @@ import java.util.TreeSet;
* <p>This class decides; it does not load. {@link #route} runs no model and touches no disk, so it
* is safe to call on every request and to test without a checkpoint.
*/
-public final class Router {
+public final class Router implements AutoCloseable {
/** The hub repository that bundles all three checkpoints. */
public static final String BUNDLE_REPO = "convaiinnovations/laya";
@@ -139,6 +144,116 @@ public final class Router {
}
}
+ /**
+ * Builds the agent for a checkpoint.
+ *
+ * <p>The reference downloads from the hub. This port takes local paths, so the mapping from a
+ * checkpoint to an {@link Agent} belongs to the caller -- see
+ * {@link Builder#checkpointsRoot(Path)} for the common layout.
+ */
+ @FunctionalInterface
+ public interface AgentFactory {
+
+ /** Build the agent for {@code checkpoint}. */
+ Agent create(Checkpoint checkpoint) throws IOException;
+ }
+
+ /**
+ * A borrowed agent, held open for as long as the lease is.
+ *
+ * <p>This exists because of a difference between the languages, not as ceremony. In the
+ * reference an evicted agent stays alive as long as a caller still refers to it: dropping the
+ * router's reference is all eviction does, and CPython's refcounting takes care of the rest.
+ * A JVM has no equivalent for a native ONNX session -- the garbage collector will not free it,
+ * so the router has to close it, and closing one that a caller is mid-prediction with would
+ * crash that prediction.
+ *
+ * <p>So eviction marks a slot retired and closes it when the last lease is released.
+ * {@link #predict} leases internally, which makes the ordinary path safe without a caller
+ * thinking about it; this is for code that needs the agent directly.
+ *
+ * <pre>{@code
+ * try (Router.Lease lease = router.lease("multilingual")) {
+ * Prediction answer = lease.agent().predict(state, questions);
+ * }
+ * }</pre>
+ */
+ public final class Lease implements AutoCloseable {
+
+ private final Checkpoint checkpoint;
+ private final Slot slot;
+ private boolean released;
+
+ private Lease(Checkpoint checkpoint, Slot slot) {
+ this.checkpoint = checkpoint;
+ this.slot = slot;
+ }
+
+ /** The agent, valid until this lease is closed. */
+ public Agent agent() {
+ if (released) {
+ throw new IllegalStateException("this lease on " + checkpoint.wireName()
+ + " has been released");
+ }
+ return slot.agent;
+ }
+
+ /** Which checkpoint this is. */
+ public Checkpoint checkpoint() {
+ return checkpoint;
+ }
+
+ /** Release it. Idempotent, so a try-with-resources around an early return is safe. */
+ @Override
+ public void close() {
+ if (released) {
+ return;
+ }
+ released = true;
+ release(slot);
+ }
+ }
+
+ /** One resident agent and what the router is allowed to do with it. */
+ private static final class Slot {
+
+ private final Agent agent;
+ /**
+ * False for an agent handed in through {@link Router#attach}: the caller kept ownership,
+ * so the router must never close it. Eviction still drops it, which is what the reference
+ * does too.
+ */
+ private final boolean ownsClosing;
+ private int leases;
+ private boolean retired;
+
+ private Slot(Agent agent, boolean ownsClosing) {
+ this.agent = agent;
+ this.ownsClosing = ownsClosing;
+ }
+ }
+
+ /**
+ * One build in progress, which every other caller for that checkpoint waits on.
+ *
+ * <p>Worth being precise about what this earns, because {@link Router#buildLock} looks like it
+ * covers the same ground. On the SUCCESS path it does: a caller that takes the build lock and
+ * then finds the checkpoint resident returns it instead of building a second copy, so one
+ * build per checkpoint holds with or without this latch.
+ *
+ * <p>The failure path is where they differ, and it is the expensive one. A build that fails
+ * leaves nothing resident, so without this latch each waiter in turn takes the build lock,
+ * finds no checkpoint, and re-enters the factory to discover the same failure for itself. For
+ * a load that fails by timing out a download, that is six slow failures instead of one.
+ * With it, the builder's error is handed to everyone waiting. Measured both ways: a mutant
+ * that removes the latch leaves one build on success and six attempts on failure.
+ */
+ private static final class InFlight {
+
+ private final CountDownLatch done = new CountDownLatch(1);
+ private volatile RuntimeException error;
+ }
+
// Names people are likely to type. Insertion order is not significant here, but the sorted
// order is: an unknown name's error message lists these, and that message is recorded.
private static final Map<String, Checkpoint> ALIASES;
@@ -216,10 +331,24 @@ public final class Router {
private static final Set<String> LANGUAGE_AGNOSTIC_CODES =
Set.of("c", "posix", "und", "zxx", "mul");
+ /** How many checkpoints stay resident unless asked otherwise, as in the reference. */
+ public static final int DEFAULT_MAX_LOADED = 2;
+
private final Map<Checkpoint, ModelSpec> models;
private final Checkpoint defaultCheckpoint;
private final boolean autoTaskDetection;
private final LanguageHint langGuess;
+ private final AgentFactory agents;
+
+ private final ReentrantLock lock = new ReentrantLock();
+ /** Serialises builds, so two cold loads do not hold two checkpoints in flight at once. */
+ private final ReentrantLock buildLock = new ReentrantLock();
+ private final Map<Checkpoint, Slot> slots = new LinkedHashMap<>();
+ /** Least recently used first, which is the end eviction takes from. */
+ private final List<Checkpoint> order = new ArrayList<>();
+ private final Map<Checkpoint, InFlight> loading = new LinkedHashMap<>();
+ private int maxLoaded;
+ private boolean closed;
private Router(Builder builder) {
Map<Checkpoint, ModelSpec> resolved =
@@ -229,6 +358,8 @@ public final class Router {
this.defaultCheckpoint = builder.defaultCheckpoint;
this.autoTaskDetection = builder.autoTaskDetection;
this.langGuess = builder.langGuess;
+ this.agents = builder.agents;
+ this.maxLoaded = builder.maxLoaded;
}
/** A router with the reference's defaults: the bundle, English as default, no auto-detection. */
@@ -249,6 +380,8 @@ public final class Router {
private boolean autoTaskDetection;
private boolean standaloneRepos;
private LanguageHint langGuess;
+ private AgentFactory agents;
+ private int maxLoaded = DEFAULT_MAX_LOADED;
private Builder() {
}
@@ -288,6 +421,53 @@ public final class Router {
return this;
}
+ /**
+ * How many checkpoints may be resident at once. Two by default, as in the reference.
+ *
+ * <p>Each is hundreds of megabytes, so this is a memory ceiling and not a cache size
+ * hint. The least recently used is evicted past it.
+ */
+ public Builder maxLoaded(int value) {
+ if (value < 1) {
+ throw new IllegalArgumentException("maxLoaded must be at least 1, got " + value);
+ }
+ this.maxLoaded = value;
+ return this;
+ }
+
+ /**
+ * How to build an agent for a checkpoint.
+ *
+ * <p>The reference downloads from the hub; this port takes local paths, so the mapping
+ * from a checkpoint to an {@link Agent} is the caller's. A seam rather than a hard-coded
+ * directory layout, for the same reason {@code InferenceSession} is one: it makes the
+ * eviction order, the lease counting and the attach-versus-build distinction testable in
+ * milliseconds, where a real checkpoint is hundreds of megabytes and would keep all of it
+ * out of any CI that does not download one.
+ */
+ public Builder agents(AgentFactory factory) {
+ this.agents = requireNonNull(factory, "factory");
+ return this;
+ }
+
+ /**
+ * Load each checkpoint from {@code <root>/<name>}, graph included.
+ *
+ * <p>The layout {@code Agent.open} expects, with the subdirectory named as the reference
+ * names the checkpoint: {@code english}, {@code multilingual}, {@code typed-decisions}.
+ */
+ public Builder checkpointsRoot(Path root) {
+ return checkpointsRoot(root, root);
+ }
+
+ /** As {@link #checkpointsRoot(Path)}, with the ONNX graphs under a separate root. */
+ public Builder checkpointsRoot(Path models, Path graphs) {
+ requireNonNull(models, "models");
+ requireNonNull(graphs, "graphs");
+ return agents(checkpoint -> Agent.open(models.resolve(checkpoint.wireName()),
+ graphs.resolve(checkpoint.wireName())));
+ }
+
/** Point one checkpoint somewhere else -- a mirror, or a local export. */
public Builder model(Checkpoint checkpoint, String repo, String subfolder) {
requireNonNull(checkpoint, "checkpoint");
@@ -599,6 +779,497 @@ public final class Router {
return new RouteDecision(key, models.get(key).repoString(), reason, detection, workflow);
}
+ // ------------------------------------------------------------------ the lifecycle
+
+ /**
+ * Borrow the agent for a checkpoint, building it on first use.
+ *
+ * <p>Concurrent callers asking for the same checkpoint share one build rather than each
+ * paying for a copy. The lease keeps the agent open; see {@link Lease} for why that matters
+ * here and not in the reference.
+ */
+ public Lease lease(String name) {
+ return lease(normaliseName(name));
+ }
+
+ /** Borrow the agent for a checkpoint. */
+ public Lease lease(Checkpoint checkpoint) {
+ requireNonNull(checkpoint, "checkpoint");
+ Slot slot = acquire(checkpoint);
+ return new Lease(checkpoint, slot);
+ }
+
+ /**
+ * The agent for a checkpoint, building it on first use.
+ *
+ * <p><b>Valid only while it stays resident.</b> Eviction and {@link #unload} close an agent
+ * the router built, so a reference held across either is a closed session. That is the price
+ * of a native resource on a JVM -- the reference can rely on refcounting here and this cannot.
+ * Prefer {@link #lease} or {@link #predict}, which hold the agent open for exactly as long as
+ * they use it; this exists for a caller doing its own lifetime management, and for
+ * {@link #preload}.
+ */
+ public Agent load(String name) {
+ Checkpoint checkpoint = normaliseName(name);
+ Slot slot = acquire(checkpoint);
+ release(slot);
+ return slot.agent;
+ }
+
+ /** The slot for a checkpoint, with one lease taken. */
+ private Slot acquire(Checkpoint checkpoint) {
+ while (true) {
+ InFlight waitFor = null;
+ InFlight mine = null;
+ lock.lock();
+ try {
+ requireOpen();
+ // The fast path, and not merely an optimisation: it returns a resident checkpoint
+ // without touching `buildLock`, so a request for a loaded model is not stalled
+ // behind a cold load of a different one. Remove it and every cached call queues
+ // on the build lock, which in a server means one checkpoint loading blocks all
+ // the traffic the other checkpoints could have served.
+ Slot resident = slots.get(checkpoint);
+ if (resident != null) {
+ touch(checkpoint);
+ resident.leases++;
+ return resident;
+ }
+ waitFor = loading.get(checkpoint);
+ if (waitFor == null) {
+ mine = new InFlight();
+ loading.put(checkpoint, mine);
+ }
+ } finally {
+ lock.unlock();
+ }
+ if (mine != null) {
+ // This caller owns the build, and `mine` is the handle the others are waiting on.
+ // It has to be the one registered above: creating a second one here would leave
+ // every waiter blocked on a latch nobody ever counts down.
+ return build(checkpoint, mine);
+ }
+ // Someone else is building this one. Wait for them rather than build a second copy.
+ await(waitFor);
+ if (waitFor.error != null) {
+ throw waitFor.error;
+ }
+ }
+ }
+
+ private Slot build(Checkpoint checkpoint, InFlight inflight) {
+ List<Checkpoint> evicted = List.of();
+ Slot built;
+ try {
+ // The build itself runs outside `lock`, so routing and eviction are not stalled for
+ // the seconds a cold checkpoint takes, and serialised by `buildLock` so two cold
+ // loads do not hold two checkpoints in flight at once.
+ buildLock.lock();
+ try {
+ lock.lock();
+ try {
+ Slot resident = slots.get(checkpoint);
+ if (resident != null) {
+ // Built or attached while this caller waited for the build lock.
+ touch(checkpoint);
+ resident.leases++;
+ finish(checkpoint, inflight);
+ return resident;
+ }
+ } finally {
+ lock.unlock();
+ }
+ Agent agent = create(checkpoint);
+ lock.lock();
+ try {
+ Slot attached = slots.get(checkpoint);
+ if (attached != null) {
+ // Attached while it was building: keep that one and close the duplicate,
+ // which the reference leaves to refcounting.
+ touch(checkpoint);
+ attached.leases++;
+ built = attached;
+ agent.close();
+ } else {
+ built = new Slot(agent, true);
+ built.leases++;
+ slots.put(checkpoint, built);
+ order.add(checkpoint);
+ evicted = evictLocked();
+ }
+ finish(checkpoint, inflight);
+ } finally {
+ lock.unlock();
+ }
+ } finally {
+ buildLock.unlock();
+ }
+ } catch (RuntimeException | Error failure) {
+ lock.lock();
+ try {
+ inflight.error = failure instanceof RuntimeException
+ ? (RuntimeException) failure
+ : new IllegalStateException("building " + checkpoint.wireName()
+ + " failed", failure);
+ loading.remove(checkpoint, inflight);
+ inflight.done.countDown();
+ } finally {
+ lock.unlock();
+ }
+ throw failure;
+ }
+ if (!evicted.isEmpty()) {
+ // Nothing to dispatch here, but keeping the list is what lets a caller see which
+ // checkpoints left: `loaded()` after a load is the observable form of it.
+ assert evicted.size() <= Checkpoint.values().length;
+ }
+ return built;
+ }
+
+ private Agent create(Checkpoint checkpoint) {
+ if (agents == null) {
+ throw new IllegalStateException(
+ "this Router can route but not load: no agent factory was configured. Use"
+ + " Router.builder().checkpointsRoot(path) or .agents(factory) to give it one,"
+ + " or call route(...) and load the checkpoint yourself.");
+ }
+ try {
+ Agent agent = agents.create(checkpoint);
+ if (agent == null) {
+ throw new IllegalStateException("the agent factory returned null for "
+ + checkpoint.wireName());
+ }
+ return agent;
+ } catch (IOException failure) {
+ throw new UncheckedIOException("cannot load the " + checkpoint.wireName()
+ + " checkpoint from " + models.get(checkpoint).repoString(), failure);
+ }
+ }
+
+ private void finish(Checkpoint checkpoint, InFlight inflight) {
+ loading.remove(checkpoint, inflight);
+ inflight.done.countDown();
+ }
+
+ private static void await(InFlight inflight) {
+ try {
+ inflight.done.await();
+ } catch (InterruptedException interrupted) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException("interrupted while waiting for a checkpoint to load",
+ interrupted);
+ }
+ }
+
+ /** Release one lease, closing the agent when it was retired and this was the last one. */
+ private void release(Slot slot) {
+ Agent toClose = null;
+ lock.lock();
+ try {
+ slot.leases--;
+ if (slot.leases <= 0 && slot.retired && slot.ownsClosing) {
+ toClose = slot.agent;
+ }
+ } finally {
+ lock.unlock();
+ }
+ if (toClose != null) {
+ toClose.close();
+ }
+ }
+
+ /** Move a checkpoint to the most-recently-used end. Caller holds {@link #lock}. */
+ private void touch(Checkpoint checkpoint) {
+ order.remove(checkpoint);
+ order.add(checkpoint);
+ }
+
+ /**
+ * Drop least-recently-used agents until {@link #maxLoaded} holds. Caller holds the lock.
+ *
+ * @return the checkpoints that left
+ */
+ private List<Checkpoint> evictLocked() {
+ List<Checkpoint> evicted = new ArrayList<>();
+ List<Agent> toClose = new ArrayList<>();
+ while (order.size() > maxLoaded) {
+ Checkpoint victim = order.remove(0);
+ Slot slot = slots.remove(victim);
+ if (slot != null) {
+ evicted.add(victim);
+ retire(slot, toClose);
+ }
+ }
+ // Keep the two views consistent, as the reference does: a slot with no place in the order
+ // is not reachable and would otherwise be held for the life of the router.
+ if (order.size() < slots.size()) {
+ for (Checkpoint stray : new ArrayList<>(slots.keySet())) {
+ if (!order.contains(stray)) {
+ Slot slot = slots.remove(stray);
+ if (slot != null) {
+ evicted.add(stray);
+ retire(slot, toClose);
+ }
+ }
+ }
+ }
+ closeAll(toClose);
+ return evicted;
+ }
+
+ /** Mark a slot gone, and collect its agent for closing when nothing is using it. */
+ private void retire(Slot slot, List<Agent> toClose) {
+ slot.retired = true;
+ if (slot.leases <= 0 && slot.ownsClosing) {
+ toClose.add(slot.agent);
+ }
+ }
+
+ /**
+ * Register an already-built agent instead of loading a second copy.
+ *
+ * <p>Useful when the process holds a checkpoint for other reasons: a service that already
+ * built the English one can hand it over rather than pay for -- and keep resident -- a
+ * duplicate 421M parameters.
+ *
+ * <p>The router never closes an attached agent: the caller keeps ownership. It also raises
+ * {@link #maxLoaded} to fit whatever is now resident, so attaching never immediately evicts
+ * what it just attached.
+ */
+ public Agent attach(String name, Agent agent) {
+ requireNonNull(agent, "agent");
+ Checkpoint checkpoint = normaliseName(name);
+ List<Agent> toClose = new ArrayList<>();
+ lock.lock();
+ try {
+ requireOpen();
+ Slot previous = slots.put(checkpoint, new Slot(agent, false));
+ if (previous != null) {
+ retire(previous, toClose);
+ }
+ touch(checkpoint);
+ maxLoaded = Math.max(maxLoaded, slots.size());
+ } finally {
+ lock.unlock();
+ }
+ closeAll(toClose);
+ return agent;
+ }
+
+ /** Build every checkpoint up front, so no request pays a cold load. */
+ public Router preload() {
+ return preload(null);
+ }
+
+ /**
+ * Build these checkpoints up front, so no request pays a cold load.
+ *
+ * <p>A cold load costs seconds; routing costs microseconds. With the checkpoints resident,
+ * routing is effectively free, which is what a server wants. {@link #maxLoaded} is raised to
+ * fit both what is asked for and whatever is already resident, so preloading in stages does
+ * not evict either.
+ *
+ * @param names checkpoint names or aliases, or null for all of them
+ */
+ public Router preload(Collection<String> names) {
+ List<Checkpoint> wanted = new ArrayList<>();
+ if (names == null) {
+ wanted.addAll(models.keySet());
+ } else {
+ for (String name : names) {
+ wanted.add(normaliseName(name));
+ }
+ }
+ lock.lock();
+ try {
+ requireOpen();
+ Set<Checkpoint> union = new LinkedHashSet<>(wanted);
+ union.addAll(slots.keySet());
+ maxLoaded = Math.max(maxLoaded, union.size());
+ } finally {
+ lock.unlock();
+ }
+ for (Checkpoint checkpoint : wanted) {
+ lock.lock();
+ boolean already;
+ try {
+ already = slots.containsKey(checkpoint);
+ } finally {
+ lock.unlock();
+ }
+ if (!already) {
+ release(acquire(checkpoint));
+ }
+ }
+ return this;
+ }
+
+ /**
+ * Free one checkpoint.
+ *
+ * <p>Waits for an in-flight build of that checkpoint only, so an unrelated cold load does not
+ * stall this call.
+ *
+ * @return the checkpoints that were freed, which is empty when it was not resident
+ */
+ public List<Checkpoint> unload(String name) {
+ Checkpoint checkpoint = normaliseName(name);
+ while (true) {
+ InFlight inflight;
+ List<Agent> toClose = new ArrayList<>();
+ List<Checkpoint> freed = new ArrayList<>();
+ lock.lock();
+ try {
+ inflight = loading.get(checkpoint);
+ if (inflight == null) {
+ Slot slot = slots.remove(checkpoint);
+ order.remove(checkpoint);
+ if (slot != null) {
+ freed.add(checkpoint);
+ retire(slot, toClose);
+ }
+ }
+ } finally {
+ lock.unlock();
+ }
+ if (inflight == null) {
+ closeAll(toClose);
+ return freed;
+ }
+ await(inflight);
+ }
+ }
+
+ /** Free every checkpoint, waiting for any in-flight build first. */
+ public List<Checkpoint> unloadAll() {
+ while (true) {
+ List<InFlight> inflights;
+ List<Agent> toClose = new ArrayList<>();
+ List<Checkpoint> freed = new ArrayList<>();
+ lock.lock();
+ try {
+ inflights = new ArrayList<>(loading.values());
+ if (inflights.isEmpty()) {
+ freed.addAll(order);
+ for (Slot slot : slots.values()) {
+ retire(slot, toClose);
+ }
+ slots.clear();
+ order.clear();
+ }
+ } finally {
+ lock.unlock();
+ }
+ if (inflights.isEmpty()) {
+ closeAll(toClose);
+ return freed;
+ }
+ for (InFlight inflight : inflights) {
+ await(inflight);
+ }
+ }
+ }
+
+ /** Which checkpoints are resident, least recently used first. */
+ public List<Checkpoint> loaded() {
+ lock.lock();
+ try {
+ return List.copyOf(order);
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ /** How many checkpoints may be resident at once; {@link #attach} and {@link #preload} raise it. */
+ public int maxLoaded() {
+ lock.lock();
+ try {
+ return maxLoaded;
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ /** Route this state and answer its questions on whichever checkpoint wins. */
+ public Prediction predict(Object state, Map<String, Question> questions) {
+ return predict(state, questions, RouteOptions.none());
+ }
+
+ /** Route this state and answer its questions, with per-call routing arguments. */
+ public Prediction predict(Object state, Map<String, Question> questions,
+ RouteOptions options) {
+ return predict(route(state, questions, options), state, questions,
+ options == null ? null : options.lang());
+ }
+
+ /**
+ * Answer these questions on an already-made decision.
+ *
+ * <p>For a caller that wants the reason as well as the answer: route once, show or log the
+ * decision, then predict with it, rather than routing twice.
+ */
+ public Prediction predict(RouteDecision decision, Object state,
+ Map<String, Question> questions) {
+ return predict(decision, state, questions, null);
+ }
+
+ private Prediction predict(RouteDecision decision, Object state,
+ Map<String, Question> questions, String lang) {
+ requireNonNull(decision, "decision");
+ // The language the checkpoint is told is the caller's if they gave one, else whatever
+ // routing detected -- which is the reference's rule, and it matters: the multilingual
+ // checkpoint takes a language and the detected one is the best available answer.
+ String language = lang;
+ if (language == null && decision.detection() != null) {
+ language = decision.detection().language();
+ }
+ try (Lease lease = lease(decision.model())) {
+ return lease.agent().predict(state, questions, language);
+ }
+ }
+
+ /**
+ * Close the router, freeing every checkpoint it built.
+ *
+ * <p>An attached agent is left alone: its owner is the caller. An agent still under lease is
+ * closed when that lease is released, so closing the router does not break a prediction
+ * already in flight.
+ */
+ @Override
+ public void close() {
+ lock.lock();
+ try {
+ closed = true;
+ } finally {
+ lock.unlock();
+ }
+ unloadAll();
+ }
+
+ private void requireOpen() {
+ if (closed) {
+ throw new IllegalStateException("this Router is closed");
+ }
+ }
+
+ private static void closeAll(List<Agent> agents) {
+ RuntimeException first = null;
+ for (Agent agent : agents) {
+ try {
+ agent.close();
+ } catch (RuntimeException failure) {
+ if (first == null) {
+ first = failure;
+ } else {
+ first.addSuppressed(failure);
+ }
+ }
+ }
+ if (first != null) {
+ throw first;
+ }
+ }
+
/** The first {@code count} code points, which is what the reference's slice takes. */
private static String headCodePoints(String text, int count) {
if (text.codePointCount(0, text.length()) <= count) {
diff --git a/laya-java/laya-java/src/test/java/com/convaiinnovations/laya/RouterLifecycleTest.java b/laya-java/laya-java/src/test/java/com/convaiinnovations/laya/RouterLifecycleTest.java
new file mode 100644
index 00000000..dd2db716
--- /dev/null
+++ b/laya-java/laya-java/src/test/java/com/convaiinnovations/laya/RouterLifecycleTest.java
@@ -0,0 +1,619 @@
+package com.convaiinnovations.laya;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import com.convaiinnovations.laya.Router.Checkpoint;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.EnumMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+/**
+ * The {@link Router}'s load-and-evict lifecycle, driven through a stub agent factory.
+ *
+ * <p>A seam rather than a real checkpoint, for the reason {@code InferenceSession} is one: the
+ * parts most likely to be wrong here -- which agent the eviction picks, whether an attached one is
+ * ever closed, whether two threads asking for the same cold checkpoint pay for two copies -- have
+ * nothing to do with ONNX, and a real checkpoint is hundreds of megabytes. These run in
+ * milliseconds and need no download, so they run in every CI job rather than only the parity lane.
+ *
+ * <p>Closing is what these tests watch most closely, because it is the one place this port cannot
+ * copy the reference. There, eviction drops a reference and CPython's refcounting keeps an agent
+ * alive for whoever still holds it. A JVM will not free a native ONNX session that way, so the
+ * router has to close it -- and must not close one that a caller is still using, or that the
+ * caller owns.
+ */
+final class RouterLifecycleTest {
+
+ private static Path checkpointRoot;
+
+ @BeforeAll
+ static void writeCheckpoint(@TempDir Path root) throws IOException {
+ // One synthetic checkpoint, reused for every Checkpoint value: the lifecycle does not care
+ // what is inside, only which instance it hands out and when it closes one.
+ checkpointRoot = root;
+ TinyCheckpoint.write(root, 64, 32);
+ }
+
+ /** A factory that counts builds and keeps every session it handed out, so closes are visible. */
+ private static final class StubAgents implements Router.AgentFactory {
+
+ private final Map<Checkpoint, Integer> builds = new EnumMap<>(Checkpoint.class);
+ private final Map<Checkpoint, List<TinyCheckpoint.RecordingSession>> sessions =
+ new EnumMap<>(Checkpoint.class);
+ private final AtomicInteger total = new AtomicInteger();
+ /** Every entry into the factory, successful or not: a failed load still costs a download. */
+ private final Map<Checkpoint, Integer> attempts = new EnumMap<>(Checkpoint.class);
+ private volatile Checkpoint failOn;
+ private volatile CountDownLatch blockOn;
+
+ @Override
+ public Agent create(Checkpoint checkpoint) throws IOException {
+ synchronized (this) {
+ attempts.merge(checkpoint, 1, Integer::sum);
+ }
+ // The gate is waited on BEFORE the failure check on purpose. A build that fails
+ // instantly is finished before a second caller even arrives, so nothing is ever
+ // shared and a test counting attempts measures the harness rather than the router.
+ // A real load that fails -- a download that times out -- takes just as long as one
+ // that succeeds.
+ CountDownLatch gate = blockOn;
+ if (gate != null) {
+ try {
+ assertTrue(gate.await(10, TimeUnit.SECONDS), "the build gate never opened");
+ } catch (InterruptedException interrupted) {
+ Thread.currentThread().interrupt();
+ throw new IOException(interrupted);
+ }
+ }
+ if (checkpoint == failOn) {
+ throw new IOException("no checkpoint for " + checkpoint.wireName());
+ }
+ TinyCheckpoint.RecordingSession session = new TinyCheckpoint.RecordingSession();
+ synchronized (this) {
+ builds.merge(checkpoint, 1, Integer::sum);
+ sessions.computeIfAbsent(checkpoint, key -> new ArrayList<>()).add(session);
+ }
+ total.incrementAndGet();
+ return TinyCheckpoint.agent(checkpointRoot, session);
+ }
+
+ synchronized int buildsOf(Checkpoint checkpoint) {
+ return builds.getOrDefault(checkpoint, 0);
+ }
+
+ synchronized int attemptsOf(Checkpoint checkpoint) {
+ return attempts.getOrDefault(checkpoint, 0);
+ }
+
+ synchronized List<TinyCheckpoint.RecordingSession> sessionsOf(Checkpoint checkpoint) {
+ return List.copyOf(sessions.getOrDefault(checkpoint, List.of()));
+ }
+
+ synchronized TinyCheckpoint.RecordingSession lastSession(Checkpoint checkpoint) {
+ List<TinyCheckpoint.RecordingSession> all = sessionsOf(checkpoint);
+ assertFalse(all.isEmpty(), "nothing was ever built for " + checkpoint.wireName());
+ return all.get(all.size() - 1);
+ }
+ }
+
+ private static Router.Builder routerWith(StubAgents agents) {
+ return Router.builder().agents(agents);
+ }
+
+ private static Map<String, Question> oneQuestion() {
+ Map<String, Question> questions = new LinkedHashMap<>();
+ questions.put("urgent", Question.noul("Is this urgent?"));
+ return questions;
+ }
+
+ @Test
+ @DisplayName("a second call reuses the loaded agent rather than building another")
+ void loadIsCached() {
+ StubAgents agents = new StubAgents();
+ try (Router router = routerWith(agents).build()) {
+ Agent first = router.load("english");
+ Agent second = router.load("en");
+ assertSame(first, second, "the alias names the same checkpoint, so the same agent");
+ assertEquals(1, agents.buildsOf(Checkpoint.ENGLISH));
+ assertEquals(List.of(Checkpoint.ENGLISH), router.loaded());
+ }
+ }
+
+ @Test
+ @DisplayName("loaded() is least-recently-used first, and a use moves a checkpoint to the end")
+ void loadedOrderIsLru() {
+ StubAgents agents = new StubAgents();
+ try (Router router = routerWith(agents).maxLoaded(3).build()) {
+ router.load("english");
+ router.load("multilingual");
+ router.load("typed-decisions");
+ assertEquals(List.of(Checkpoint.ENGLISH, Checkpoint.MULTILINGUAL,
+ Checkpoint.TYPED_DECISIONS), router.loaded());
+ router.load("english");
+ assertEquals(List.of(Checkpoint.MULTILINGUAL, Checkpoint.TYPED_DECISIONS,
+ Checkpoint.ENGLISH), router.loaded(),
+ "using english must move it to the most-recently-used end");
+ }
+ }
+
+ @Test
+ @DisplayName("past maxLoaded the least recently used is evicted, and closed")
+ void evictionClosesTheVictim() {
+ StubAgents agents = new StubAgents();
+ try (Router router = routerWith(agents).maxLoaded(2).build()) {
+ router.load("english");
+ router.load("multilingual");
+ TinyCheckpoint.RecordingSession englishSession = agents.lastSession(Checkpoint.ENGLISH);
+ assertFalse(englishSession.closed, "nothing has been evicted yet");
+
+ router.load("typed-decisions");
+ assertEquals(List.of(Checkpoint.MULTILINGUAL, Checkpoint.TYPED_DECISIONS),
+ router.loaded(), "english was least recently used");
+ // The part the reference gets for free and this does not: a native session that is
+ // merely dropped is leaked, so the router has to close it.
+ assertTrue(englishSession.closed, "the evicted agent must be closed, not just dropped");
+ assertFalse(agents.lastSession(Checkpoint.MULTILINGUAL).closed);
+ }
+ }
+
+ @Test
+ @DisplayName("an attached agent is never closed by the router, even when evicted")
+ void attachedAgentsAreNotClosed() throws IOException {
+ StubAgents agents = new StubAgents();
+ TinyCheckpoint.RecordingSession mine = new TinyCheckpoint.RecordingSession();
+ Agent owned = TinyCheckpoint.agent(checkpointRoot, mine);
+ try (Router router = routerWith(agents).maxLoaded(1).build()) {
+ assertSame(owned, router.attach("english", owned));
+ assertSame(owned, router.load("english"), "an attached agent is already built");
+ assertEquals(0, agents.buildsOf(Checkpoint.ENGLISH), "nothing should have been built");
+
+ // Force it out, with maxLoaded back at a level that evicts.
+ router.load("multilingual");
+ router.load("typed-decisions");
+ assertFalse(router.loaded().contains(Checkpoint.ENGLISH), "it should have been evicted");
+ assertFalse(mine.closed, "the caller owns an attached agent; the router must not close it");
+ }
+ assertFalse(mine.closed, "closing the router must not close an attached agent either");
+ owned.close();
+ assertTrue(mine.closed, "and the owner can still close it afterwards");
+ }
+
+ @Test
+ @DisplayName("attach raises maxLoaded, so it never immediately evicts what it attached")
+ void attachRaisesMaxLoaded() throws IOException {
+ StubAgents agents = new StubAgents();
+ try (Router router = routerWith(agents).maxLoaded(1).build()) {
+ assertEquals(1, router.maxLoaded());
+ router.load("english");
+ router.attach("multilingual", TinyCheckpoint.agent(checkpointRoot,
+ new TinyCheckpoint.RecordingSession()));
+ assertEquals(2, router.maxLoaded(), "it had to grow to fit both");
+ assertTrue(router.loaded().contains(Checkpoint.MULTILINGUAL),
+ "the agent just attached must still be resident");
+ }
+ }
+
+ @Test
+ @DisplayName("preload builds everything and raises maxLoaded to fit it")
+ void preloadFitsEverything() {
+ StubAgents agents = new StubAgents();
+ try (Router router = routerWith(agents).maxLoaded(1).build()) {
+ router.preload();
+ assertEquals(3, router.maxLoaded());
+ assertEquals(3, router.loaded().size(), "all three must be resident, none evicted");
+ for (Checkpoint checkpoint : Checkpoint.values()) {
+ assertEquals(1, agents.buildsOf(checkpoint), checkpoint + " built exactly once");
+ }
+ // A cold load costs seconds and routing costs microseconds, so a preloaded router
+ // answers without ever paying a load. A second preload must not rebuild anything.
+ router.preload();
+ assertEquals(3, agents.total.get(), "preloading twice must not rebuild");
+ }
+ }
+
+ @Test
+ @DisplayName("preloading in stages does not evict what an earlier stage loaded")
+ void preloadInStages() {
+ StubAgents agents = new StubAgents();
+ try (Router router = routerWith(agents).maxLoaded(1).build()) {
+ router.preload(List.of("english"));
+ router.preload(List.of("multilingual"));
+ assertEquals(List.of(Checkpoint.ENGLISH, Checkpoint.MULTILINGUAL), router.loaded(),
+ "the second stage must not have evicted the first");
+ assertFalse(agents.lastSession(Checkpoint.ENGLISH).closed);
+ }
+ }
+
+ @Test
+ @DisplayName("unload frees one checkpoint and closes it; unloadAll frees the rest")
+ void unloadFrees() {
+ StubAgents agents = new StubAgents();
+ try (Router router = routerWith(agents).maxLoaded(3).build()) {
+ router.preload();
+ assertEquals(List.of(Checkpoint.ENGLISH),
+ router.unload("en"), "it reports what it freed");
+ assertTrue(agents.lastSession(Checkpoint.ENGLISH).closed);
+ assertEquals(2, router.loaded().size());
+ assertEquals(List.of(), router.unload("en"), "unloading it twice frees nothing");
+
+ List<Checkpoint> rest = router.unloadAll();
+ assertEquals(2, rest.size(), "both remaining checkpoints were freed");
+ assertEquals(List.of(), router.loaded());
+ for (Checkpoint checkpoint : Checkpoint.values()) {
+ assertTrue(agents.lastSession(checkpoint).closed, checkpoint + " must be closed");
+ }
+ }
+ }
+
+ @Test
+ @DisplayName("a lease keeps an evicted agent open until it is released")
+ void leaseOutlivesEviction() {
+ StubAgents agents = new StubAgents();
+ try (Router router = routerWith(agents).maxLoaded(1).build()) {
+ Router.Lease lease = router.lease("english");
+ TinyCheckpoint.RecordingSession session = agents.lastSession(Checkpoint.ENGLISH);
+ Agent leased = lease.agent();
+
+ // Evict it while the lease is open. Closing it here would break a prediction already
+ // running on it, which is the whole reason a lease exists.
+ router.load("multilingual");
+ assertFalse(router.loaded().contains(Checkpoint.ENGLISH), "it was evicted");
+ assertFalse(session.closed, "an agent under lease must stay open");
+ assertSame(leased, lease.agent(), "and must still be reachable through the lease");
+
+ lease.close();
+ assertTrue(session.closed, "released, retired, and now closed");
+ lease.close(); // idempotent, so try-with-resources round an early return is safe
+ assertThrows(IllegalStateException.class, lease::agent,
+ "a released lease must not hand out an agent it no longer holds");
+ }
+ }
+
+ @Test
+ @DisplayName("two threads asking for the same cold checkpoint share one build")
+ void concurrentLoadsShareOneBuild() throws Exception {
+ // This is the case the in-flight handle exists for. An earlier draft registered one handle
+ // and then waited on a different one, which left every waiter blocked on a latch nobody
+ // counted down -- a deadlock, not a duplicate build, so it would not have shown up as a
+ // wasted load.
+ StubAgents agents = new StubAgents();
+ CountDownLatch gate = new CountDownLatch(1);
+ agents.blockOn = gate;
+ try (Router router = routerWith(agents).maxLoaded(3).build()) {
+ int threads = 6;
+ CountDownLatch ready = new CountDownLatch(threads);
+ CountDownLatch go = new CountDownLatch(1);
+ List<Agent> seen = Collections.synchronizedList(new ArrayList<>());
+ List<Throwable> failures = Collections.synchronizedList(new ArrayList<>());
+ List<Thread> workers = new ArrayList<>();
+ for (int i = 0; i < threads; i++) {
+ Thread worker = new Thread(() -> {
+ ready.countDown();
+ try {
+ assertTrue(go.await(10, TimeUnit.SECONDS));
+ seen.add(router.load("english"));
+ } catch (Throwable failure) {
+ failures.add(failure);
+ }
+ }, "load-" + i);
+ worker.start();
+ workers.add(worker);
+ }
+ assertTrue(ready.await(10, TimeUnit.SECONDS), "the workers never started");
+ go.countDown();
+ // Let them pile up on the one build, then let it finish.
+ Thread.sleep(50);
+ gate.countDown();
+ for (Thread worker : workers) {
+ worker.join(TimeUnit.SECONDS.toMillis(20));
+ assertFalse(worker.isAlive(), worker.getName() + " never finished: a waiter is"
+ + " blocked on a latch that was never counted down");
+ }
+ assertEquals(List.of(), failures, "a worker failed");
+ assertEquals(threads, seen.size());
+ assertEquals(1, agents.buildsOf(Checkpoint.ENGLISH),
+ "six callers must share one build, not pay for six copies");
+ for (Agent agent : seen) {
+ assertSame(seen.get(0), agent, "every caller must get the same agent");
+ }
+ }
+ }
+
+ @Test
+ @DisplayName("a resident checkpoint answers while a different one is still loading")
+ void residentLoadDoesNotWaitForAColdBuild() throws Exception {
+ // The property the cache fast path in `acquire` exists for, and the only observable
+ // difference between having it and not: a resident checkpoint is returned without
+ // touching the build lock, so a request it can already serve is not stalled behind a cold
+ // load of another model. In a server that difference is one checkpoint loading versus all
+ // traffic stopping. A mutant that removed the fast path passed every other test here.
+ StubAgents agents = new StubAgents();
+ try (Router router = routerWith(agents).maxLoaded(3).build()) {
+ router.load("english");
+ assertEquals(1, agents.buildsOf(Checkpoint.ENGLISH));
+
+ CountDownLatch gate = new CountDownLatch(1);
+ agents.blockOn = gate; // the next build, whichever it is, waits here
+ CountDownLatch coldStarted = new CountDownLatch(1);
+ List<Throwable> failures = Collections.synchronizedList(new ArrayList<>());
+ Thread cold = new Thread(() -> {
+ coldStarted.countDown();
+ try {
+ router.load("multilingual");
+ } catch (Throwable failure) {
+ failures.add(failure);
+ }
+ }, "cold-load");
+ cold.start();
+ assertTrue(coldStarted.await(10, TimeUnit.SECONDS), "the cold load never started");
+ Thread.sleep(150); // let it reach the blocked factory
+
+ CountDownLatch warmDone = new CountDownLatch(1);
+ Thread warm = new Thread(() -> {
+ try {
+ router.load("en");
+ warmDone.countDown();
+ } catch (Throwable failure) {
+ failures.add(failure);
+ }
+ }, "warm-load");
+ warm.start();
+ assertTrue(warmDone.await(5, TimeUnit.SECONDS),
+ "a request for an already-loaded checkpoint blocked behind a cold load of a"
+ + " different one");
+
+ gate.countDown();
+ cold.join(TimeUnit.SECONDS.toMillis(20));
+ warm.join(TimeUnit.SECONDS.toMillis(20));
+ assertFalse(cold.isAlive(), "the cold load never finished");
+ assertEquals(List.of(), failures, "a worker failed");
+ assertEquals(1, agents.buildsOf(Checkpoint.ENGLISH), "english was never rebuilt");
+ assertEquals(1, agents.buildsOf(Checkpoint.MULTILINGUAL));
+ }
+ }
+
+ @Test
+ @DisplayName("a failed build propagates and leaves no in-flight entry behind")
+ void failedBuildDoesNotWedgeTheRouter() {
+ StubAgents agents = new StubAgents();
+ agents.failOn = Checkpoint.MULTILINGUAL;
+ try (Router router = routerWith(agents).build()) {
+ UncheckedIOException failure = assertThrows(UncheckedIOException.class,
+ () -> router.load("multilingual"));
+ assertTrue(failure.getMessage().contains("multilingual"), failure.getMessage());
+ assertEquals(List.of(), router.loaded());
+
+ // The point: a second attempt must get the same error rather than hang on a latch the
+ // failed attempt never counted down.
+ assertThrows(UncheckedIOException.class, () -> router.load("multilingual"));
+ // and an unrelated checkpoint still loads
+ agents.failOn = null;
+ router.load("english");
+ assertEquals(List.of(Checkpoint.ENGLISH), router.loaded());
+ }
+ }
+
+ @Test
+ @DisplayName("concurrent callers share a FAILING build too, rather than each retrying it")
+ void concurrentCallersShareAFailure() throws Exception {
+ // This is what the in-flight latch earns that the build lock does not. Correctness-wise
+ // the two are the same -- a mutant that removed the latch still produced one build per
+ // checkpoint and passed every other test here -- but on the failure path the latch hands
+ // the builder's error to the waiters, where without it each waiter re-enters the factory
+ // and pays for the attempt again. For a load that fails by timing out a download, that is
+ // the difference between one slow failure and six.
+ StubAgents agents = new StubAgents();
+ agents.failOn = Checkpoint.MULTILINGUAL;
+ CountDownLatch gate = new CountDownLatch(1);
+ agents.blockOn = gate;
+ try (Router router = routerWith(agents).build()) {
+ int threads = 6;
+ CountDownLatch ready = new CountDownLatch(threads);
+ CountDownLatch go = new CountDownLatch(1);
+ List<Throwable> thrown = Collections.synchronizedList(new ArrayList<>());
+ List<Thread> workers = new ArrayList<>();
+ for (int i = 0; i < threads; i++) {
+ Thread worker = new Thread(() -> {
+ ready.countDown();
+ try {
+ assertTrue(go.await(10, TimeUnit.SECONDS));
+ router.load("multilingual");
+ thrown.add(new AssertionError("the load should have failed"));
+ } catch (UncheckedIOException expected) {
+ thrown.add(expected);
+ } catch (Throwable other) {
+ thrown.add(other);
+ }
+ }, "failing-load-" + i);
+ worker.start();
+ workers.add(worker);
+ }
+ assertTrue(ready.await(10, TimeUnit.SECONDS));
+ go.countDown();
+ Thread.sleep(100); // let them pile up on the one attempt
+ gate.countDown();
+ for (Thread worker : workers) {
+ worker.join(TimeUnit.SECONDS.toMillis(20));
+ assertFalse(worker.isAlive(), worker.getName() + " never finished");
+ }
+ assertEquals(threads, thrown.size());
+ for (Throwable failure : thrown) {
+ assertTrue(failure instanceof UncheckedIOException,
+ "every caller must see the failure, got " + failure);
+ }
+ assertEquals(1, agents.attemptsOf(Checkpoint.MULTILINGUAL),
+ "six callers must share one failed attempt, not pay for six");
+ assertEquals(List.of(), router.loaded());
+ }
+ }
+
+ @Test
+ @DisplayName("a router with no factory says so instead of failing obscurely")
+ void routingWithoutAFactory() {
+ try (Router router = Router.withDefaults()) {
+ // Routing still works: it reads no disk. That is what makes this a usable mode and
+ // not a misconfiguration.
+ assertEquals(Checkpoint.ENGLISH,
+ router.route("I cannot log in to my account today").model());
+ IllegalStateException failure = assertThrows(IllegalStateException.class,
+ () -> router.load("english"));
+ assertTrue(failure.getMessage().contains("route but not load"), failure.getMessage());
+ assertTrue(failure.getMessage().contains("checkpointsRoot"),
+ "the message must say how to fix it: " + failure.getMessage());
+ }
+ }
+
+ @Test
+ @DisplayName("predict routes, then answers on the checkpoint that won")
+ void predictRoutesThenAnswers() {
+ StubAgents agents = new StubAgents();
+ try (Router router = routerWith(agents).maxLoaded(3).build()) {
+ Prediction english = router.predict("I cannot log in to my account at all today",
+ oneQuestion());
+ assertEquals(1, agents.buildsOf(Checkpoint.ENGLISH));
+ assertEquals(0, agents.buildsOf(Checkpoint.MULTILINGUAL));
+ assertEquals(1, english.answers().size());
+
+ router.predict("계정에 로그인할 수 없어요",
+ oneQuestion());
+ assertEquals(1, agents.buildsOf(Checkpoint.MULTILINGUAL),
+ "Korean must not be answered on the English checkpoint");
+ }
+ }
+
+ @Test
+ @DisplayName("predict leaves no lease behind, so the agent is closable afterwards")
+ void predictReleasesItsLease() {
+ StubAgents agents = new StubAgents();
+ try (Router router = routerWith(agents).maxLoaded(1).build()) {
+ router.predict("I cannot log in to my account at all today", oneQuestion());
+ TinyCheckpoint.RecordingSession session = agents.lastSession(Checkpoint.ENGLISH);
+ // If predict leaked its lease, eviction would retire the slot but never close it --
+ // a leak that no other test here would notice.
+ router.load("multilingual");
+ assertTrue(session.closed, "predict must release its lease when it returns");
+ }
+ }
+
+ @Test
+ @DisplayName("predict tells the checkpoint the language routing detected")
+ void predictPassesTheDetectedLanguage(@TempDir Path root) throws IOException {
+ // The multilingual checkpoint takes a language and applies that language's fitted
+ // temperatures, and the detected language is the best answer available. A port that
+ // passed null here would route correctly and then ask the question WITHOUT the language
+ // it had just worked out, which changes the confidence it reports -- silently.
+ //
+ // Made observable rather than asserted: this checkpoint overrides the noul temperature
+ // for `pt`, so the same state and question answer differently with and without the
+ // language. An earlier version of this test compared "multilingual" to "multilingual",
+ // which is true whatever the router does.
+ TinyCheckpoint.write(root, 64, 32);
+ Files.writeString(root.resolve("rl_agent_config.json"),
+ "{\"max_len\": 64, \"head_max_len\": 32,"
+ + " \"temperature\": [1.0, 1.0, 1.0], \"temperature_by_options\": {},"
+ + " \"lang_temperatures\": {\"pt\": {\"temperature\": [1.0, 1.0, 2.0]}}}",
+ StandardCharsets.UTF_8);
+
+ String portuguese = "Você pode me mandar a nota fiscal do pedido que eu fiz ontem?";
+ double withoutLanguage;
+ double withPortuguese;
+ try (Agent direct = TinyCheckpoint.agent(root, new TinyCheckpoint.RecordingSession())) {
+ withoutLanguage = noulOf(direct.predict(portuguese, oneQuestion(), null));
+ withPortuguese = noulOf(direct.predict(portuguese, oneQuestion(), "pt"));
+ }
+ assertNotEquals(withoutLanguage, withPortuguese,
+ "this checkpoint must answer differently with the language, or the assertion"
+ + " below cannot tell whether the router passed it");
+
+ Router.AgentFactory factory = checkpoint ->
+ TinyCheckpoint.agent(root, new TinyCheckpoint.RecordingSession());
+ try (Router router = Router.builder().agents(factory).maxLoaded(3).build()) {
+ Router.RouteDecision decided = router.route(portuguese);
+ assertEquals(Checkpoint.MULTILINGUAL, decided.model());
+ assertEquals("pt", decided.detection().language(), "the detector named Portuguese");
+
+ double routed = noulOf(router.predict(decided, portuguese, oneQuestion()));
+ assertEquals(withPortuguese, routed,
+ "the router must pass the language it detected, not null");
+
+ // An explicit language wins over the detected one, which is the reference's rule --
+ // and it decides without running detection at all.
+ Router.RouteDecision pinned = router.route("plain english words here now", null,
+ Router.RouteOptions.none().lang("fr"));
+ assertEquals(Checkpoint.MULTILINGUAL, pinned.model());
+ assertNull(pinned.detection(),
+ "an explicit language decides without running detection");
+ }
+ }
+
+ /** The probability a noul answer reports, which the fitted temperature moves. */
+ private static double noulOf(Prediction prediction) {
+ Answer answer = prediction.answers().get("urgent");
+ assertTrue(answer instanceof Answer.Noul,
+ "expected a noul answer, got " + answer.getClass().getSimpleName());
+ return ((Answer.Noul) answer).noul();
+ }
+
+ @Test
+ @DisplayName("close frees what the router built and refuses further loads")
+ void closeFreesAndSeals() {
+ StubAgents agents = new StubAgents();
+ Router router = routerWith(agents).maxLoaded(3).build();
+ router.preload();
+ router.close();
+ for (Checkpoint checkpoint : Checkpoint.values()) {
+ assertTrue(agents.lastSession(checkpoint).closed, checkpoint + " must be closed");
+ }
+ assertEquals(List.of(), router.loaded());
+ assertThrows(IllegalStateException.class, () -> router.load("english"));
+ assertThrows(IllegalStateException.class, () -> router.lease("english"));
+ router.close(); // idempotent
+ // and routing a closed router is still harmless, because it touches nothing
+ assertEquals(Checkpoint.ENGLISH, router.route("plain english text here now").model());
+ }
+
+ @Test
+ @DisplayName("maxLoaded below one is refused, because zero resident checkpoints answers nothing")
+ void maxLoadedMustBePositive() {
+ assertThrows(IllegalArgumentException.class, () -> Router.builder().maxLoaded(0));
+ assertThrows(IllegalArgumentException.class, () -> Router.builder().maxLoaded(-1));
+ }
+
+ @Test
+ @DisplayName("attaching over an existing agent retires the one it replaced")
+ void attachReplaces() throws IOException {
+ StubAgents agents = new StubAgents();
+ try (Router router = routerWith(agents).maxLoaded(3).build()) {
+ router.load("english");
+ TinyCheckpoint.RecordingSession built = agents.lastSession(Checkpoint.ENGLISH);
+ Agent replacement = TinyCheckpoint.agent(checkpointRoot,
+ new TinyCheckpoint.RecordingSession());
+ router.attach("english", replacement);
+ assertSame(replacement, router.load("english"));
+ assertTrue(built.closed,
+ "the agent the router built and then replaced must be closed, not leaked");
+ assertEquals(1, router.loaded().size(), "and english is resident exactly once");
+ }
+ }
+}