Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(java): the embedding shortlist, and a Predictor seam for it

Adeshina Lasisi55d8df96f72c
diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml
index 3fd76506..747a562a 100644
--- a/.github/workflows/java.yml
+++ b/.github/workflows/java.yml
@@ -70,7 +70,7 @@ jobs:
if: ${{ !cancelled() }}
run: >
python laya-java/scripts/check_test_results.py
- laya-java/laya-java/build/test-results/test --min-tests 390 --allow-aborted
+ laya-java/laya-java/build/test-results/test --min-tests 440 --allow-aborted
# LAST, deliberately. This gate and the tests are independent signals -- the tests compare
# against the COMMITTED fixture, this compares the committed fixture against a fresh
@@ -172,7 +172,7 @@ jobs:
if: ${{ !cancelled() }}
run: >
python laya-java/scripts/check_test_results.py
- laya-java/laya-java/build/test-results/test --min-tests 540
+ laya-java/laya-java/build/test-results/test --min-tests 590
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
if: ${{ failure() }}
diff --git a/laya-java/fixtures/shortlist.json b/laya-java/fixtures/shortlist.json
new file mode 100644
index 00000000..5efb112e
--- /dev/null
+++ b/laya-java/fixtures/shortlist.json
@@ -0,0 +1,1584 @@
+{
+ "notes": "`laya.shortlist`'s ranking, its edge cases, and what it hands to `predict`.\n\n Choice options share one `head_max_len`, so a large label set leaves a few tokens per label.\n Shortlisting embeds the state and each option, keeps the top k, and runs ONE predict on the\n reduced set. The parts that need gating are not the embedding -- a caller supplies that -- but\n the ranking rules around it, and each of them is a place a port goes wrong silently:\n\n * the sort is STABLE and descending, so a tie keeps the earlier label. `argsort` without\n `kind=\"mergesort\"` reorders ties, which changes which labels survive at the cut.\n * the score is a SIGNED cosine, not a similarity floor. A label scoring 0 -- no signal, or a\n non-finite vector treated as none -- outranks one scoring negative, and k drops the\n negatives first.\n * a zero-norm query scores everything 0, so the kept set is the first k in criteria order.\n * non-finite components become 0 rather than propagating or raising.\n * k >= the label count is a passthrough: the labels come back in criteria order, scores are\n None, and the embedder is NEVER called.\n ",
+ "score_decimals": 12,
+ "default_k": 20,
+ "embed_dim": 8,
+ "embed_probe": {
+ "": [
+ 0.9430000000000001,
+ 0.732,
+ -0.635,
+ -0.846,
+ 0.09899999999999998,
+ -0.11199999999999999,
+ 0.5209999999999999,
+ 0.31000000000000005
+ ],
+ "a": [
+ 0.20599999999999996,
+ 0.41700000000000004,
+ -0.21599999999999997,
+ -0.0050000000000000044,
+ -0.95,
+ -0.739,
+ 0.6279999999999999,
+ 0.839
+ ],
+ "refund: money back": [
+ -0.17600000000000005,
+ 0.03499999999999992,
+ 0.246,
+ 0.4570000000000001,
+ 0.6679999999999999,
+ 0.879,
+ -0.91,
+ -0.6990000000000001
+ ],
+ "My card was charged twice for the same order last Tuesday.": [
+ 0.861,
+ 0.6499999999999999,
+ 0.43900000000000006,
+ 0.22799999999999998,
+ 0.016999999999999904,
+ -0.19399999999999995,
+ -0.405,
+ -0.616
+ ]
+ },
+ "cases": [
+ {
+ "name": "large-top-5",
+ "state": {
+ "kind": "string",
+ "value": "My card was charged twice for the same order last Tuesday."
+ },
+ "criteria": {
+ "card_arrival": "where is my new card",
+ "card_delivery_estimate": "when will the card arrive",
+ "card_not_working": "the card is declined",
+ "cash_withdrawal_charge": "charged for an ATM withdrawal",
+ "declined_card_payment": "a payment was declined",
+ "direct_debit_payment_not_recognised": "an unrecognised direct debit",
+ "exchange_rate": "what rate was applied",
+ "failed_transfer": "a transfer did not arrive",
+ "lost_or_stolen_card": "the card is gone",
+ "pending_card_payment": "a payment is still pending",
+ "refund_not_showing_up": "a refund has not arrived",
+ "request_refund": "I want money back",
+ "reverted_card_payment": "a payment was reversed",
+ "top_up_failed": "a top up did not work",
+ "transfer_fee_charged": "charged a fee for a transfer",
+ "verify_my_identity": "identity verification",
+ "wrong_amount_of_cash_received": "the ATM gave the wrong amount"
+ },
+ "k": 5,
+ "instructions": null,
+ "query_text": "My card was charged twice for the same order last Tuesday.",
+ "option_texts": [
+ "card_arrival: where is my new card",
+ "card_delivery_estimate: when will the card arrive",
+ "card_not_working: the card is declined",
+ "cash_withdrawal_charge: charged for an ATM withdrawal",
+ "declined_card_payment: a payment was declined",
+ "direct_debit_payment_not_recognised: an unrecognised direct debit",
+ "exchange_rate: what rate was applied",
+ "failed_transfer: a transfer did not arrive",
+ "lost_or_stolen_card: the card is gone",
+ "pending_card_payment: a payment is still pending",
+ "refund_not_showing_up: a refund has not arrived",
+ "request_refund: I want money back",
+ "reverted_card_payment: a payment was reversed",
+ "top_up_failed: a top up did not work",
+ "transfer_fee_charged: charged a fee for a transfer",
+ "verify_my_identity: identity verification",
+ "wrong_amount_of_cash_received: the ATM gave the wrong amount"
+ ],
+ "labels": [
+ "wrong_amount_of_cash_received",
+ "card_arrival",
+ "failed_transfer",
+ "card_not_working",
+ "card_delivery_estimate"
+ ],
+ "scores": [
+ 0.947468947226,
+ 0.907542741113,
+ 0.584010550099,
+ 0.523549809346,
+ 0.430750026379
+ ],
+ "passthrough": false,
+ "n": 17,
+ "subset_criteria": {
+ "wrong_amount_of_cash_received": "the ATM gave the wrong amount",
+ "card_arrival": "where is my new card",
+ "failed_transfer": "a transfer did not arrive",
+ "card_not_working": "the card is declined",
+ "card_delivery_estimate": "when will the card arrive"
+ }
+ },
+ {
+ "name": "large-top-1",
+ "state": {
+ "kind": "string",
+ "value": "My card was charged twice for the same order last Tuesday."
+ },
+ "criteria": {
+ "card_arrival": "where is my new card",
+ "card_delivery_estimate": "when will the card arrive",
+ "card_not_working": "the card is declined",
+ "cash_withdrawal_charge": "charged for an ATM withdrawal",
+ "declined_card_payment": "a payment was declined",
+ "direct_debit_payment_not_recognised": "an unrecognised direct debit",
+ "exchange_rate": "what rate was applied",
+ "failed_transfer": "a transfer did not arrive",
+ "lost_or_stolen_card": "the card is gone",
+ "pending_card_payment": "a payment is still pending",
+ "refund_not_showing_up": "a refund has not arrived",
+ "request_refund": "I want money back",
+ "reverted_card_payment": "a payment was reversed",
+ "top_up_failed": "a top up did not work",
+ "transfer_fee_charged": "charged a fee for a transfer",
+ "verify_my_identity": "identity verification",
+ "wrong_amount_of_cash_received": "the ATM gave the wrong amount"
+ },
+ "k": 1,
+ "instructions": null,
+ "query_text": "My card was charged twice for the same order last Tuesday.",
+ "option_texts": [
+ "card_arrival: where is my new card",
+ "card_delivery_estimate: when will the card arrive",
+ "card_not_working: the card is declined",
+ "cash_withdrawal_charge: charged for an ATM withdrawal",
+ "declined_card_payment: a payment was declined",
+ "direct_debit_payment_not_recognised: an unrecognised direct debit",
+ "exchange_rate: what rate was applied",
+ "failed_transfer: a transfer did not arrive",
+ "lost_or_stolen_card: the card is gone",
+ "pending_card_payment: a payment is still pending",
+ "refund_not_showing_up: a refund has not arrived",
+ "request_refund: I want money back",
+ "reverted_card_payment: a payment was reversed",
+ "top_up_failed: a top up did not work",
+ "transfer_fee_charged: charged a fee for a transfer",
+ "verify_my_identity: identity verification",
+ "wrong_amount_of_cash_received: the ATM gave the wrong amount"
+ ],
+ "labels": [
+ "wrong_amount_of_cash_received"
+ ],
+ "scores": [
+ 0.947468947226
+ ],
+ "passthrough": false,
+ "n": 17,
+ "subset_criteria": {
+ "wrong_amount_of_cash_received": "the ATM gave the wrong amount"
+ }
+ },
+ {
+ "name": "large-top-16-of-17",
+ "state": {
+ "kind": "string",
+ "value": "My card was charged twice for the same order last Tuesday."
+ },
+ "criteria": {
+ "card_arrival": "where is my new card",
+ "card_delivery_estimate": "when will the card arrive",
+ "card_not_working": "the card is declined",
+ "cash_withdrawal_charge": "charged for an ATM withdrawal",
+ "declined_card_payment": "a payment was declined",
+ "direct_debit_payment_not_recognised": "an unrecognised direct debit",
+ "exchange_rate": "what rate was applied",
+ "failed_transfer": "a transfer did not arrive",
+ "lost_or_stolen_card": "the card is gone",
+ "pending_card_payment": "a payment is still pending",
+ "refund_not_showing_up": "a refund has not arrived",
+ "request_refund": "I want money back",
+ "reverted_card_payment": "a payment was reversed",
+ "top_up_failed": "a top up did not work",
+ "transfer_fee_charged": "charged a fee for a transfer",
+ "verify_my_identity": "identity verification",
+ "wrong_amount_of_cash_received": "the ATM gave the wrong amount"
+ },
+ "k": 16,
+ "instructions": null,
+ "query_text": "My card was charged twice for the same order last Tuesday.",
+ "option_texts": [
+ "card_arrival: where is my new card",
+ "card_delivery_estimate: when will the card arrive",
+ "card_not_working: the card is declined",
+ "cash_withdrawal_charge: charged for an ATM withdrawal",
+ "declined_card_payment: a payment was declined",
+ "direct_debit_payment_not_recognised: an unrecognised direct debit",
+ "exchange_rate: what rate was applied",
+ "failed_transfer: a transfer did not arrive",
+ "lost_or_stolen_card: the card is gone",
+ "pending_card_payment: a payment is still pending",
+ "refund_not_showing_up: a refund has not arrived",
+ "request_refund: I want money back",
+ "reverted_card_payment: a payment was reversed",
+ "top_up_failed: a top up did not work",
+ "transfer_fee_charged: charged a fee for a transfer",
+ "verify_my_identity: identity verification",
+ "wrong_amount_of_cash_received: the ATM gave the wrong amount"
+ ],
+ "labels": [
+ "wrong_amount_of_cash_received",
+ "card_arrival",
+ "failed_transfer",
+ "card_not_working",
+ "card_delivery_estimate",
+ "direct_debit_payment_not_recognised",
+ "request_refund",
+ "verify_my_identity",
+ "exchange_rate",
+ "declined_card_payment",
+ "pending_card_payment",
+ "cash_withdrawal_charge",
+ "reverted_card_payment",
+ "top_up_failed",
+ "transfer_fee_charged",
+ "lost_or_stolen_card"
+ ],
+ "scores": [
+ 0.947468947226,
+ 0.907542741113,
+ 0.584010550099,
+ 0.523549809346,
+ 0.430750026379,
+ 0.416119355798,
+ 0.367579134772,
+ 0.298956810054,
+ 0.196754725434,
+ 0.132558511906,
+ 0.107369708039,
+ -0.133314503668,
+ -0.164844834842,
+ -0.247795808363,
+ -0.920650595907,
+ -0.947114785336
+ ],
+ "passthrough": false,
+ "n": 17,
+ "subset_criteria": {
+ "wrong_amount_of_cash_received": "the ATM gave the wrong amount",
+ "card_arrival": "where is my new card",
+ "failed_transfer": "a transfer did not arrive",
+ "card_not_working": "the card is declined",
+ "card_delivery_estimate": "when will the card arrive",
+ "direct_debit_payment_not_recognised": "an unrecognised direct debit",
+ "request_refund": "I want money back",
+ "verify_my_identity": "identity verification",
+ "exchange_rate": "what rate was applied",
+ "declined_card_payment": "a payment was declined",
+ "pending_card_payment": "a payment is still pending",
+ "cash_withdrawal_charge": "charged for an ATM withdrawal",
+ "reverted_card_payment": "a payment was reversed",
+ "top_up_failed": "a top up did not work",
+ "transfer_fee_charged": "charged a fee for a transfer",
+ "lost_or_stolen_card": "the card is gone"
+ }
+ },
+ {
+ "name": "large-k-equals-n",
+ "state": {
+ "kind": "string",
+ "value": "My card was charged twice for the same order last Tuesday."
+ },
+ "criteria": {
+ "card_arrival": "where is my new card",
+ "card_delivery_estimate": "when will the card arrive",
+ "card_not_working": "the card is declined",
+ "cash_withdrawal_charge": "charged for an ATM withdrawal",
+ "declined_card_payment": "a payment was declined",
+ "direct_debit_payment_not_recognised": "an unrecognised direct debit",
+ "exchange_rate": "what rate was applied",
+ "failed_transfer": "a transfer did not arrive",
+ "lost_or_stolen_card": "the card is gone",
+ "pending_card_payment": "a payment is still pending",
+ "refund_not_showing_up": "a refund has not arrived",
+ "request_refund": "I want money back",
+ "reverted_card_payment": "a payment was reversed",
+ "top_up_failed": "a top up did not work",
+ "transfer_fee_charged": "charged a fee for a transfer",
+ "verify_my_identity": "identity verification",
+ "wrong_amount_of_cash_received": "the ATM gave the wrong amount"
+ },
+ "k": 17,
+ "instructions": null,
+ "query_text": "My card was charged twice for the same order last Tuesday.",
+ "option_texts": [
+ "card_arrival: where is my new card",
+ "card_delivery_estimate: when will the card arrive",
+ "card_not_working: the card is declined",
+ "cash_withdrawal_charge: charged for an ATM withdrawal",
+ "declined_card_payment: a payment was declined",
+ "direct_debit_payment_not_recognised: an unrecognised direct debit",
+ "exchange_rate: what rate was applied",
+ "failed_transfer: a transfer did not arrive",
+ "lost_or_stolen_card: the card is gone",
+ "pending_card_payment: a payment is still pending",
+ "refund_not_showing_up: a refund has not arrived",
+ "request_refund: I want money back",
+ "reverted_card_payment: a payment was reversed",
+ "top_up_failed: a top up did not work",
+ "transfer_fee_charged: charged a fee for a transfer",
+ "verify_my_identity: identity verification",
+ "wrong_amount_of_cash_received: the ATM gave the wrong amount"
+ ],
+ "labels": [
+ "card_arrival",
+ "card_delivery_estimate",
+ "card_not_working",
+ "cash_withdrawal_charge",
+ "declined_card_payment",
+ "direct_debit_payment_not_recognised",
+ "exchange_rate",
+ "failed_transfer",
+ "lost_or_stolen_card",
+ "pending_card_payment",
+ "refund_not_showing_up",
+ "request_refund",
+ "reverted_card_payment",
+ "top_up_failed",
+ "transfer_fee_charged",
+ "verify_my_identity",
+ "wrong_amount_of_cash_received"
+ ],
+ "scores": null,
+ "passthrough": true,
+ "n": 17,
+ "subset_criteria": null
+ },
+ {
+ "name": "large-k-over-n",
+ "state": {
+ "kind": "string",
+ "value": "My card was charged twice for the same order last Tuesday."
+ },
+ "criteria": {
+ "card_arrival": "where is my new card",
+ "card_delivery_estimate": "when will the card arrive",
+ "card_not_working": "the card is declined",
+ "cash_withdrawal_charge": "charged for an ATM withdrawal",
+ "declined_card_payment": "a payment was declined",
+ "direct_debit_payment_not_recognised": "an unrecognised direct debit",
+ "exchange_rate": "what rate was applied",
+ "failed_transfer": "a transfer did not arrive",
+ "lost_or_stolen_card": "the card is gone",
+ "pending_card_payment": "a payment is still pending",
+ "refund_not_showing_up": "a refund has not arrived",
+ "request_refund": "I want money back",
+ "reverted_card_payment": "a payment was reversed",
+ "top_up_failed": "a top up did not work",
+ "transfer_fee_charged": "charged a fee for a transfer",
+ "verify_my_identity": "identity verification",
+ "wrong_amount_of_cash_received": "the ATM gave the wrong amount"
+ },
+ "k": 99,
+ "instructions": null,
+ "query_text": "My card was charged twice for the same order last Tuesday.",
+ "option_texts": [
+ "card_arrival: where is my new card",
+ "card_delivery_estimate: when will the card arrive",
+ "card_not_working: the card is declined",
+ "cash_withdrawal_charge: charged for an ATM withdrawal",
+ "declined_card_payment: a payment was declined",
+ "direct_debit_payment_not_recognised: an unrecognised direct debit",
+ "exchange_rate: what rate was applied",
+ "failed_transfer: a transfer did not arrive",
+ "lost_or_stolen_card: the card is gone",
+ "pending_card_payment: a payment is still pending",
+ "refund_not_showing_up: a refund has not arrived",
+ "request_refund: I want money back",
+ "reverted_card_payment: a payment was reversed",
+ "top_up_failed: a top up did not work",
+ "transfer_fee_charged: charged a fee for a transfer",
+ "verify_my_identity: identity verification",
+ "wrong_amount_of_cash_received: the ATM gave the wrong amount"
+ ],
+ "labels": [
+ "card_arrival",
+ "card_delivery_estimate",
+ "card_not_working",
+ "cash_withdrawal_charge",
+ "declined_card_payment",
+ "direct_debit_payment_not_recognised",
+ "exchange_rate",
+ "failed_transfer",
+ "lost_or_stolen_card",
+ "pending_card_payment",
+ "refund_not_showing_up",
+ "request_refund",
+ "reverted_card_payment",
+ "top_up_failed",
+ "transfer_fee_charged",
+ "verify_my_identity",
+ "wrong_amount_of_cash_received"
+ ],
+ "scores": null,
+ "passthrough": true,
+ "n": 17,
+ "subset_criteria": null
+ },
+ {
+ "name": "small-k-2",
+ "state": {
+ "kind": "string",
+ "value": "My card was charged twice for the same order last Tuesday."
+ },
+ "criteria": {
+ "refund": "money back",
+ "technical": "a bug",
+ "billing": "an invoice"
+ },
+ "k": 2,
+ "instructions": null,
+ "query_text": "My card was charged twice for the same order last Tuesday.",
+ "option_texts": [
+ "refund: money back",
+ "technical: a bug",
+ "billing: an invoice"
+ ],
+ "labels": [
+ "refund",
+ "technical"
+ ],
+ "scores": [
+ 0.304439081979,
+ -0.609827188998
+ ],
+ "passthrough": false,
+ "n": 3,
+ "subset_criteria": {
+ "refund": "money back",
+ "technical": "a bug"
+ }
+ },
+ {
+ "name": "one-option-k-1",
+ "state": {
+ "kind": "string",
+ "value": "My card was charged twice for the same order last Tuesday."
+ },
+ "criteria": {
+ "only": "the only option"
+ },
+ "k": 1,
+ "instructions": null,
+ "query_text": "My card was charged twice for the same order last Tuesday.",
+ "option_texts": [
+ "only: the only option"
+ ],
+ "labels": [
+ "only"
+ ],
+ "scores": null,
+ "passthrough": true,
+ "n": 1,
+ "subset_criteria": null
+ },
+ {
+ "name": "structured-state",
+ "state": {
+ "kind": "json",
+ "value": {
+ "subject": "duplicate charge",
+ "body": "billed twice in March"
+ }
+ },
+ "criteria": {
+ "card_arrival": "where is my new card",
+ "card_delivery_estimate": "when will the card arrive",
+ "card_not_working": "the card is declined",
+ "cash_withdrawal_charge": "charged for an ATM withdrawal",
+ "declined_card_payment": "a payment was declined",
+ "direct_debit_payment_not_recognised": "an unrecognised direct debit",
+ "exchange_rate": "what rate was applied",
+ "failed_transfer": "a transfer did not arrive",
+ "lost_or_stolen_card": "the card is gone",
+ "pending_card_payment": "a payment is still pending",
+ "refund_not_showing_up": "a refund has not arrived",
+ "request_refund": "I want money back",
+ "reverted_card_payment": "a payment was reversed",
+ "top_up_failed": "a top up did not work",
+ "transfer_fee_charged": "charged a fee for a transfer",
+ "verify_my_identity": "identity verification",
+ "wrong_amount_of_cash_received": "the ATM gave the wrong amount"
+ },
+ "k": 4,
+ "instructions": null,
+ "query_text": "{\"subject\": \"duplicate charge\", \"body\": \"billed twice in March\"}",
+ "option_texts": [
+ "card_arrival: where is my new card",
+ "card_delivery_estimate: when will the card arrive",
+ "card_not_working: the card is declined",
+ "cash_withdrawal_charge: charged for an ATM withdrawal",
+ "declined_card_payment: a payment was declined",
+ "direct_debit_payment_not_recognised: an unrecognised direct debit",
+ "exchange_rate: what rate was applied",
+ "failed_transfer: a transfer did not arrive",
+ "lost_or_stolen_card: the card is gone",
+ "pending_card_payment: a payment is still pending",
+ "refund_not_showing_up: a refund has not arrived",
+ "request_refund: I want money back",
+ "reverted_card_payment: a payment was reversed",
+ "top_up_failed: a top up did not work",
+ "transfer_fee_charged: charged a fee for a transfer",
+ "verify_my_identity: identity verification",
+ "wrong_amount_of_cash_received: the ATM gave the wrong amount"
+ ],
+ "labels": [
+ "lost_or_stolen_card",
+ "refund_not_showing_up",
+ "transfer_fee_charged",
+ "pending_card_payment"
+ ],
+ "scores": [
+ 0.772003687973,
+ 0.697649917343,
+ 0.695222725058,
+ 0.318564649233
+ ],
+ "passthrough": false,
+ "n": 17,
+ "subset_criteria": {
+ "lost_or_stolen_card": "the card is gone",
+ "refund_not_showing_up": "a refund has not arrived",
+ "transfer_fee_charged": "charged a fee for a transfer",
+ "pending_card_payment": "a payment is still pending"
+ }
+ },
+ {
+ "name": "empty-state",
+ "state": {
+ "kind": "string",
+ "value": ""
+ },
+ "criteria": {
+ "card_arrival": "where is my new card",
+ "card_delivery_estimate": "when will the card arrive",
+ "card_not_working": "the card is declined",
+ "cash_withdrawal_charge": "charged for an ATM withdrawal",
+ "declined_card_payment": "a payment was declined",
+ "direct_debit_payment_not_recognised": "an unrecognised direct debit",
+ "exchange_rate": "what rate was applied",
+ "failed_transfer": "a transfer did not arrive",
+ "lost_or_stolen_card": "the card is gone",
+ "pending_card_payment": "a payment is still pending",
+ "refund_not_showing_up": "a refund has not arrived",
+ "request_refund": "I want money back",
+ "reverted_card_payment": "a payment was reversed",
+ "top_up_failed": "a top up did not work",
+ "transfer_fee_charged": "charged a fee for a transfer",
+ "verify_my_identity": "identity verification",
+ "wrong_amount_of_cash_received": "the ATM gave the wrong amount"
+ },
+ "k": 4,
+ "instructions": null,
+ "query_text": "",
+ "option_texts": [
+ "card_arrival: where is my new card",
+ "card_delivery_estimate: when will the card arrive",
+ "card_not_working: the card is declined",
+ "cash_withdrawal_charge: charged for an ATM withdrawal",
+ "declined_card_payment: a payment was declined",
+ "direct_debit_payment_not_recognised: an unrecognised direct debit",
+ "exchange_rate: what rate was applied",
+ "failed_transfer: a transfer did not arrive",
+ "lost_or_stolen_card: the card is gone",
+ "pending_card_payment: a payment is still pending",
+ "refund_not_showing_up: a refund has not arrived",
+ "request_refund: I want money back",
+ "reverted_card_payment: a payment was reversed",
+ "top_up_failed: a top up did not work",
+ "transfer_fee_charged: charged a fee for a transfer",
+ "verify_my_identity: identity verification",
+ "wrong_amount_of_cash_received: the ATM gave the wrong amount"
+ ],
+ "labels": [
+ "pending_card_payment",
+ "verify_my_identity",
+ "reverted_card_payment",
+ "direct_debit_payment_not_recognised"
+ ],
+ "scores": [
+ 0.939301271345,
+ 0.882771141579,
+ 0.468142800581,
+ 0.46266025283
+ ],
+ "passthrough": false,
+ "n": 17,
+ "subset_criteria": {
+ "pending_card_payment": "a payment is still pending",
+ "verify_my_identity": "identity verification",
+ "reverted_card_payment": "a payment was reversed",
+ "direct_debit_payment_not_recognised": "an unrecognised direct debit"
+ }
+ },
+ {
+ "name": "none-state",
+ "state": {
+ "kind": "json",
+ "value": null
+ },
+ "criteria": {
+ "card_arrival": "where is my new card",
+ "card_delivery_estimate": "when will the card arrive",
+ "card_not_working": "the card is declined",
+ "cash_withdrawal_charge": "charged for an ATM withdrawal",
+ "declined_card_payment": "a payment was declined",
+ "direct_debit_payment_not_recognised": "an unrecognised direct debit",
+ "exchange_rate": "what rate was applied",
+ "failed_transfer": "a transfer did not arrive",
+ "lost_or_stolen_card": "the card is gone",
+ "pending_card_payment": "a payment is still pending",
+ "refund_not_showing_up": "a refund has not arrived",
+ "request_refund": "I want money back",
+ "reverted_card_payment": "a payment was reversed",
+ "top_up_failed": "a top up did not work",
+ "transfer_fee_charged": "charged a fee for a transfer",
+ "verify_my_identity": "identity verification",
+ "wrong_amount_of_cash_received": "the ATM gave the wrong amount"
+ },
+ "k": 4,
+ "instructions": null,
+ "query_text": "null",
+ "option_texts": [
+ "card_arrival: where is my new card",
+ "card_delivery_estimate: when will the card arrive",
+ "card_not_working: the card is declined",
+ "cash_withdrawal_charge: charged for an ATM withdrawal",
+ "declined_card_payment: a payment was declined",
+ "direct_debit_payment_not_recognised: an unrecognised direct debit",
+ "exchange_rate: what rate was applied",
+ "failed_transfer: a transfer did not arrive",
+ "lost_or_stolen_card: the card is gone",
+ "pending_card_payment: a payment is still pending",
+ "refund_not_showing_up: a refund has not arrived",
+ "request_refund: I want money back",
+ "reverted_card_payment: a payment was reversed",
+ "top_up_failed: a top up did not work",
+ "transfer_fee_charged: charged a fee for a transfer",
+ "verify_my_identity: identity verification",
+ "wrong_amount_of_cash_received: the ATM gave the wrong amount"
+ ],
+ "labels": [
+ "lost_or_stolen_card",
+ "refund_not_showing_up",
+ "transfer_fee_charged",
+ "pending_card_payment"
+ ],
+ "scores": [
+ 0.696734262273,
+ 0.61900567629,
+ 0.612605418621,
+ 0.359794006663
+ ],
+ "passthrough": false,
+ "n": 17,
+ "subset_criteria": {
+ "lost_or_stolen_card": "the card is gone",
+ "refund_not_showing_up": "a refund has not arrived",
+ "transfer_fee_charged": "charged a fee for a transfer",
+ "pending_card_payment": "a payment is still pending"
+ }
+ },
+ {
+ "name": "non-latin-state",
+ "state": {
+ "kind": "string",
+ "value": "注文が届きません"
+ },
+ "criteria": {
+ "card_arrival": "where is my new card",
+ "card_delivery_estimate": "when will the card arrive",
+ "card_not_working": "the card is declined",
+ "cash_withdrawal_charge": "charged for an ATM withdrawal",
+ "declined_card_payment": "a payment was declined",
+ "direct_debit_payment_not_recognised": "an unrecognised direct debit",
+ "exchange_rate": "what rate was applied",
+ "failed_transfer": "a transfer did not arrive",
+ "lost_or_stolen_card": "the card is gone",
+ "pending_card_payment": "a payment is still pending",
+ "refund_not_showing_up": "a refund has not arrived",
+ "request_refund": "I want money back",
+ "reverted_card_payment": "a payment was reversed",
+ "top_up_failed": "a top up did not work",
+ "transfer_fee_charged": "charged a fee for a transfer",
+ "verify_my_identity": "identity verification",
+ "wrong_amount_of_cash_received": "the ATM gave the wrong amount"
+ },
+ "k": 4,
+ "instructions": null,
+ "query_text": "注文が届きません",
+ "option_texts": [
+ "card_arrival: where is my new card",
+ "card_delivery_estimate: when will the card arrive",
+ "card_not_working: the card is declined",
+ "cash_withdrawal_charge: charged for an ATM withdrawal",
+ "declined_card_payment: a payment was declined",
+ "direct_debit_payment_not_recognised: an unrecognised direct debit",
+ "exchange_rate: what rate was applied",
+ "failed_transfer: a transfer did not arrive",
+ "lost_or_stolen_card: the card is gone",
+ "pending_card_payment: a payment is still pending",
+ "refund_not_showing_up: a refund has not arrived",
+ "request_refund: I want money back",
+ "reverted_card_payment: a payment was reversed",
+ "top_up_failed: a top up did not work",
+ "transfer_fee_charged: charged a fee for a transfer",
+ "verify_my_identity: identity verification",
+ "wrong_amount_of_cash_received: the ATM gave the wrong amount"
+ ],
+ "labels": [
+ "card_arrival",
+ "wrong_amount_of_cash_received",
+ "top_up_failed",
+ "request_refund"
+ ],
+ "scores": [
+ 0.330675885316,
+ 0.257026035401,
+ 0.243526050327,
+ 0.223691482192
+ ],
+ "passthrough": false,
+ "n": 17,
+ "subset_criteria": {
+ "card_arrival": "where is my new card",
+ "wrong_amount_of_cash_received": "the ATM gave the wrong amount",
+ "top_up_failed": "a top up did not work",
+ "request_refund": "I want money back"
+ }
+ },
+ {
+ "name": "with-instructions",
+ "state": {
+ "kind": "string",
+ "value": "My card was charged twice for the same order last Tuesday."
+ },
+ "criteria": {
+ "card_arrival": "where is my new card",
+ "card_delivery_estimate": "when will the card arrive",
+ "card_not_working": "the card is declined",
+ "cash_withdrawal_charge": "charged for an ATM withdrawal",
+ "declined_card_payment": "a payment was declined",
+ "direct_debit_payment_not_recognised": "an unrecognised direct debit",
+ "exchange_rate": "what rate was applied",
+ "failed_transfer": "a transfer did not arrive",
+ "lost_or_stolen_card": "the card is gone",
+ "pending_card_payment": "a payment is still pending",
+ "refund_not_showing_up": "a refund has not arrived",
+ "request_refund": "I want money back",
+ "reverted_card_payment": "a payment was reversed",
+ "top_up_failed": "a top up did not work",
+ "transfer_fee_charged": "charged a fee for a transfer",
+ "verify_my_identity": "identity verification",
+ "wrong_amount_of_cash_received": "the ATM gave the wrong amount"
+ },
+ "k": 4,
+ "instructions": "What does the customer want in `body`?",
+ "query_text": "What does the customer want in `body`?\nMy card was charged twice for the same order last Tuesday.",
+ "option_texts": [
+ "card_arrival: where is my new card",
+ "card_delivery_estimate: when will the card arrive",
+ "card_not_working: the card is declined",
+ "cash_withdrawal_charge: charged for an ATM withdrawal",
+ "declined_card_payment: a payment was declined",
+ "direct_debit_payment_not_recognised: an unrecognised direct debit",
+ "exchange_rate: what rate was applied",
+ "failed_transfer: a transfer did not arrive",
+ "lost_or_stolen_card: the card is gone",
+ "pending_card_payment: a payment is still pending",
+ "refund_not_showing_up: a refund has not arrived",
+ "request_refund: I want money back",
+ "reverted_card_payment: a payment was reversed",
+ "top_up_failed: a top up did not work",
+ "transfer_fee_charged: charged a fee for a transfer",
+ "verify_my_identity: identity verification",
+ "wrong_amount_of_cash_received: the ATM gave the wrong amount"
+ ],
+ "labels": [
+ "request_refund",
+ "exchange_rate",
+ "card_arrival",
+ "failed_transfer"
+ ],
+ "scores": [
+ 0.986196757011,
+ 0.936378299317,
+ 0.484560069972,
+ 0.44363985375
+ ],
+ "passthrough": false,
+ "n": 17,
+ "subset_criteria": {
+ "request_refund": "I want money back",
+ "exchange_rate": "what rate was applied",
+ "card_arrival": "where is my new card",
+ "failed_transfer": "a transfer did not arrive"
+ }
+ },
+ {
+ "name": "instructions-empty-string",
+ "state": {
+ "kind": "string",
+ "value": "My card was charged twice for the same order last Tuesday."
+ },
+ "criteria": {
+ "card_arrival": "where is my new card",
+ "card_delivery_estimate": "when will the card arrive",
+ "card_not_working": "the card is declined",
+ "cash_withdrawal_charge": "charged for an ATM withdrawal",
+ "declined_card_payment": "a payment was declined",
+ "direct_debit_payment_not_recognised": "an unrecognised direct debit",
+ "exchange_rate": "what rate was applied",
+ "failed_transfer": "a transfer did not arrive",
+ "lost_or_stolen_card": "the card is gone",
+ "pending_card_payment": "a payment is still pending",
+ "refund_not_showing_up": "a refund has not arrived",
+ "request_refund": "I want money back",
+ "reverted_card_payment": "a payment was reversed",
+ "top_up_failed": "a top up did not work",
+ "transfer_fee_charged": "charged a fee for a transfer",
+ "verify_my_identity": "identity verification",
+ "wrong_amount_of_cash_received": "the ATM gave the wrong amount"
+ },
+ "k": 4,
+ "instructions": "",
+ "query_text": "My card was charged twice for the same order last Tuesday.",
+ "option_texts": [
+ "card_arrival: where is my new card",
+ "card_delivery_estimate: when will the card arrive",
+ "card_not_working: the card is declined",
+ "cash_withdrawal_charge: charged for an ATM withdrawal",
+ "declined_card_payment: a payment was declined",
+ "direct_debit_payment_not_recognised: an unrecognised direct debit",
+ "exchange_rate: what rate was applied",
+ "failed_transfer: a transfer did not arrive",
+ "lost_or_stolen_card: the card is gone",
+ "pending_card_payment: a payment is still pending",
+ "refund_not_showing_up: a refund has not arrived",
+ "request_refund: I want money back",
+ "reverted_card_payment: a payment was reversed",
+ "top_up_failed: a top up did not work",
+ "transfer_fee_charged: charged a fee for a transfer",
+ "verify_my_identity: identity verification",
+ "wrong_amount_of_cash_received: the ATM gave the wrong amount"
+ ],
+ "labels": [
+ "wrong_amount_of_cash_received",
+ "card_arrival",
+ "failed_transfer",
+ "card_not_working"
+ ],
+ "scores": [
+ 0.947468947226,
+ 0.907542741113,
+ 0.584010550099,
+ 0.523549809346
+ ],
+ "passthrough": false,
+ "n": 17,
+ "subset_criteria": {
+ "wrong_amount_of_cash_received": "the ATM gave the wrong amount",
+ "card_arrival": "where is my new card",
+ "failed_transfer": "a transfer did not arrive",
+ "card_not_working": "the card is declined"
+ }
+ },
+ {
+ "name": "no-descriptions",
+ "state": {
+ "kind": "string",
+ "value": "My card was charged twice for the same order last Tuesday."
+ },
+ "criteria": {
+ "a": null,
+ "b": null,
+ "c": null,
+ "d": null
+ },
+ "k": 2,
+ "instructions": null,
+ "query_text": "My card was charged twice for the same order last Tuesday.",
+ "option_texts": [
+ "a",
+ "b",
+ "c",
+ "d"
+ ],
+ "labels": [
+ "c",
+ "a"
+ ],
+ "scores": [
+ 0.304439081979,
+ -0.123272321994
+ ],
+ "passthrough": false,
+ "n": 4,
+ "subset_criteria": {
+ "c": null,
+ "a": null
+ }
+ },
+ {
+ "name": "list-criteria",
+ "state": {
+ "kind": "string",
+ "value": "My card was charged twice for the same order last Tuesday."
+ },
+ "criteria": [
+ "alpha",
+ "beta",
+ "gamma",
+ "delta"
+ ],
+ "k": 2,
+ "instructions": null,
+ "query_text": "My card was charged twice for the same order last Tuesday.",
+ "option_texts": [
+ "alpha",
+ "beta",
+ "gamma",
+ "delta"
+ ],
+ "labels": [
+ "delta",
+ "beta"
+ ],
+ "scores": [
+ 0.426002656466,
+ 0.128698817888
+ ],
+ "passthrough": false,
+ "n": 4,
+ "subset_criteria": [
+ "delta",
+ "beta"
+ ]
+ }
+ ],
+ "cosine": [
+ {
+ "name": "ordinary",
+ "query": [
+ 1.0,
+ 0.0
+ ],
+ "docs": [
+ [
+ 1.0,
+ 0.0
+ ],
+ [
+ 0.0,
+ 1.0
+ ],
+ [
+ -1.0,
+ 0.0
+ ]
+ ],
+ "sims": [
+ 1.0,
+ 0.0,
+ -1.0
+ ]
+ },
+ {
+ "name": "zero-query",
+ "query": [
+ 0.0,
+ 0.0
+ ],
+ "docs": [
+ [
+ 1.0,
+ 0.0
+ ],
+ [
+ 0.0,
+ 1.0
+ ]
+ ],
+ "sims": [
+ 0.0,
+ 0.0
+ ]
+ },
+ {
+ "name": "zero-doc",
+ "query": [
+ 1.0,
+ 0.0
+ ],
+ "docs": [
+ [
+ 0.0,
+ 0.0
+ ],
+ [
+ 1.0,
+ 0.0
+ ]
+ ],
+ "sims": [
+ 0.0,
+ 1.0
+ ]
+ },
+ {
+ "name": "all-zero",
+ "query": [
+ 0.0,
+ 0.0
+ ],
+ "docs": [
+ [
+ 0.0,
+ 0.0
+ ]
+ ],
+ "sims": [
+ 0.0
+ ]
+ },
+ {
+ "name": "negatives",
+ "query": [
+ 1.0,
+ 1.0
+ ],
+ "docs": [
+ [
+ -1.0,
+ -1.0
+ ],
+ [
+ 1.0,
+ 1.0
+ ],
+ [
+ 0.0,
+ 0.0
+ ]
+ ],
+ "sims": [
+ -1.0,
+ 1.0,
+ 0.0
+ ]
+ },
+ {
+ "name": "needs-clipping",
+ "query": [
+ 1e-200,
+ 1e-200
+ ],
+ "docs": [
+ [
+ 1e-200,
+ 1e-200
+ ]
+ ],
+ "sims": [
+ 0.0
+ ]
+ },
+ {
+ "name": "large-magnitudes",
+ "query": [
+ 1e+200,
+ 1e+200
+ ],
+ "docs": [
+ [
+ 1e+200,
+ 1e+200
+ ],
+ [
+ -1e+200,
+ 1e+200
+ ]
+ ],
+ "sims": [
+ "nan",
+ "nan"
+ ]
+ },
+ {
+ "name": "no-docs",
+ "query": [
+ 1.0,
+ 1.0
+ ],
+ "docs": [],
+ "sims": []
+ },
+ {
+ "name": "underflowing-query",
+ "query": [
+ 1e-200,
+ 1e-200
+ ],
+ "docs": [
+ [
+ 1.0,
+ 1.0
+ ]
+ ],
+ "sims": [
+ 0.0
+ ]
+ },
+ {
+ "name": "self-cosine-over-one",
+ "query": [
+ 0.3333333333333333,
+ 0.3333333333333333,
+ 2.6666666666666665
+ ],
+ "docs": [
+ [
+ 0.3333333333333333,
+ 0.3333333333333333,
+ 2.6666666666666665
+ ]
+ ],
+ "sims": [
+ 1.0
+ ]
+ }
+ ],
+ "nonfinite": [
+ {
+ "name": "nan",
+ "rows": [
+ [
+ "nan",
+ 1.0
+ ],
+ [
+ 1.0,
+ 0.0
+ ]
+ ],
+ "cleaned": [
+ [
+ 0.0,
+ 1.0
+ ],
+ [
+ 1.0,
+ 0.0
+ ]
+ ]
+ },
+ {
+ "name": "posinf",
+ "rows": [
+ [
+ "inf",
+ 1.0
+ ],
+ [
+ 1.0,
+ 0.0
+ ]
+ ],
+ "cleaned": [
+ [
+ 0.0,
+ 1.0
+ ],
+ [
+ 1.0,
+ 0.0
+ ]
+ ]
+ },
+ {
+ "name": "neginf",
+ "rows": [
+ [
+ "-inf",
+ 1.0
+ ],
+ [
+ 1.0,
+ 0.0
+ ]
+ ],
+ "cleaned": [
+ [
+ 0.0,
+ 1.0
+ ],
+ [
+ 1.0,
+ 0.0
+ ]
+ ]
+ },
+ {
+ "name": "all-nan",
+ "rows": [
+ [
+ "nan",
+ "nan"
+ ],
+ [
+ 1.0,
+ 0.0
+ ]
+ ],
+ "cleaned": [
+ [
+ 0.0,
+ 0.0
+ ],
+ [
+ 1.0,
+ 0.0
+ ]
+ ]
+ }
+ ],
+ "refusals": [
+ {
+ "name": "k-zero",
+ "error": "ValueError",
+ "message": "k must be a positive integer, got 0"
+ },
+ {
+ "name": "k-negative",
+ "error": "ValueError",
+ "message": "k must be a positive integer, got -1"
+ },
+ {
+ "name": "k-bool",
+ "error": "ValueError",
+ "message": "k must be a positive integer, got True"
+ },
+ {
+ "name": "criteria-empty-dict",
+ "error": "ValueError",
+ "message": "choice criteria must contain at least one option"
+ },
+ {
+ "name": "criteria-empty-list",
+ "error": "ValueError",
+ "message": "choice criteria must contain at least one option"
+ },
+ {
+ "name": "criteria-wrong-type",
+ "error": "TypeError",
+ "message": "choice criteria must be a dict or list, got str"
+ },
+ {
+ "name": "criteria-duplicate-label",
+ "error": "ValueError",
+ "message": "choice criteria label 'a' is duplicated"
+ },
+ {
+ "name": "embed-not-callable",
+ "error": "TypeError",
+ "message": "embed_fn must be callable"
+ },
+ {
+ "name": "embed-wrong-rows",
+ "error": "ValueError",
+ "message": "embed_fn must return an array of shape (4, dim), got (1, 1)"
+ },
+ {
+ "name": "embed-one-dimensional",
+ "error": "ValueError",
+ "message": "embed_fn must return an array of shape (4, dim), got (4,)"
+ },
+ {
+ "name": "questions-not-dict",
+ "error": "TypeError",
+ "message": "questions must be a dict of question id -> definition"
+ },
+ {
+ "name": "choice-without-criteria",
+ "error": "ValueError",
+ "message": "question 'q' is a choice but has no criteria"
+ }
+ ],
+ "predicts": [
+ {
+ "name": "choice-reduced",
+ "k": 4,
+ "questions": {
+ "intent": {
+ "type": "choice",
+ "instructions": "What does the customer want?",
+ "criteria": {
+ "card_arrival": "where is my new card",
+ "card_delivery_estimate": "when will the card arrive",
+ "card_not_working": "the card is declined",
+ "cash_withdrawal_charge": "charged for an ATM withdrawal",
+ "declined_card_payment": "a payment was declined",
+ "direct_debit_payment_not_recognised": "an unrecognised direct debit",
+ "exchange_rate": "what rate was applied",
+ "failed_transfer": "a transfer did not arrive",
+ "lost_or_stolen_card": "the card is gone",
+ "pending_card_payment": "a payment is still pending",
+ "refund_not_showing_up": "a refund has not arrived",
+ "request_refund": "I want money back",
+ "reverted_card_payment": "a payment was reversed",
+ "top_up_failed": "a top up did not work",
+ "transfer_fee_charged": "charged a fee for a transfer",
+ "verify_my_identity": "identity verification",
+ "wrong_amount_of_cash_received": "the ATM gave the wrong amount"
+ }
+ }
+ },
+ "asked": {
+ "intent": {
+ "type": "choice",
+ "instructions": "What does the customer want?",
+ "criteria": {
+ "pending_card_payment": "a payment is still pending",
+ "verify_my_identity": "identity verification",
+ "cash_withdrawal_charge": "charged for an ATM withdrawal",
+ "direct_debit_payment_not_recognised": "an unrecognised direct debit"
+ }
+ }
+ },
+ "shortlist": {
+ "intent": {
+ "labels": [
+ "pending_card_payment",
+ "verify_my_identity",
+ "cash_withdrawal_charge",
+ "direct_debit_payment_not_recognised"
+ ],
+ "scores": [
+ 0.9979763443195673,
+ 0.7960748581967693,
+ 0.47010577346806354,
+ 0.3953627184004995
+ ],
+ "k": 4,
+ "n": 17,
+ "passthrough": false
+ }
+ },
+ "caller_questions_unmutated": true
+ },
+ {
+ "name": "choice-passthrough",
+ "k": 20,
+ "questions": {
+ "intent": {
+ "type": "choice",
+ "instructions": "x",
+ "criteria": {
+ "refund": "money back",
+ "technical": "a bug",
+ "billing": "an invoice"
+ }
+ }
+ },
+ "asked": {
+ "intent": {
+ "type": "choice",
+ "instructions": "x",
+ "criteria": {
+ "refund": "money back",
+ "technical": "a bug",
+ "billing": "an invoice"
+ }
+ }
+ },
+ "shortlist": {
+ "intent": {
+ "labels": [
+ "refund",
+ "technical",
+ "billing"
+ ],
+ "scores": null,
+ "k": 20,
+ "n": 3,
+ "passthrough": true
+ }
+ },
+ "caller_questions_unmutated": true
+ },
+ {
+ "name": "mixed-questions",
+ "k": 3,
+ "questions": {
+ "intent": {
+ "type": "choice",
+ "instructions": "What?",
+ "criteria": {
+ "card_arrival": "where is my new card",
+ "card_delivery_estimate": "when will the card arrive",
+ "card_not_working": "the card is declined",
+ "cash_withdrawal_charge": "charged for an ATM withdrawal",
+ "declined_card_payment": "a payment was declined",
+ "direct_debit_payment_not_recognised": "an unrecognised direct debit",
+ "exchange_rate": "what rate was applied",
+ "failed_transfer": "a transfer did not arrive",
+ "lost_or_stolen_card": "the card is gone",
+ "pending_card_payment": "a payment is still pending",
+ "refund_not_showing_up": "a refund has not arrived",
+ "request_refund": "I want money back",
+ "reverted_card_payment": "a payment was reversed",
+ "top_up_failed": "a top up did not work",
+ "transfer_fee_charged": "charged a fee for a transfer",
+ "verify_my_identity": "identity verification",
+ "wrong_amount_of_cash_received": "the ATM gave the wrong amount"
+ }
+ },
+ "urgent": {
+ "type": "noul",
+ "instructions": "Is this urgent?"
+ },
+ "severity": {
+ "type": "score",
+ "instructions": "How bad?",
+ "criteria": [
+ "low",
+ "high"
+ ]
+ }
+ },
+ "asked": {
+ "intent": {
+ "type": "choice",
+ "instructions": "What?",
+ "criteria": {
+ "cash_withdrawal_charge": "charged for an ATM withdrawal",
+ "pending_card_payment": "a payment is still pending",
+ "verify_my_identity": "identity verification"
+ }
+ },
+ "urgent": {
+ "type": "noul",
+ "instructions": "Is this urgent?"
+ },
+ "severity": {
+ "type": "score",
+ "instructions": "How bad?",
+ "criteria": [
+ "low",
+ "high"
+ ]
+ }
+ },
+ "shortlist": {
+ "intent": {
+ "labels": [
+ "cash_withdrawal_charge",
+ "pending_card_payment",
+ "verify_my_identity"
+ ],
+ "scores": [
+ 0.9528938726697158,
+ 0.47144784517409144,
+ 0.44400321698018963
+ ],
+ "k": 3,
+ "n": 17,
+ "passthrough": false
+ }
+ },
+ "caller_questions_unmutated": true
+ },
+ {
+ "name": "two-choices",
+ "k": 2,
+ "questions": {
+ "a": {
+ "type": "choice",
+ "instructions": "A?",
+ "criteria": {
+ "card_arrival": "where is my new card",
+ "card_delivery_estimate": "when will the card arrive",
+ "card_not_working": "the card is declined",
+ "cash_withdrawal_charge": "charged for an ATM withdrawal",
+ "declined_card_payment": "a payment was declined",
+ "direct_debit_payment_not_recognised": "an unrecognised direct debit",
+ "exchange_rate": "what rate was applied",
+ "failed_transfer": "a transfer did not arrive",
+ "lost_or_stolen_card": "the card is gone",
+ "pending_card_payment": "a payment is still pending",
+ "refund_not_showing_up": "a refund has not arrived",
+ "request_refund": "I want money back",
+ "reverted_card_payment": "a payment was reversed",
+ "top_up_failed": "a top up did not work",
+ "transfer_fee_charged": "charged a fee for a transfer",
+ "verify_my_identity": "identity verification",
+ "wrong_amount_of_cash_received": "the ATM gave the wrong amount"
+ }
+ },
+ "b": {
+ "type": "choice",
+ "instructions": "B?",
+ "criteria": {
+ "refund": "money back",
+ "technical": "a bug",
+ "billing": "an invoice"
+ }
+ }
+ },
+ "asked": {
+ "a": {
+ "type": "choice",
+ "instructions": "A?",
+ "criteria": {
+ "lost_or_stolen_card": "the card is gone",
+ "refund_not_showing_up": "a refund has not arrived"
+ }
+ },
+ "b": {
+ "type": "choice",
+ "instructions": "B?",
+ "criteria": {
+ "technical": "a bug",
+ "billing": "an invoice"
+ }
+ }
+ },
+ "shortlist": {
+ "a": {
+ "labels": [
+ "lost_or_stolen_card",
+ "refund_not_showing_up"
+ ],
+ "scores": [
+ 0.5259653204777385,
+ 0.45371261735814744
+ ],
+ "k": 2,
+ "n": 17,
+ "passthrough": false
+ },
+ "b": {
+ "labels": [
+ "technical",
+ "billing"
+ ],
+ "scores": [
+ 0.3577362690770829,
+ 0.12186924644473365
+ ],
+ "k": 2,
+ "n": 3,
+ "passthrough": false
+ }
+ },
+ "caller_questions_unmutated": true
+ }
+ ]
+}
diff --git a/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/Agent.java b/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/Agent.java
index 29f6eaeb..2c013107 100644
--- a/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/Agent.java
+++ b/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/Agent.java
@@ -40,7 +40,7 @@ import java.util.Map;
* <p>One session is not safe for concurrent {@code predict} calls unless ONNX Runtime is
* configured for it; hold one {@code Agent} per worker, or serialise access.
*/
-public final class Agent implements AutoCloseable {
+public final class Agent implements AutoCloseable, Predictor {
private final Tokenizer tokenizer;
private final AgentConfig config;
diff --git a/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/Predictor.java b/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/Predictor.java
new file mode 100644
index 00000000..f34992cc
--- /dev/null
+++ b/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/Predictor.java
@@ -0,0 +1,23 @@
+package com.convaiinnovations.laya;
+
+import java.util.Map;
+
+/**
+ * Anything that can answer a set of questions about a state.
+ *
+ * <p>Implemented by both {@link Agent}, which asks one checkpoint, and {@link Router}, which picks
+ * the checkpoint first. The distinction matters to a caller choosing between them and not at all
+ * to code that merely needs an answer -- {@link Shortlist} takes this rather than one or the
+ * other, so shortlisting works the same whether the questions go to a fixed checkpoint or a routed
+ * one.
+ *
+ * <p>Deliberately the narrow overload. {@code Agent} also takes a language and {@code Router} also
+ * takes routing arguments, and neither belongs here: a {@code Router} chooses the language itself
+ * from what it detected, so an interface carrying one would invite a caller to override the very
+ * thing routing had just worked out.
+ */
+public interface Predictor {
+
+ /** Ask every question about one state. */
+ Prediction predict(Object state, Map<String, Question> questions);
+}
diff --git a/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/Router.java b/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/Router.java
index bf96792a..e34bd6e9 100644
--- a/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/Router.java
+++ b/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/Router.java
@@ -39,7 +39,7 @@ import java.util.concurrent.locks.ReentrantLock;
* <p>This class decides; it does not load. {@link #route} runs no model and touches no disk, so it
* is safe to call on every request and to test without a checkpoint.
*/
-public final class Router implements AutoCloseable {
+public final class Router implements AutoCloseable, Predictor {
/** The hub repository that bundles all three checkpoints. */
public static final String BUNDLE_REPO = "convaiinnovations/laya";
diff --git a/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/Shortlist.java b/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/Shortlist.java
new file mode 100644
index 00000000..5630203e
--- /dev/null
+++ b/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/Shortlist.java
@@ -0,0 +1,510 @@
+package com.convaiinnovations.laya;
+
+import com.convaiinnovations.laya.json.PythonJson;
+import com.convaiinnovations.laya.sequence.SequenceBuilder;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.locks.ReentrantLock;
+
+/**
+ * Opt-in embedding shortlist for a choice question with many labels. A port of
+ * {@code laya.shortlist}.
+ *
+ * <p>Every option of a choice shares one {@code head_max_len} budget, so a large label set leaves
+ * only a few tokens per label and the model is shown truncated options. Shortlisting embeds the
+ * state and each option with a caller-supplied embedder, keeps the top {@code k}, and then runs a
+ * single {@link Predictor#predict} over that reduced set. It adds no second decision pass: the
+ * model still scores every criterion it is handed, there are just fewer of them.
+ *
+ * <p>The coarse-to-fine pattern is the one the project's README recommends. Ranking is cosine
+ * similarity on whatever vectors the embedder returns; the quality of the shortlist is the
+ * embedder's business, and nothing here measures it.
+ *
+ * <p><b>Four rules decide which labels survive</b>, and each is a place a reimplementation goes
+ * wrong quietly rather than loudly:
+ *
+ * <ul>
+ * <li>The sort is <b>stable</b> and descending, so a tie keeps the earlier label. An unstable
+ * sort reorders ties and silently changes which labels survive the cut.
+ * <li>The score is a <b>signed</b> cosine, not a similarity floor. A label scoring zero -- no
+ * signal, or a non-finite vector treated as none -- outranks one scoring negative, and
+ * {@code k} drops the negatives first.
+ * <li>A zero-norm query scores everything zero, so the kept set is the first {@code k} in
+ * criteria order rather than an arbitrary one.
+ * <li>{@code k} at or above the label count is a <b>passthrough</b>: the labels come back in
+ * criteria order, there are no scores, and <b>the embedder is never called</b>. A caller
+ * paying per embedding call cares about that last part.
+ * </ul>
+ *
+ * <p>{@code embed_fn_from_agent} is deliberately not ported. It mean-pools the checkpoint's own
+ * encoder, which needs the encoder's hidden states; the ONNX graph this runtime loads exposes
+ * logits, not hidden states. A dedicated bi-encoder passed as an {@link Embedder} shortlists
+ * better anyway, which is what the reference says of its own helper.
+ */
+public final class Shortlist {
+
+ /** The reference's default: keep twenty labels. */
+ public static final int DEFAULT_K = 20;
+
+ private Shortlist() {
+ }
+
+ /**
+ * Maps texts to vectors, one row per text, every row the same width.
+ *
+ * <p>Called once per shortlisted question, with the query text first and then one string per
+ * option in criteria order.
+ */
+ @FunctionalInterface
+ public interface Embedder {
+
+ /** Embed these texts. The result must be {@code texts.size()} rows of equal width. */
+ double[][] embed(List<String> texts);
+ }
+
+ /**
+ * What a ranking decided.
+ *
+ * @param labels the kept labels in rank order, or every label in criteria order on a
+ * passthrough
+ * @param scores the signed cosine of each kept label in that order, or null when nothing
+ * was dropped -- negative values included, never clamped to zero
+ * @param passthrough whether {@code k} covered every label, in which case nothing was embedded
+ * @param total how many labels the question had
+ */
+ public record Ranking(List<String> labels, double[] scores, boolean passthrough, int total) {
+
+ public Ranking {
+ labels = List.copyOf(labels);
+ scores = scores == null ? null : scores.clone();
+ }
+
+ /** The scores, copied, or null when nothing was dropped. */
+ @Override
+ public double[] scores() {
+ return scores == null ? null : scores.clone();
+ }
+ }
+
+ /** A prediction over shortlisted questions, and what each shortlist decided. */
+ public record Shortlisted(Prediction prediction, Map<String, Ranking> shortlist) {
+
+ public Shortlisted {
+ shortlist = Collections.unmodifiableMap(new LinkedHashMap<>(shortlist));
+ }
+ }
+
+ /** The top {@code k} labels of a choice question for this state. */
+ public static List<String> choice(Object state, Question question, Embedder embedder, int k) {
+ return rank(state, question, embedder, k).labels();
+ }
+
+ /** The top {@code k} of these labels for this state, with no instruction in the query. */
+ public static List<String> choice(Object state, Map<String, ?> criteria, Embedder embedder,
+ int k) {
+ return rank(state, criteria, embedder, k, null).labels();
+ }
+
+ /**
+ * Rank a choice question's labels for this state, keeping the best {@code k}.
+ *
+ * <p>The question's instruction goes into the query text, which is what the reference does
+ * when one is supplied.
+ *
+ * @throws IllegalArgumentException if {@code k} is below one, or the question is not a choice
+ */
+ public static Ranking rank(Object state, Question question, Embedder embedder, int k) {
+ requireChoice(question);
+ return rank(state, question.instructions(), criteriaOf(question), question.renderOptions(),
+ embedder, k);
+ }
+
+ /** Rank these labels for this state, with no instruction in the query. */
+ public static Ranking rank(Object state, Map<String, ?> criteria, Embedder embedder, int k) {
+ return rank(state, criteria, embedder, k, null);
+ }
+
+ /**
+ * Rank these labels for this state, optionally prefixing the query with an instruction.
+ *
+ * <p>The entry point that takes criteria rather than a {@link Question}, because the reference
+ * takes the two separately and its instruction is OPTIONAL -- its default is to embed the
+ * state alone. A typed {@code Question} always carries a non-blank instruction, so the
+ * {@code Question} overload cannot express that case, and shortlisting a label set a caller
+ * has not yet turned into a question is a reasonable thing to want.
+ *
+ * @param instructions prefixed to the query text when present; null or empty embeds the state
+ * alone, which is the reference's default
+ */
+ public static Ranking rank(Object state, Map<String, ?> criteria, Embedder embedder, int k,
+ String instructions) {
+ Map<String, Object> options = new LinkedHashMap<>();
+ if (criteria == null) {
+ throw new IllegalArgumentException("criteria must not be null");
+ }
+ options.putAll(criteria);
+ if (options.isEmpty()) {
+ throw new IllegalArgumentException("choice criteria must contain at least one option");
+ }
+ // Rendered through a Question rather than by a second copy of the same rule: the option
+ // text is what gets embedded, so a divergence here would silently rank against different
+ // strings than the model is shown. The instruction below is not used for rendering.
+ List<String> optionTexts = Question.choice("placeholder", options).renderOptions();
+ return rank(state, instructions, options, optionTexts, embedder, k);
+ }
+
+ /**
+ * Shortlist every choice question, then ask all of them in one prediction.
+ *
+ * <p>A non-choice question is forwarded unchanged, and so is a choice whose label count is at
+ * or below {@code k} -- which also means its embedder is never called. The caller's map is not
+ * modified.
+ *
+ * <p>Probabilities on a shortlisted choice are over the kept labels only. That is the point
+ * and worth stating: the model was never shown the dropped ones, so its distribution cannot
+ * include them.
+ */
+ public static Shortlisted predict(Predictor predictor, Object state,
+ Map<String, Question> questions, Embedder embedder, int k) {
+ if (predictor == null) {
+ throw new IllegalArgumentException("predictor must not be null");
+ }
+ if (questions == null) {
+ throw new IllegalArgumentException("questions must not be null");
+ }
+ int checked = requirePositive(k, "k");
+ Map<String, Question> reduced = new LinkedHashMap<>();
+ Map<String, Ranking> meta = new LinkedHashMap<>();
+ for (Map.Entry<String, Question> entry : questions.entrySet()) {
+ String id = entry.getKey();
+ Question question = entry.getValue();
+ if (question == null || question.type() != Question.Type.CHOICE) {
+ reduced.put(id, question);
+ continue;
+ }
+ Map<String, Object> criteria = criteriaOf(question);
+ Ranking ranking = rank(state, question.instructions(), criteria,
+ question.renderOptions(), embedder, checked);
+ meta.put(id, ranking);
+ if (ranking.passthrough()) {
+ reduced.put(id, question);
+ continue;
+ }
+ Map<String, Object> subset = new LinkedHashMap<>();
+ for (String label : ranking.labels()) {
+ subset.put(label, criteria.get(label));
+ }
+ reduced.put(id, Question.choice(question.instructions(), subset));
+ }
+ return new Shortlisted(predictor.predict(state, reduced), meta);
+ }
+
+ /** Shortlist with the reference's default {@code k}. */
+ public static Shortlisted predict(Predictor predictor, Object state,
+ Map<String, Question> questions, Embedder embedder) {
+ return predict(predictor, state, questions, embedder, DEFAULT_K);
+ }
+
+ // ------------------------------------------------------------------ the ranking
+
+ /** The one implementation. {@code instructions} may be null, which embeds the state alone. */
+ private static Ranking rank(Object state, String instructions, Map<String, Object> criteria,
+ List<String> optionTexts, Embedder embedder, int k) {
+ int checked = requirePositive(k, "k");
+ List<String> labels = new ArrayList<>(criteria.keySet());
+ int total = labels.size();
+ if (total == 0) {
+ throw new IllegalArgumentException("choice criteria must contain at least one option");
+ }
+ if (checked >= total) {
+ // Passthrough. The embedder is NOT called, which a caller paying per call relies on.
+ return new Ranking(labels, null, true, total);
+ }
+ List<String> texts = new ArrayList<>(total + 1);
+ texts.add(queryText(state, instructions));
+ texts.addAll(optionTexts);
+ double[][] matrix = embeddings(embedder, texts);
+ double[] sims = cosine(matrix[0], matrix, 1);
+
+ // A stable descending sort: ties keep the earlier label, which decides which labels
+ // survive the cut whenever two score the same -- and with a zero-norm query every label
+ // scores the same.
+ Integer[] order = new Integer[total];
+ for (int i = 0; i < total; i++) {
+ order[i] = i;
+ }
+ Arrays.sort(order, (left, right) -> Double.compare(sims[right], sims[left]));
+
+ List<String> kept = new ArrayList<>(checked);
+ double[] scores = new double[checked];
+ for (int i = 0; i < checked; i++) {
+ int index = order[i];
+ kept.add(labels.get(index));
+ scores[i] = sims[index];
+ }
+ return new Ranking(kept, scores, false, total);
+ }
+
+ /** The text the query vector is taken from: the instruction, then the serialised state. */
+ static String queryText(Object state, String instructions) {
+ String body = SequenceBuilder.serializeState(state);
+ if (instructions == null || instructions.isEmpty()) {
+ return body;
+ }
+ return instructions + "\n" + body;
+ }
+
+ /**
+ * The embedder's output, validated and made finite.
+ *
+ * <p>A non-finite component becomes zero rather than propagating into every score or raising:
+ * one bad row must cost that label its ranking, not the whole request.
+ */
+ static double[][] embeddings(Embedder embedder, List<String> texts) {
+ if (embedder == null) {
+ throw new IllegalArgumentException("embedder must not be null");
+ }
+ double[][] raw = embedder.embed(List.copyOf(texts));
+ if (raw == null || raw.length != texts.size()) {
+ throw new IllegalArgumentException(String.format(
+ "the embedder must return %d rows, got %s",
+ texts.size(), raw == null ? "null" : Integer.toString(raw.length)));
+ }
+ int width = -1;
+ double[][] out = new double[raw.length][];
+ for (int row = 0; row < raw.length; row++) {
+ double[] values = raw[row];
+ if (values == null || values.length < 1) {
+ throw new IllegalArgumentException(
+ "the embedder returned an empty row at index " + row);
+ }
+ if (width < 0) {
+ width = values.length;
+ } else if (values.length != width) {
+ throw new IllegalArgumentException(String.format(
+ "the embedder returned rows of different widths: %d then %d at index %d",
+ width, values.length, row));
+ }
+ out[row] = new double[width];
+ for (int i = 0; i < width; i++) {
+ double value = values[i];
+ out[row][i] = Double.isFinite(value) ? value : 0.0;
+ }
+ }
+ return out;
+ }
+
+ /**
+ * Signed cosine of {@code query} against each row of {@code docs} from {@code from} onwards.
+ *
+ * <p>Zero-norm on either side scores zero -- not an error and not a skip, because every label
+ * has to get a comparable number. Clamped to [-1, 1] because floating-point division can step
+ * just outside it.
+ */
+ static double[] cosine(double[] query, double[][] docs, int from) {
+ int count = docs.length - from;
+ double[] sims = new double[Math.max(0, count)];
+ double queryNorm = norm(query);
+ if (queryNorm == 0.0 || count <= 0) {
+ return sims;
+ }
+ for (int i = 0; i < count; i++) {
+ double[] doc = docs[from + i];
+ double denominator = norm(doc) * queryNorm;
+ if (!(denominator > 0.0)) {
+ continue;
+ }
+ double dot = 0.0;
+ for (int j = 0; j < Math.min(query.length, doc.length); j++) {
+ dot += doc[j] * query[j];
+ }
+ double value = dot / denominator;
+ // NaN survives on purpose: it is what the reference produces when a norm overflows to
+ // infinity, and clamping it to a number would invent a ranking the reference does not
+ // have. Math.min/max would do exactly that, so the comparison is explicit.
+ if (Double.isNaN(value)) {
+ sims[i] = value;
+ } else {
+ sims[i] = value < -1.0 ? -1.0 : (value > 1.0 ? 1.0 : value);
+ }
+ }
+ return sims;
+ }
+
+ private static double norm(double[] vector) {
+ double total = 0.0;
+ for (double value : vector) {
+ total += value * value;
+ }
+ return Math.sqrt(total);
+ }
+
+ private static void requireChoice(Question question) {
+ if (question == null) {
+ throw new IllegalArgumentException("question must not be null");
+ }
+ if (question.type() != Question.Type.CHOICE) {
+ throw new IllegalArgumentException(
+ "only a choice question has labels to shortlist, not " + question.type());
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private static Map<String, Object> criteriaOf(Question question) {
+ Object criteria = question.spec().get("criteria");
+ if (!(criteria instanceof Map)) {
+ throw new IllegalArgumentException("this choice question has no criteria to shortlist");
+ }
+ return (Map<String, Object>) criteria;
+ }
+
+ private static int requirePositive(int value, String what) {
+ if (value < 1) {
+ throw new IllegalArgumentException(what + " must be at least 1, got " + value);
+ }
+ return value;
+ }
+
+ // ------------------------------------------------------------------ caching
+
+ /**
+ * An {@link Embedder} that remembers each text it has embedded, under an LRU bound.
+ *
+ * <p>Shortlisting embeds the query plus every option on every call. When the option set is
+ * fixed -- a label list, an intent taxonomy -- those rows never change between requests and
+ * are re-embedded every time. Wrapping the embedder once leaves the first call unchanged and
+ * reduces each repeat to embedding the new query alone.
+ *
+ * <p>A cold cache costs the same number of embedder calls as the unwrapped function: the
+ * misses are deduplicated and embedded in one call. Nothing is cached when the embedder
+ * throws or returns a bad shape, so a transient failure cannot poison the cache.
+ *
+ * <p>Safe to share between threads. The lock covers cache reads and writes only, never the
+ * embedding call, so one slow embed does not block every other caller's cache hits.
+ */
+ public static final class CachedEmbedder implements Embedder {
+
+ private final Embedder delegate;
+ private final int maxSize;
+ private final LinkedHashMap<String, double[]> rows;
+ private final ReentrantLock lock = new ReentrantLock();
+ private long hits;
+ private long misses;
+
+ private CachedEmbedder(Embedder delegate, int maxSize) {
+ this.delegate = delegate;
+ this.maxSize = maxSize;
+ // accessOrder = true makes this an LRU: a get moves the entry to the end.
+ this.rows = new LinkedHashMap<>(16, 0.75f, true);
+ }
+
+ @Override
+ public double[][] embed(List<String> texts) {
+ List<String> keys = new ArrayList<>(texts.size());
+ for (String text : texts) {
+ keys.add(text == null ? "" : text);
+ }
+ double[][] out = new double[keys.size()][];
+ Set<String> wanted = new LinkedHashSet<>();
+ lock.lock();
+ try {
+ for (int i = 0; i < keys.size(); i++) {
+ double[] row = rows.get(keys.get(i));
+ if (row != null) {
+ out[i] = row;
+ hits++;
+ } else {
+ wanted.add(keys.get(i));
+ }
+ }
+ misses += wanted.size();
+ } finally {
+ lock.unlock();
+ }
+ if (wanted.isEmpty()) {
+ return out;
+ }
+ // Deduplicated and embedded in ONE call, outside the lock, so a cold cache costs what
+ // the unwrapped embedder costs and a slow embed does not block other callers' hits.
+ List<String> toEmbed = new ArrayList<>(wanted);
+ double[][] fresh = embeddings(delegate, toEmbed);
+ lock.lock();
+ try {
+ for (int i = 0; i < toEmbed.size(); i++) {
+ rows.put(toEmbed.get(i), fresh[i]);
+ }
+ while (rows.size() > maxSize) {
+ rows.remove(rows.keySet().iterator().next());
+ }
+ } finally {
+ lock.unlock();
+ }
+ Map<String, double[]> byText = new LinkedHashMap<>();
+ for (int i = 0; i < toEmbed.size(); i++) {
+ byText.put(toEmbed.get(i), fresh[i]);
+ }
+ for (int i = 0; i < keys.size(); i++) {
+ if (out[i] == null) {
+ out[i] = byText.get(keys.get(i));
+ }
+ }
+ return out;
+ }
+
+ /** How many texts are cached, the bound, and the hit and miss counts. */
+ public Map<String, Long> cacheInfo() {
+ lock.lock();
+ try {
+ Map<String, Long> info = new LinkedHashMap<>();
+ info.put("size", (long) rows.size());
+ info.put("maxsize", (long) maxSize);
+ info.put("hits", hits);
+ info.put("misses", misses);
+ return Collections.unmodifiableMap(info);
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ /** Forget everything. Call this if the weights behind the embedder change. */
+ public void cacheClear() {
+ lock.lock();
+ try {
+ rows.clear();
+ hits = 0;
+ misses = 0;
+ } finally {
+ lock.unlock();
+ }
+ }
+ }
+
+ /** Wrap an embedder so each text is embedded once, under the reference's default bound. */
+ public static CachedEmbedder cached(Embedder embedder) {
+ return cached(embedder, 4096);
+ }
+
+ /** Wrap an embedder so each text is embedded once, keeping at most {@code maxSize} of them. */
+ public static CachedEmbedder cached(Embedder embedder, int maxSize) {
+ if (embedder == null) {
+ throw new IllegalArgumentException("embedder must not be null");
+ }
+ return new CachedEmbedder(embedder, requirePositive(maxSize, "maxSize"));
+ }
+
+ /** The query text for a question, exposed so a caller can see what is being embedded. */
+ public static String queryTextFor(Object state, Question question) {
+ return queryText(state, question == null ? null : question.instructions());
+ }
+
+ /** What the reference writes for a non-string criterion, used when rendering an option. */
+ static String renderCriterion(Object value) {
+ return value instanceof String ? (String) value : PythonJson.dumps(value);
+ }
+}
diff --git a/laya-java/laya-java/src/test/java/com/convaiinnovations/laya/ShortlistTest.java b/laya-java/laya-java/src/test/java/com/convaiinnovations/laya/ShortlistTest.java
new file mode 100644
index 00000000..1704dfb4
--- /dev/null
+++ b/laya-java/laya-java/src/test/java/com/convaiinnovations/laya/ShortlistTest.java
@@ -0,0 +1,712 @@
+package com.convaiinnovations.laya;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.DynamicTest;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestFactory;
+
+/**
+ * {@link Shortlist} against the rankings recorded from {@code laya.shortlist}.
+ *
+ * <p>A real bi-encoder cannot go in a fixture, and it is not what needs gating: the embedder is
+ * the caller's, and what the module decides is the ranking around it. So both sides embed with the
+ * same deterministic stand-in -- FNV-1a over the UTF-8 bytes of {@code text:component}, mapped into
+ * [-1, 1) -- which is reproducible in any language with 64-bit integers, with no floating point,
+ * no library and no seeded generator whose stream differs between runtimes.
+ *
+ * <p>What that lets the fixture pin is every rule that decides which labels survive: the stable
+ * tie order, the signed cosine, the zero-norm query, the non-finite cleaning, and the passthrough
+ * that must not call the embedder at all.
+ */
+class ShortlistTest {
+
+ private static final int EMBED_DIM = 8;
+ private static final long FNV_OFFSET = 0xCBF29CE484222325L;
+ private static final long FNV_PRIME = 0x100000001B3L;
+
+ private static Map<String, Object> fixture() {
+ return Fixtures.load("shortlist.json");
+ }
+
+ @SuppressWarnings("unchecked")
+ private static Map<String, Object> map(Object value) {
+ return (Map<String, Object>) value;
+ }
+
+ @SuppressWarnings("unchecked")
+ private static List<Object> list(Object value) {
+ return (List<Object>) value;
+ }
+
+ /**
+ * The stand-in embedder, byte for byte what the generator used.
+ *
+ * <p>{@code Long.remainderUnsigned} rather than {@code %}: the hash is a 64-bit unsigned value
+ * and Java's remainder is signed, so half the components would come out negative and every
+ * ranking would differ.
+ */
+ private static double[][] stubEmbed(List<String> texts) {
+ double[][] rows = new double[texts.size()][EMBED_DIM];
+ for (int row = 0; row < texts.size(); row++) {
+ String text = texts.get(row) == null ? "" : texts.get(row);
+ for (int component = 0; component < EMBED_DIM; component++) {
+ byte[] key = (text + ":" + component).getBytes(StandardCharsets.UTF_8);
+ long hash = FNV_OFFSET;
+ for (byte value : key) {
+ hash = (hash ^ (value & 0xFFL)) * FNV_PRIME;
+ }
+ rows[row][component] = Long.remainderUnsigned(hash, 2000L) / 1000.0 - 1.0;
+ }
+ }
+ return rows;
+ }
+
+ /**
+ * A recorded cosine against a computed one, to the precision the fixture records.
+ *
+ * <p>The one tolerance in this suite, and it is not a convenience. The reference computes its
+ * scores with a BLAS matrix-vector product, and how BLAS blocks that accumulation differs
+ * between implementations -- measured on this project: with Apple Accelerate, one row of a
+ * four-row product differs in the last bit from the same row computed alone, and CI's
+ * OpenBLAS blocks it differently again. So the reference's own score is not reproducible
+ * across machines and no port can match it bit for bit.
+ *
+ * <p>The fixture therefore records twelve decimals, which is roughly four orders of magnitude
+ * tighter than any porting error this repository has seen and four orders looser than the
+ * 1e-16 BLAS spread. What is still compared exactly is the label ORDER -- the only thing a
+ * consumer observes, and the thing a ranking bug actually moves.
+ */
+ private static void assertScore(double expected, double actual, String what) {
+ double tolerance = 5e-13;
+ assertTrue(Math.abs(expected - actual) <= tolerance,
+ what + ": expected " + expected + " +/- " + tolerance + ", got " + actual
+ + " (difference " + Math.abs(expected - actual) + ")");
+ }
+
+ /** A double from the fixture, where a non-finite value is tagged as a string. */
+ private static double number(Object value) {
+ if (value instanceof String) {
+ switch ((String) value) {
+ case "nan":
+ return Double.NaN;
+ case "inf":
+ return Double.POSITIVE_INFINITY;
+ case "-inf":
+ return Double.NEGATIVE_INFINITY;
+ default:
+ throw new IllegalArgumentException("not a tagged number: " + value);
+ }
+ }
+ return ((Number) value).doubleValue();
+ }
+
+ private static Object state(Map<String, Object> encoded) {
+ return encoded.get("value");
+ }
+
+ /**
+ * The recorded criteria as a choice question.
+ *
+ * <p>The reference accepts a bare list of labels as criteria; a typed {@link Question} always
+ * has a map, so a list becomes a map with null descriptions -- which is exactly what the
+ * reference does with it internally, and the recorded option texts prove it.
+ */
+ private static Map<String, Object> criteriaOf(Object criteria) {
+ Map<String, Object> options = new LinkedHashMap<>();
+ if (criteria instanceof List) {
+ for (Object label : list(criteria)) {
+ options.put((String) label, null);
+ }
+ } else {
+ options.putAll(map(criteria));
+ }
+ return options;
+ }
+
+ @Test
+ @DisplayName("the stand-in embedder matches the one the fixture was recorded with")
+ void embedderMatchesTheGenerator() {
+ // Checked first and on its own: if this drifts, every ranking below fails for a reason
+ // that has nothing to do with the ranking, and the failures would be unreadable.
+ Map<String, Object> probe = map(fixture().get("embed_probe"));
+ assertEquals(EMBED_DIM, ((Number) fixture().get("embed_dim")).intValue());
+ for (Map.Entry<String, Object> entry : probe.entrySet()) {
+ double[] expected = new double[EMBED_DIM];
+ List<Object> recorded = list(entry.getValue());
+ for (int i = 0; i < EMBED_DIM; i++) {
+ expected[i] = number(recorded.get(i));
+ }
+ double[] actual = stubEmbed(List.of(entry.getKey()))[0];
+ for (int i = 0; i < EMBED_DIM; i++) {
+ assertEquals(expected[i], actual[i],
+ "component " + i + " of " + PythonJsonQuote.of(entry.getKey()));
+ }
+ }
+ assertTrue(probe.size() >= 3, "only " + probe.size() + " probe texts");
+ }
+
+ @TestFactory
+ @DisplayName("every recorded ranking matches, label order and score")
+ List<DynamicTest> rankings() {
+ List<DynamicTest> tests = new ArrayList<>();
+ for (Object entry : list(fixture().get("cases"))) {
+ Map<String, Object> row = map(entry);
+ String name = (String) row.get("name");
+ tests.add(DynamicTest.dynamicTest(name, () -> assertCase(name, row)));
+ }
+ return tests;
+ }
+
+ private void assertCase(String name, Map<String, Object> row) {
+ Object recordedState = state(map(row.get("state")));
+ String instructions = (String) row.get("instructions");
+ Map<String, Object> criteria = criteriaOf(row.get("criteria"));
+ int k = ((Number) row.get("k")).intValue();
+
+ // The two inputs to the ranking, asserted before the ranking itself, so a failure says
+ // which of the three stages broke rather than only that the answer differs.
+ assertEquals(row.get("query_text"),
+ Shortlist.queryText(recordedState,
+ instructions == null || instructions.isEmpty() ? null : instructions),
+ name + ": the query text");
+ List<String> expectedOptions = new ArrayList<>();
+ for (Object text : list(row.get("option_texts"))) {
+ expectedOptions.add((String) text);
+ }
+ assertEquals(expectedOptions, Question.choice("placeholder", criteria).renderOptions(),
+ name + ": the option texts");
+
+ AtomicInteger calls = new AtomicInteger();
+ Shortlist.Embedder counting = texts -> {
+ calls.incrementAndGet();
+ return stubEmbed(texts);
+ };
+ // The criteria overload, because a recorded case may have had no instruction at all --
+ // which is the reference's default and which the Question overload cannot express.
+ Shortlist.Ranking ranking = Shortlist.rank(recordedState, criteria, counting, k,
+ instructions);
+
+ List<String> expectedLabels = new ArrayList<>();
+ for (Object label : list(row.get("labels"))) {
+ expectedLabels.add((String) label);
+ }
+ assertEquals(expectedLabels, ranking.labels(), name + ": the kept labels, in rank order");
+ assertEquals(row.get("passthrough"), ranking.passthrough(), name + ": passthrough");
+ assertEquals(((Number) row.get("n")).intValue(), ranking.total(), name + ": label count");
+
+ if (Boolean.TRUE.equals(row.get("passthrough"))) {
+ assertNull(ranking.scores(), name + ": a passthrough reports no scores");
+ assertEquals(0, calls.get(),
+ name + ": a passthrough must NOT call the embedder, and called it "
+ + calls.get() + " times");
+ } else {
+ List<Object> recordedScores = list(row.get("scores"));
+ double[] scores = ranking.scores();
+ assertNotNull(scores, name + ": scores");
+ assertEquals(recordedScores.size(), scores.length, name + ": score count");
+ for (int i = 0; i < scores.length; i++) {
+ assertScore(number(recordedScores.get(i)), scores[i],
+ name + ": score " + i + " (" + ranking.labels().get(i) + ")");
+ }
+ assertEquals(1, calls.get(), name + ": the embedder is called exactly once");
+ // The reduced criteria the model will be shown: the kept labels, in rank order. The
+ // reference records that mapping separately, so this checks the order survives the
+ // subsetting and is not merely the criteria order again.
+ // The reference subsets a dict to a dict and a list to a list, so the recorded shape
+ // follows the criteria it was given. Both carry the same thing: the kept labels in
+ // rank order.
+ Object recordedSubset = row.get("subset_criteria");
+ List<String> subsetLabels = new ArrayList<>();
+ if (recordedSubset instanceof Map) {
+ for (Object label : map(recordedSubset).keySet()) {
+ subsetLabels.add((String) label);
+ }
+ } else {
+ for (Object label : list(recordedSubset)) {
+ subsetLabels.add((String) label);
+ }
+ }
+ assertEquals(subsetLabels, ranking.labels(),
+ name + ": the reduced criteria are the kept labels in rank order");
+ }
+ }
+
+ @TestFactory
+ @DisplayName("the cosine matches on every vector shape, including the ones that overflow")
+ List<DynamicTest> cosine() {
+ List<DynamicTest> tests = new ArrayList<>();
+ int nonFinite = 0;
+ for (Object entry : list(fixture().get("cosine"))) {
+ Map<String, Object> row = map(entry);
+ String name = (String) row.get("name");
+ List<Object> recordedSims = list(row.get("sims"));
+ for (Object value : recordedSims) {
+ if (value instanceof String) {
+ nonFinite++;
+ }
+ }
+ tests.add(DynamicTest.dynamicTest(name, () -> {
+ List<Object> queryValues = list(row.get("query"));
+ double[] query = new double[queryValues.size()];
+ for (int i = 0; i < query.length; i++) {
+ query[i] = number(queryValues.get(i));
+ }
+ List<Object> docRows = list(row.get("docs"));
+ double[][] matrix = new double[docRows.size() + 1][];
+ matrix[0] = query;
+ for (int r = 0; r < docRows.size(); r++) {
+ List<Object> values = list(docRows.get(r));
+ matrix[r + 1] = new double[values.size()];
+ for (int c = 0; c < values.size(); c++) {
+ matrix[r + 1][c] = number(values.get(c));
+ }
+ }
+ double[] sims = Shortlist.cosine(query, matrix, 1);
+ assertEquals(recordedSims.size(), sims.length, name + ": count");
+ for (int i = 0; i < sims.length; i++) {
+ double expected = number(recordedSims.get(i));
+ if (Double.isNaN(expected)) {
+ assertTrue(Double.isNaN(sims[i]),
+ name + ": sim " + i + " must be NaN, got " + sims[i]);
+ } else {
+ assertScore(expected, sims[i], name + ": sim " + i);
+ }
+ }
+ }));
+ }
+ assertTrue(nonFinite >= 1, "no recorded cosine is non-finite, so the case where a norm"
+ + " overflows to infinity is not covered -- and clamping that to a number would"
+ + " invent a ranking the reference does not have");
+ return tests;
+ }
+
+ @TestFactory
+ @DisplayName("a non-finite component becomes zero rather than spreading or raising")
+ List<DynamicTest> nonFiniteCleaning() {
+ List<DynamicTest> tests = new ArrayList<>();
+ for (Object entry : list(fixture().get("nonfinite"))) {
+ Map<String, Object> row = map(entry);
+ String name = (String) row.get("name");
+ tests.add(DynamicTest.dynamicTest(name, () -> {
+ List<Object> rawRows = list(row.get("rows"));
+ double[][] raw = new double[rawRows.size()][];
+ for (int r = 0; r < rawRows.size(); r++) {
+ List<Object> values = list(rawRows.get(r));
+ raw[r] = new double[values.size()];
+ for (int c = 0; c < values.size(); c++) {
+ raw[r][c] = number(values.get(c));
+ }
+ }
+ double[][] cleaned = Shortlist.embeddings(texts -> raw, List.of("q", "d"));
+ List<Object> expectedRows = list(row.get("cleaned"));
+ for (int r = 0; r < expectedRows.size(); r++) {
+ List<Object> values = list(expectedRows.get(r));
+ for (int c = 0; c < values.size(); c++) {
+ assertEquals(number(values.get(c)), cleaned[r][c],
+ name + ": row " + r + " component " + c);
+ }
+ }
+ }));
+ }
+ return tests;
+ }
+
+ @TestFactory
+ @DisplayName("what predict hands the model is the reduced question set")
+ List<DynamicTest> predictAsksTheReducedSet() {
+ List<DynamicTest> tests = new ArrayList<>();
+ for (Object entry : list(fixture().get("predicts"))) {
+ Map<String, Object> row = map(entry);
+ String name = (String) row.get("name");
+ tests.add(DynamicTest.dynamicTest(name, () -> {
+ int k = ((Number) row.get("k")).intValue();
+ Map<String, Object> declared = map(row.get("questions"));
+ Map<String, Question> questions = new LinkedHashMap<>();
+ for (Map.Entry<String, Object> q : declared.entrySet()) {
+ questions.put(q.getKey(), fromSpec(map(q.getValue())));
+ }
+ Map<String, Question> original = new LinkedHashMap<>(questions);
+
+ List<Map<String, Question>> asked = new ArrayList<>();
+ Predictor recorder = (s, qs) -> {
+ asked.add(new LinkedHashMap<>(qs));
+ return new Prediction("stub", Map.of(),
+ new Usage(0, 0, 0, 0, false, List.of(), Map.of()));
+ };
+ Shortlist.Shortlisted result = Shortlist.predict(recorder,
+ "My card was charged twice for the same order last Tuesday.",
+ questions, ShortlistTest::stubEmbed, k);
+
+ assertEquals(1, asked.size(), name + ": exactly one prediction, never one per"
+ + " question -- shortlisting must not add a second pass");
+ Map<String, Question> seen = asked.get(0);
+ Map<String, Object> expectedAsked = map(row.get("asked"));
+ assertEquals(new ArrayList<>(expectedAsked.keySet()), new ArrayList<>(seen.keySet()),
+ name + ": the questions asked, and their order");
+ for (Map.Entry<String, Object> expected : expectedAsked.entrySet()) {
+ Map<String, Object> spec = map(expected.getValue());
+ Question actual = seen.get(expected.getKey());
+ assertEquals(spec.get("type"), actual.type().wireName(),
+ name + "." + expected.getKey() + ": type");
+ Object criteria = spec.get("criteria");
+ if (criteria instanceof Map) {
+ assertEquals(new ArrayList<>(map(criteria).keySet()),
+ new ArrayList<>(map(actual.spec().get("criteria")).keySet()),
+ name + "." + expected.getKey() + ": the criteria kept, in order");
+ }
+ }
+ Map<String, Object> expectedMeta = map(row.get("shortlist"));
+ assertEquals(expectedMeta.keySet(), result.shortlist().keySet(),
+ name + ": which questions were shortlisted");
+ for (Map.Entry<String, Object> expected : expectedMeta.entrySet()) {
+ Map<String, Object> meta = map(expected.getValue());
+ Shortlist.Ranking ranking = result.shortlist().get(expected.getKey());
+ assertEquals(((Number) meta.get("n")).intValue(), ranking.total());
+ assertEquals(meta.get("passthrough"), ranking.passthrough());
+ List<String> labels = new ArrayList<>();
+ for (Object label : list(meta.get("labels"))) {
+ labels.add((String) label);
+ }
+ assertEquals(labels, ranking.labels(),
+ name + "." + expected.getKey() + ": shortlist labels");
+ }
+ assertEquals(original, questions, name + ": the caller's map was modified");
+ }));
+ }
+ return tests;
+ }
+
+ private static Question fromSpec(Map<String, Object> spec) {
+ String type = (String) spec.get("type");
+ String instructions = (String) spec.get("instructions");
+ Object criteria = spec.get("criteria");
+ if ("choice".equals(type)) {
+ return Question.choice(instructions, map(criteria));
+ }
+ if ("score".equals(type)) {
+ return Question.score(instructions, list(criteria));
+ }
+ return Question.noul(instructions);
+ }
+
+ @Test
+ @DisplayName("the inputs the reference refuses are refused here too, or cannot be written")
+ void refusals() {
+ Map<String, Object> small = new LinkedHashMap<>();
+ small.put("refund", "money back");
+ small.put("technical", "a bug");
+ small.put("billing", "an invoice");
+ Question question = Question.choice("What?", small);
+
+ // Refused at run time, as the reference refuses them.
+ assertThrows(IllegalArgumentException.class,
+ () -> Shortlist.rank("s", question, ShortlistTest::stubEmbed, 0), "k = 0");
+ assertThrows(IllegalArgumentException.class,
+ () -> Shortlist.rank("s", question, ShortlistTest::stubEmbed, -1), "k = -1");
+ assertThrows(IllegalArgumentException.class,
+ () -> Shortlist.rank("s", question, null, 2), "a null embedder");
+ assertThrows(IllegalArgumentException.class,
+ () -> Shortlist.rank("s", question, texts -> new double[1][1], 2),
+ "too few rows");
+ assertThrows(IllegalArgumentException.class,
+ () -> Shortlist.rank("s", question, texts -> new double[texts.size()][0], 2),
+ "zero-width rows");
+ assertThrows(IllegalArgumentException.class,
+ () -> Shortlist.rank("s", question, texts -> null, 2), "a null result");
+ assertThrows(IllegalArgumentException.class,
+ () -> Shortlist.rank("s", Question.noul("Is this urgent?"),
+ ShortlistTest::stubEmbed, 2),
+ "a noul has no labels to shortlist");
+ assertThrows(IllegalArgumentException.class,
+ () -> Shortlist.cached(ShortlistTest::stubEmbed, 0), "a cache bound of zero");
+
+ // Refused by ragged rows, which the reference cannot even see: numpy would have made the
+ // array object-dtype and failed the shape check instead.
+ assertThrows(IllegalArgumentException.class,
+ () -> Shortlist.rank("s", question, texts -> {
+ double[][] rows = new double[texts.size()][];
+ for (int i = 0; i < rows.length; i++) {
+ rows[i] = new double[i + 1];
+ }
+ return rows;
+ }, 2), "rows of different widths");
+
+ // And the ones the type system refuses before a test can run: a bool k, criteria that are
+ // neither a map nor a list, a duplicated label (a Map cannot hold one), a one-dimensional
+ // embedding, and questions that are not a map. Each is a compile error here, which is
+ // strictly earlier than the reference's run-time check.
+ }
+
+ @Test
+ @DisplayName("a zero-norm query keeps the first k labels in criteria order")
+ void zeroNormQueryKeepsCriteriaOrder() {
+ // Every score ties at zero, so the only thing deciding which labels survive is the
+ // stability of the sort. An unstable sort passes every other test in this class and fails
+ // this one.
+ Map<String, Object> criteria = new LinkedHashMap<>();
+ for (String label : List.of("a", "b", "c", "d", "e", "f")) {
+ criteria.put(label, label + " description");
+ }
+ Question question = Question.choice("What?", criteria);
+ Shortlist.Ranking ranking = Shortlist.rank("state", question,
+ texts -> new double[texts.size()][EMBED_DIM], 3);
+ assertEquals(List.of("a", "b", "c"), ranking.labels());
+ for (double score : ranking.scores()) {
+ assertEquals(0.0, score);
+ }
+ }
+
+ @Test
+ @DisplayName("a query whose norm underflows scores exactly zero, not almost zero")
+ void underflowingQueryScoresExactlyZero() {
+ // Asserted exactly, and separately from the recorded scores, because the rounded
+ // comparison cannot see it: without the zero-norm guard the score is 1.4e-200, which
+ // agrees with 0 to twelve decimals and to any tolerance worth having. A norm of zero does
+ // NOT imply a dot product of zero once components underflow, which is the trap.
+ double[] query = {1e-200, 1e-200};
+ double[][] matrix = {query, {1.0, 1.0}};
+ assertEquals(0.0, Shortlist.cosine(query, matrix, 1)[0],
+ "an underflowing query must score zero outright");
+ }
+
+ @Test
+ @DisplayName("a cosine computing above one is clamped to exactly one")
+ void cosineIsClampedToOne() {
+ // The self-cosine of this vector computes to 1.0000000000000002 with sequential
+ // arithmetic, so the clamp is observable -- and again only exactly: the unclamped value
+ // agrees with 1.0 to twelve decimals. Asserted on an exact equality for that reason.
+ double[] vector = {1.0 / 3.0, 1.0 / 3.0, 8.0 / 3.0};
+ double[][] matrix = {vector, vector};
+ double[] sims = Shortlist.cosine(vector, matrix, 1);
+ assertEquals(1.0, sims[0], "a cosine above one must be clamped, not reported");
+ // and the other side of the clamp
+ double[] opposed = {-vector[0], -vector[1], -vector[2]};
+ assertEquals(-1.0, Shortlist.cosine(vector, new double[][] {vector, opposed}, 1)[0],
+ "and a cosine below minus one must be clamped too");
+ }
+
+ @Test
+ @DisplayName("a label scoring zero outranks one scoring negative")
+ void zeroOutranksNegative() {
+ // The signed-cosine rule. A similarity floor that clamped negatives to zero would tie
+ // these and then keep the earlier label, which is the opposite answer.
+ Map<String, Object> criteria = new LinkedHashMap<>();
+ criteria.put("opposite", "x");
+ criteria.put("orthogonal", "y");
+ Question question = Question.choice("What?", criteria);
+ Shortlist.Ranking ranking = Shortlist.rank("state", question, texts -> new double[][] {
+ {1.0, 0.0}, // the query
+ {-1.0, 0.0}, // opposite: cosine -1
+ {0.0, 0.0}, // orthogonal: zero norm, so zero
+ }, 1);
+ assertEquals(List.of("orthogonal"), ranking.labels());
+ assertEquals(0.0, ranking.scores()[0]);
+ }
+
+ @Test
+ @DisplayName("the cache embeds each text once and leaves the first call unchanged")
+ void cacheEmbedsEachTextOnce() {
+ AtomicInteger calls = new AtomicInteger();
+ List<Integer> batchSizes = Collections.synchronizedList(new ArrayList<>());
+ Shortlist.Embedder counting = texts -> {
+ calls.incrementAndGet();
+ batchSizes.add(texts.size());
+ return stubEmbed(texts);
+ };
+ Shortlist.CachedEmbedder cached = Shortlist.cached(counting, 64);
+
+ double[][] first = cached.embed(List.of("q1", "a", "b", "c"));
+ assertEquals(1, calls.get(), "a cold cache costs one call, as the unwrapped embedder does");
+ assertEquals(4, batchSizes.get(0));
+
+ double[][] second = cached.embed(List.of("q2", "a", "b", "c"));
+ assertEquals(2, calls.get());
+ assertEquals(1, batchSizes.get(1),
+ "a repeat must embed the new query alone, not the options again");
+ for (int row = 1; row < 4; row++) {
+ assertSame(first[row], second[row], "the cached row must be the one handed back");
+ }
+
+ Map<String, Long> info = cached.cacheInfo();
+ assertEquals(5L, info.get("size"), "q1, a, b, c and q2 -- the second query is cached too");
+ assertEquals(64L, info.get("maxsize"));
+ assertEquals(3L, info.get("hits"));
+ assertEquals(5L, info.get("misses"));
+
+ cached.cacheClear();
+ assertEquals(0L, cached.cacheInfo().get("size"));
+ assertEquals(0L, cached.cacheInfo().get("hits"));
+ }
+
+ @Test
+ @DisplayName("the cache deduplicates within one call")
+ void cacheDeduplicatesWithinOneCall() {
+ List<Integer> batchSizes = new ArrayList<>();
+ Shortlist.CachedEmbedder cached = Shortlist.cached(texts -> {
+ batchSizes.add(texts.size());
+ return stubEmbed(texts);
+ }, 64);
+ double[][] rows = cached.embed(List.of("a", "a", "a", "b"));
+ assertEquals(List.of(2), batchSizes, "four texts, two distinct, one embedding call of two");
+ assertSame(rows[0], rows[1], "the repeats share one row");
+ assertSame(rows[0], rows[2]);
+ assertFalse(rows[0] == rows[3]);
+ }
+
+ @Test
+ @DisplayName("the cache evicts least-recently-used past its bound")
+ void cacheEvictsLru() {
+ AtomicInteger calls = new AtomicInteger();
+ Shortlist.CachedEmbedder cached = Shortlist.cached(texts -> {
+ calls.incrementAndGet();
+ return stubEmbed(texts);
+ }, 2);
+ cached.embed(List.of("a"));
+ cached.embed(List.of("b"));
+ cached.embed(List.of("a")); // a is now the most recent
+ cached.embed(List.of("c")); // evicts b
+ assertEquals(2L, cached.cacheInfo().get("size"));
+ int before = calls.get();
+ cached.embed(List.of("a"));
+ assertEquals(before, calls.get(), "a was kept, so this is a hit");
+ cached.embed(List.of("b"));
+ assertEquals(before + 1, calls.get(), "b was evicted, so this is a miss");
+ }
+
+ @Test
+ @DisplayName("a failing embedder caches nothing, so a transient failure cannot poison it")
+ void aFailingEmbedderCachesNothing() {
+ AtomicInteger attempts = new AtomicInteger();
+ Shortlist.CachedEmbedder cached = Shortlist.cached(texts -> {
+ if (attempts.incrementAndGet() == 1) {
+ throw new IllegalStateException("transient");
+ }
+ return stubEmbed(texts);
+ }, 64);
+ assertThrows(IllegalStateException.class, () -> cached.embed(List.of("a")));
+ assertEquals(0L, cached.cacheInfo().get("size"), "nothing may be cached from a failure");
+ double[][] rows = cached.embed(List.of("a"));
+ assertEquals(EMBED_DIM, rows[0].length, "and the retry succeeds");
+ assertEquals(1L, cached.cacheInfo().get("size"));
+ }
+
+ @Test
+ @DisplayName("the cache is safe to share between threads")
+ void cacheIsThreadSafe() throws Exception {
+ AtomicInteger calls = new AtomicInteger();
+ Shortlist.CachedEmbedder cached = Shortlist.cached(texts -> {
+ calls.incrementAndGet();
+ return stubEmbed(texts);
+ }, 1024);
+ int threads = 8;
+ CountDownLatch go = new CountDownLatch(1);
+ List<Throwable> failures = Collections.synchronizedList(new ArrayList<>());
+ List<Thread> workers = new ArrayList<>();
+ for (int t = 0; t < threads; t++) {
+ Thread worker = new Thread(() -> {
+ try {
+ assertTrue(go.await(10, TimeUnit.SECONDS));
+ for (int i = 0; i < 50; i++) {
+ double[][] rows = cached.embed(List.of("shared-" + (i % 10)));
+ assertEquals(EMBED_DIM, rows[0].length);
+ }
+ } catch (Throwable failure) {
+ failures.add(failure);
+ }
+ }, "cache-" + t);
+ worker.start();
+ workers.add(worker);
+ }
+ go.countDown();
+ for (Thread worker : workers) {
+ worker.join(TimeUnit.SECONDS.toMillis(20));
+ assertFalse(worker.isAlive(), worker.getName() + " never finished");
+ }
+ assertEquals(List.of(), failures);
+ assertEquals(10L, cached.cacheInfo().get("size"), "ten distinct texts, however they raced");
+ }
+
+ @Test
+ @DisplayName("shortlisting works against a Router as well as an Agent")
+ void worksAgainstEitherPredictor() {
+ // The reason Predictor exists. The reference accepts anything with predict or system_one;
+ // here one interface covers both, so a caller routing between checkpoints shortlists the
+ // same way as one asking a fixed checkpoint.
+ assertTrue(Predictor.class.isAssignableFrom(Agent.class),
+ "Agent must be usable as a Predictor");
+ assertTrue(Predictor.class.isAssignableFrom(Router.class),
+ "Router must be usable as a Predictor");
+ }
+
+ @Test
+ @DisplayName("the default k is the reference's")
+ void defaultK() {
+ assertEquals(((Number) fixture().get("default_k")).intValue(), Shortlist.DEFAULT_K);
+ }
+
+ @Test
+ @DisplayName("the recorded scores are rounded far below the tolerance they are checked at")
+ void scorePrecisionIsHonest() {
+ // Guards the one tolerance in this suite from quietly widening. The fixture rounds to
+ // twelve decimals and the comparison allows 5e-13 -- half of the last recorded digit --
+ // so the tolerance admits nothing the rounding did not already discard. If either number
+ // moves without the other, this fails rather than letting a looser check pass unnoticed.
+ int decimals = ((Number) fixture().get("score_decimals")).intValue();
+ assertEquals(12, decimals, "the fixture's recorded precision changed");
+ double halfOfLastDigit = 0.5 * Math.pow(10, -decimals);
+ assertEquals(5e-13, halfOfLastDigit, 1e-18,
+ "the tolerance in assertScore must stay half of the last recorded digit");
+ // and the recorded values really are rounded, not full precision stored at a width that
+ // happens to look rounded
+ int checked = 0;
+ for (Object entry : list(fixture().get("cases"))) {
+ Object scores = map(entry).get("scores");
+ if (scores == null) {
+ continue;
+ }
+ for (Object score : list(scores)) {
+ if (score instanceof String) {
+ continue;
+ }
+ // Via the shortest decimal that round-trips, not by scaling: multiplying a
+ // 12-decimal value by 1e12 is not exact in binary, and the first version of this
+ // check failed on its own arithmetic rather than on the data.
+ double value = ((Number) score).doubleValue();
+ int scale = new java.math.BigDecimal(Double.toString(value)).stripTrailingZeros()
+ .scale();
+ assertTrue(scale <= decimals,
+ "a recorded score carries " + scale + " decimals, more than " + decimals
+ + ": " + value);
+ checked++;
+ }
+ }
+ assertTrue(checked >= 40, "only " + checked + " recorded scores checked");
+ }
+
+ /** Quotes a string for a failure message without pulling in the JSON writer's rules. */
+ private static final class PythonJsonQuote {
+ private PythonJsonQuote() {
+ }
+
+ static String of(String value) {
+ return "\"" + value.replace("\"", "\\\"") + "\"";
+ }
+ }
+}
diff --git a/laya-java/scripts/gen_fixtures.py b/laya-java/scripts/gen_fixtures.py
index 483d12ff..050469c4 100644
--- a/laya-java/scripts/gen_fixtures.py
+++ b/laya-java/scripts/gen_fixtures.py
@@ -791,6 +791,297 @@ def router():
"question_schema": [[schema, mod._question_schema(schema)] for schema in schemas],
"routes": [dict(route_case(options), name=name) for name, options in cases],
}
+# A deterministic stand-in for a real embedder, so the ranking can be gated across languages.
+# FNV-1a over the UTF-8 bytes of "text:component", mapped into [-1, 1). Chosen because it is
+# exactly reproducible in any language with 64-bit integer arithmetic -- no floating point, no
+# library, no seeded PRNG whose stream differs between runtimes. A real bi-encoder cannot be put
+# in a fixture; what needs gating is the ranking, the tie order and the passthrough rule, and those
+# do not care where the vectors came from.
+_FNV_OFFSET = 0xCBF29CE484222325
+_FNV_PRIME = 0x100000001B3
+_FNV_MASK = 0xFFFFFFFFFFFFFFFF
+_EMBED_DIM = 8
+
+
+def _fnv1a(data: bytes) -> int:
+ value = _FNV_OFFSET
+ for byte in data:
+ value = ((value ^ byte) * _FNV_PRIME) & _FNV_MASK
+ return value
+
+
+def _stub_embed(texts):
+ import numpy as np
+
+ rows = []
+ for text in texts:
+ row = []
+ for component in range(_EMBED_DIM):
+ key = ("%s:%d" % ("" if text is None else text, component)).encode("utf-8")
+ row.append((_fnv1a(key) % 2000) / 1000.0 - 1.0)
+ rows.append(row)
+ return np.asarray(rows, dtype=np.float64)
+
+
+class _FakeAgent:
+ """Records what `predict_shortlist` actually asked, which is the thing to gate."""
+
+ def __init__(self):
+ self.seen = None
+
+ def predict(self, state, questions, **kwargs):
+ self.seen = {
+ "questions": {qid: dict(qdef) for qid, qdef in questions.items()},
+ "kwargs": {k: v for k, v in sorted(kwargs.items())},
+ }
+ return {"answers": {qid: {"stub": True} for qid in questions}, "model": "stub"}
+
+
+def shortlist():
+ """`laya.shortlist`'s ranking, its edge cases, and what it hands to `predict`.
+
+ Choice options share one `head_max_len`, so a large label set leaves a few tokens per label.
+ Shortlisting embeds the state and each option, keeps the top k, and runs ONE predict on the
+ reduced set. The parts that need gating are not the embedding -- a caller supplies that -- but
+ the ranking rules around it, and each of them is a place a port goes wrong silently:
+
+ * the sort is STABLE and descending, so a tie keeps the earlier label. `argsort` without
+ `kind="mergesort"` reorders ties, which changes which labels survive at the cut.
+ * the score is a SIGNED cosine, not a similarity floor. A label scoring 0 -- no signal, or a
+ non-finite vector treated as none -- outranks one scoring negative, and k drops the
+ negatives first.
+ * a zero-norm query scores everything 0, so the kept set is the first k in criteria order.
+ * non-finite components become 0 rather than propagating or raising.
+ * k >= the label count is a passthrough: the labels come back in criteria order, scores are
+ None, and the embedder is NEVER called.
+ """
+ from laya import shortlist as mod
+
+ criteria_small = {"refund": "money back", "technical": "a bug", "billing": "an invoice"}
+ criteria_large = {
+ "card_arrival": "where is my new card",
+ "card_delivery_estimate": "when will the card arrive",
+ "card_not_working": "the card is declined",
+ "cash_withdrawal_charge": "charged for an ATM withdrawal",
+ "declined_card_payment": "a payment was declined",
+ "direct_debit_payment_not_recognised": "an unrecognised direct debit",
+ "exchange_rate": "what rate was applied",
+ "failed_transfer": "a transfer did not arrive",
+ "lost_or_stolen_card": "the card is gone",
+ "pending_card_payment": "a payment is still pending",
+ "refund_not_showing_up": "a refund has not arrived",
+ "request_refund": "I want money back",
+ "reverted_card_payment": "a payment was reversed",
+ "top_up_failed": "a top up did not work",
+ "transfer_fee_charged": "charged a fee for a transfer",
+ "verify_my_identity": "identity verification",
+ "wrong_amount_of_cash_received": "the ATM gave the wrong amount",
+ }
+ criteria_tied = {"a": None, "b": None, "c": None, "d": None}
+ criteria_one = {"only": "the only option"}
+
+ states = {
+ "string": "My card was charged twice for the same order last Tuesday.",
+ "structured": {"subject": "duplicate charge", "body": "billed twice in March"},
+ "empty": "",
+ "none": None,
+ "non_latin": "注文が届きません",
+ }
+
+ cases = []
+
+ def record(name, state, criteria, k, instructions=None):
+ labels, scores = mod.shortlist_choice(
+ state, criteria, _stub_embed, k, instructions=instructions, return_scores=True)
+ items = mod._criteria_items(criteria)
+ cases.append({
+ "name": name,
+ "state": _state_repr(state),
+ "criteria": criteria,
+ "k": k,
+ "instructions": instructions,
+ "query_text": mod._query_text(state, instructions),
+ "option_texts": mod._option_texts(items),
+ "labels": list(labels),
+ "scores": None if scores is None else [_round_score(v) for v in scores],
+ "passthrough": scores is None,
+ "n": len(items),
+ "subset_criteria": (None if scores is None
+ else mod._subset_criteria(criteria, labels)),
+ })
+
+ record("large-top-5", states["string"], criteria_large, 5)
+ record("large-top-1", states["string"], criteria_large, 1)
+ record("large-top-16-of-17", states["string"], criteria_large, 16)
+ record("large-k-equals-n", states["string"], criteria_large, 17)
+ record("large-k-over-n", states["string"], criteria_large, 99)
+ record("small-k-2", states["string"], criteria_small, 2)
+ record("one-option-k-1", states["string"], criteria_one, 1)
+ record("structured-state", states["structured"], criteria_large, 4)
+ record("empty-state", states["empty"], criteria_large, 4)
+ record("none-state", states["none"], criteria_large, 4)
+ record("non-latin-state", states["non_latin"], criteria_large, 4)
+ record("with-instructions", states["string"], criteria_large, 4,
+ "What does the customer want in `body`?")
+ record("instructions-empty-string", states["string"], criteria_large, 4, "")
+ # criteria with no descriptions: the option text is the bare label
+ record("no-descriptions", states["string"], criteria_tied, 2)
+ # a list of labels is a legal choice criteria in the reference
+ record("list-criteria", states["string"], ["alpha", "beta", "gamma", "delta"], 2)
+
+ # --- the cosine itself, on the vectors a port gets wrong
+ import numpy as np
+
+ cosine_cases = []
+ for name, query, docs in [
+ ("ordinary", [1.0, 0.0], [[1.0, 0.0], [0.0, 1.0], [-1.0, 0.0]]),
+ ("zero-query", [0.0, 0.0], [[1.0, 0.0], [0.0, 1.0]]),
+ ("zero-doc", [1.0, 0.0], [[0.0, 0.0], [1.0, 0.0]]),
+ ("all-zero", [0.0, 0.0], [[0.0, 0.0]]),
+ ("negatives", [1.0, 1.0], [[-1.0, -1.0], [1.0, 1.0], [0.0, 0.0]]),
+ ("needs-clipping", [1e-200, 1e-200], [[1e-200, 1e-200]]),
+ ("large-magnitudes", [1e200, 1e200], [[1e200, 1e200], [-1e200, 1e200]]),
+ ("no-docs", [1.0, 1.0], []),
+ # A query whose norm UNDERFLOWS to zero while its dot product does not. Without the
+ # zero-norm guard the score would be 1.4e-200 instead of 0 -- a difference no rounded
+ # comparison can see, which is why the port asserts this one exactly.
+ ("underflowing-query", [1e-200, 1e-200], [[1.0, 1.0]]),
+ # A vector whose self-cosine computes just ABOVE 1, so the clamp is observable. Also
+ # invisible to a rounded comparison: 1.0000000000000002 and 1.0 agree to twelve decimals.
+ ("self-cosine-over-one", [1.0 / 3.0, 1.0 / 3.0, 8.0 / 3.0],
+ [[1.0 / 3.0, 1.0 / 3.0, 8.0 / 3.0]]),
+ ]:
+ q = np.asarray(query, dtype=np.float64)
+ d = np.asarray(docs, dtype=np.float64).reshape(len(docs), len(query) if docs else 0)
+ if not docs:
+ d = np.zeros((0, len(query)), dtype=np.float64)
+ # Tagged, not written bare: a vector whose norm overflows gives inf/inf, so the
+ # reference's cosine really does return NaN here -- and `NaN` is not JSON. Writing it
+ # bare produced a fixture that Python reads back happily and a strict reader rejects.
+ cosine_cases.append({"name": name, "query": query, "docs": docs,
+ "sims": [_round_score(v) for v in mod._cosine(q, d)]})
+
+ # --- non-finite components must become zero, not propagate
+ nonfinite = []
+ for name, rows in [
+ ("nan", [[float("nan"), 1.0], [1.0, 0.0]]),
+ ("posinf", [[float("inf"), 1.0], [1.0, 0.0]]),
+ ("neginf", [[float("-inf"), 1.0], [1.0, 0.0]]),
+ ("all-nan", [[float("nan"), float("nan")], [1.0, 0.0]]),
+ ]:
+ cleaned = mod._embeddings(lambda texts, rows=rows: np.asarray(rows, dtype=np.float64),
+ ["q", "d"])
+ nonfinite.append({"name": name, "rows": [[_json_float(v) for v in row] for row in rows],
+ "cleaned": [[float(v) for v in row] for row in cleaned]})
+
+ # --- refusals: every input the reference rejects, and the message it rejects it with
+ refusals = []
+ for name, call in [
+ ("k-zero", lambda: mod.shortlist_choice("s", criteria_small, _stub_embed, 0)),
+ ("k-negative", lambda: mod.shortlist_choice("s", criteria_small, _stub_embed, -1)),
+ ("k-bool", lambda: mod.shortlist_choice("s", criteria_small, _stub_embed, True)),
+ ("criteria-empty-dict", lambda: mod.shortlist_choice("s", {}, _stub_embed, 2)),
+ ("criteria-empty-list", lambda: mod.shortlist_choice("s", [], _stub_embed, 2)),
+ ("criteria-wrong-type", lambda: mod.shortlist_choice("s", "nope", _stub_embed, 2)),
+ ("criteria-duplicate-label",
+ lambda: mod.shortlist_choice("s", ["a", "a"], _stub_embed, 1)),
+ ("embed-not-callable", lambda: mod.shortlist_choice("s", criteria_small, None, 2)),
+ ("embed-wrong-rows",
+ lambda: mod.shortlist_choice("s", criteria_small, lambda t: [[1.0]], 2)),
+ ("embed-one-dimensional",
+ lambda: mod.shortlist_choice("s", criteria_small, lambda t: [1.0] * (len(t)), 2)),
+ ("questions-not-dict",
+ lambda: mod.predict_shortlist(_FakeAgent(), "s", [], _stub_embed, 2)),
+ ("choice-without-criteria",
+ lambda: mod.predict_shortlist(_FakeAgent(), "s",
+ {"q": {"type": "choice"}}, _stub_embed, 2)),
+ ]:
+ try:
+ call()
+ refusals.append({"name": name, "error": None})
+ except Exception as failure:
+ refusals.append({"name": name, "error": type(failure).__name__,
+ "message": str(failure)})
+
+ # --- what predict_shortlist hands to predict, which is the whole point of the module
+ predicts = []
+ for name, questions, k in [
+ ("choice-reduced", {"intent": {"type": "choice",
+ "instructions": "What does the customer want?",
+ "criteria": criteria_large}}, 4),
+ ("choice-passthrough", {"intent": {"type": "choice", "instructions": "x",
+ "criteria": criteria_small}}, 20),
+ ("mixed-questions", {
+ "intent": {"type": "choice", "instructions": "What?", "criteria": criteria_large},
+ "urgent": {"type": "noul", "instructions": "Is this urgent?"},
+ "severity": {"type": "score", "instructions": "How bad?",
+ "criteria": ["low", "high"]},
+ }, 3),
+ ("two-choices", {
+ "a": {"type": "choice", "instructions": "A?", "criteria": criteria_large},
+ "b": {"type": "choice", "instructions": "B?", "criteria": criteria_small},
+ }, 2),
+ ]:
+ agent = _FakeAgent()
+ original = {qid: dict(qdef) for qid, qdef in questions.items()}
+ result = mod.predict_shortlist(agent, states["string"], questions, _stub_embed, k)
+ predicts.append({
+ "name": name,
+ "k": k,
+ "questions": original,
+ "asked": agent.seen["questions"],
+ "shortlist": result["shortlist"],
+ "caller_questions_unmutated": original == {qid: dict(q)
+ for qid, q in questions.items()},
+ })
+
+ return {
+ "notes": shortlist.__doc__,
+ "score_decimals": _SCORE_DECIMALS,
+ "default_k": mod.DEFAULT_SHORTLIST_K,
+ "embed_dim": _EMBED_DIM,
+ "embed_probe": {text: [float(v) for v in _stub_embed([text])[0]]
+ for text in ["", "a", "refund: money back", states["string"]]},
+ "cases": cases,
+ "cosine": cosine_cases,
+ "nonfinite": nonfinite,
+ "refusals": refusals,
+ "predicts": predicts,
+ }
+
+
+# How many decimals of a cosine are safe to record. The reference computes its scores with
+# `np.dot(matrix, vector)`, a BLAS matrix-vector product, and how BLAS blocks that accumulation
+# differs between implementations -- measured here: on Apple Accelerate one row of a four-row
+# product differs from the same row computed on its own, in the last bit. So the raw score is not
+# reproducible across machines, let alone across languages: recording it verbatim would make this
+# fixture disagree with itself between a developer's macOS and CI's Linux, and the drift gate would
+# fail for a reason that has nothing to do with any port.
+#
+# Twelve decimals is about four orders of magnitude tighter than any porting error that has ever
+# shown up in this repository, and about four orders LOOSER than the 1e-16 BLAS spread, so it
+# separates the two cleanly. What stays exact is the LABEL ORDER, which is the only thing a
+# consumer can observe -- and the thing a ranking bug actually changes.
+_SCORE_DECIMALS = 12
+
+
+def _round_score(value):
+ """A cosine rounded to a precision BLAS blocking cannot move, or tagged when non-finite."""
+ value = float(value)
+ if value != value or value in (float("inf"), float("-inf")):
+ return _json_float(value)
+ return round(value, _SCORE_DECIMALS)
+
+
+def _json_float(value):
+ """NaN and the infinities have no JSON spelling, so they are tagged for a port to rebuild."""
+ if value != value:
+ return "nan"
+ if value == float("inf"):
+ return "inf"
+ if value == float("-inf"):
+ return "-inf"
+ return value
def tokenizer_ids():
"""Exact token ids per checkpoint for a corpus designed to break a port.
@@ -1385,6 +1676,7 @@ FAMILIES = {
"lang_detect.json": lang_detect,
"presets.json": presets,
"router.json": router,
+ "shortlist.json": shortlist,
"tokenizer_ids.json": tokenizer_ids,
"sequences.json": sequences,
"decode.json": decode_answers,