Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(ts): truncateLeft kept the whole state when no room was left (#112 parity)
divit aggarwale8513264ae88
diff --git a/laya-ts/src/common.ts b/laya-ts/src/common.tsindex 323bad34..0ab78b56 100644--- a/laya-ts/src/common.ts+++ b/laya-ts/src/common.ts@@ -86,7 +86,8 @@ export function buildQuestionPrefix(tok: TokenizerLike, q: InternalQ,export function sequenceWithState(prefix: QuestionPrefix, stateIds: number[], sepId: number,maxLen = 512, truncateLeft = false): { ids: number[]; markers: number[] } {const room = Math.max(0, maxLen - prefix.ids.length - 1);- const st = truncateLeft ? stateIds.slice(-room) : stateIds.slice(0, room);+ // not stateIds.slice(-room): with no room left, slice(-0) is the whole state rather than none of it+ const st = truncateLeft ? stateIds.slice(Math.max(0, stateIds.length - room)) : stateIds.slice(0, room);const ids = [...prefix.ids, ...st, sepId].slice(0, maxLen);return { ids, markers: prefix.markers.filter((m) => m < maxLen) };}diff --git a/laya-ts/tests/parity-sequence.test.ts b/laya-ts/tests/parity-sequence.test.tsindex bbe12cd2..995a72b7 100644--- a/laya-ts/tests/parity-sequence.test.ts+++ b/laya-ts/tests/parity-sequence.test.ts@@ -1,6 +1,7 @@// laya-ts/tests/parity-sequence.test.tsimport { describe, expect, it } from "vitest";-import { renderOptions, serializeState, buildSequence, collateItems } from "../src/common.js";+import { renderOptions, serializeState, buildSequence, collateItems, type InternalQ } from "../src/common.js";+import type { TokenizerLike } from "../src/tokenizer.js";describe("common parity", () => {it("renders choice with JSON criteria", () => {expect(renderOptions({ t: "choice", ins: "x", crit: { a: "yes", b: null } }))@@ -37,6 +38,30 @@ describe("common parity", () => {expect(a.ids.slice(12, 15)).toEqual([1000, 1001, 1002]);expect(b.ids.slice(12, 15)).toEqual([1017, 1018, 1019]);});+ // With no room left for the state, `slice(-0)` kept all of it: the closing [SEP] was replaced by the+ // *first* state token, i.e. the wrong end of the state and an unterminated sequence.+ it.each<[number, string[]]>([+ [0, []],+ [2, ["two", "three"]],+ [4, ["one", "two", "three"]],+ [10, ["one", "two", "three"]],+ ])("truncate_left/room=%d keeps the tail", (room, kept) => {+ // same stub as Python's _SeqTok: one token per word, ids assigned in first-seen order+ const vocab = new Map<string, number>();+ const tok: TokenizerLike = {+ clsId: 2, sepId: 3, maskId: 1, padId: 0, maskToken: "[MASK]",+ encode(s: string): number[] {+ return s.split(/\s+/).filter(Boolean).map((w) => {+ if (!vocab.has(w)) vocab.set(w, 100 + vocab.size);+ return vocab.get(w)!;+ });+ },+ };+ const q: InternalQ = { t: "noul", ins: "Is it urgent?", crit: null };+ const full = buildSequence(tok, "", q, 1e6, 192).ids.length; // prompt + closing [SEP], no state+ const { ids } = buildSequence(tok, "one two three", q, full + room, 192, undefined, true);+ expect(ids.slice(full - 1)).toEqual([...kept.map((w) => vocab.get(w)!), tok.sepId]);+ });it("optionOrder reorders option blocks (py parity)", () => {const tok = {clsId: 101, sepId: 102, maskId: 103, maskToken: "[MASK]",