Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(ts): truncateLeft kept the whole state when no room was left (#112 parity)

divit aggarwale8513264ae88
diff --git a/laya-ts/src/common.ts b/laya-ts/src/common.ts
index 323bad34..0ab78b56 100644
--- a/laya-ts/src/common.ts
+++ b/laya-ts/src/common.ts
@@ -86,7 +86,8 @@ export function buildQuestionPrefix(tok: TokenizerLike, q: InternalQ,
export function sequenceWithState(prefix: QuestionPrefix, stateIds: number[], sepId: number,
maxLen = 512, truncateLeft = false): { ids: number[]; markers: number[] } {
const room = Math.max(0, maxLen - prefix.ids.length - 1);
- const st = truncateLeft ? stateIds.slice(-room) : stateIds.slice(0, room);
+ // not stateIds.slice(-room): with no room left, slice(-0) is the whole state rather than none of it
+ const st = truncateLeft ? stateIds.slice(Math.max(0, stateIds.length - room)) : stateIds.slice(0, room);
const ids = [...prefix.ids, ...st, sepId].slice(0, maxLen);
return { ids, markers: prefix.markers.filter((m) => m < maxLen) };
}
diff --git a/laya-ts/tests/parity-sequence.test.ts b/laya-ts/tests/parity-sequence.test.ts
index bbe12cd2..995a72b7 100644
--- a/laya-ts/tests/parity-sequence.test.ts
+++ b/laya-ts/tests/parity-sequence.test.ts
@@ -1,6 +1,7 @@
// laya-ts/tests/parity-sequence.test.ts
import { describe, expect, it } from "vitest";
-import { renderOptions, serializeState, buildSequence, collateItems } from "../src/common.js";
+import { renderOptions, serializeState, buildSequence, collateItems, type InternalQ } from "../src/common.js";
+import type { TokenizerLike } from "../src/tokenizer.js";
describe("common parity", () => {
it("renders choice with JSON criteria", () => {
expect(renderOptions({ t: "choice", ins: "x", crit: { a: "yes", b: null } }))
@@ -37,6 +38,30 @@ describe("common parity", () => {
expect(a.ids.slice(12, 15)).toEqual([1000, 1001, 1002]);
expect(b.ids.slice(12, 15)).toEqual([1017, 1018, 1019]);
});
+ // With no room left for the state, `slice(-0)` kept all of it: the closing [SEP] was replaced by the
+ // *first* state token, i.e. the wrong end of the state and an unterminated sequence.
+ it.each<[number, string[]]>([
+ [0, []],
+ [2, ["two", "three"]],
+ [4, ["one", "two", "three"]],
+ [10, ["one", "two", "three"]],
+ ])("truncate_left/room=%d keeps the tail", (room, kept) => {
+ // same stub as Python's _SeqTok: one token per word, ids assigned in first-seen order
+ const vocab = new Map<string, number>();
+ const tok: TokenizerLike = {
+ clsId: 2, sepId: 3, maskId: 1, padId: 0, maskToken: "[MASK]",
+ encode(s: string): number[] {
+ return s.split(/\s+/).filter(Boolean).map((w) => {
+ if (!vocab.has(w)) vocab.set(w, 100 + vocab.size);
+ return vocab.get(w)!;
+ });
+ },
+ };
+ const q: InternalQ = { t: "noul", ins: "Is it urgent?", crit: null };
+ const full = buildSequence(tok, "", q, 1e6, 192).ids.length; // prompt + closing [SEP], no state
+ const { ids } = buildSequence(tok, "one two three", q, full + room, 192, undefined, true);
+ expect(ids.slice(full - 1)).toEqual([...kept.map((w) => vocab.get(w)!), tok.sepId]);
+ });
it("optionOrder reorders option blocks (py parity)", () => {
const tok = {
clsId: 101, sepId: 102, maskId: 103, maskToken: "[MASK]",