Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(finetune): the remaining training scripts fit temperatures outside the runtime clamp

Bruce-Yiibbc47f76377f
diff --git a/notebooks/laya_finetune_typed_decisions_mps.py b/notebooks/laya_finetune_typed_decisions_mps.py
index eb70be37..6f854be3 100644
--- a/notebooks/laya_finetune_typed_decisions_mps.py
+++ b/notebooks/laya_finetune_typed_decisions_mps.py
@@ -26,7 +26,7 @@ from safetensors.torch import load_file, save_file
from transformers import AutoTokenizer
from laya.agent import _fix_tokenizer_config
-from laya.common import QTYPES, build_model, build_sequence, proper_reward, render_options
+from laya.common import QTYPES, TEMP_MAX, TEMP_MIN, build_model, build_sequence, proper_reward, render_options
MODEL_ID = "convaiinnovations/laya"
DATASET_ID = "LocalLLaMA/typed-decisions"
@@ -215,7 +215,7 @@ def fit_temperature(samples):
return loss
optimizer.step(closure)
- return float(torch.clamp(log_temperature.exp(), 0.1, 10.0).item())
+ return float(torch.clamp(log_temperature.exp(), TEMP_MIN, TEMP_MAX).item())
def save_checkpoint(model, tokenizer, cfg, output_dir, epoch, final=False):
diff --git a/research/scripts/finetune_single_device.py b/research/scripts/finetune_single_device.py
index 0fd99d61..fb5af748 100644
--- a/research/scripts/finetune_single_device.py
+++ b/research/scripts/finetune_single_device.py
@@ -31,6 +31,8 @@ from transformers import AutoTokenizer
from laya.agent import _fix_tokenizer_config
from laya.common import (
QTYPES,
+ TEMP_MAX,
+ TEMP_MIN,
build_model,
build_sequence,
proper_reward,
@@ -151,7 +153,7 @@ def fit_one_temp(sel):
return loss
opt.step(closure)
- return float(torch.clamp(log_t.exp(), 0.1, 10.0).item())
+ return float(torch.clamp(log_t.exp(), TEMP_MIN, TEMP_MAX).item())
def main():