The best ChatGPT that $100 can buy.
Python
Ensure that inputs and targets are contiguous (#569)
Sofie Van Landeghem752abc836e70
diff --git a/scripts/chat_sft.py b/scripts/chat_sft.pyindex f31a2d3..cb9e078 100644--- a/scripts/chat_sft.py+++ b/scripts/chat_sft.py@@ -282,8 +282,8 @@ def sft_data_generator_bos_bestfit(split, buffer_size=100):# Build tensorsuse_cuda = device_type == "cuda"batch_tensor = torch.tensor(rows, dtype=torch.long, pin_memory=use_cuda)- inputs = batch_tensor[:, :-1].to(device=device, dtype=torch.int32, non_blocking=use_cuda)- targets = batch_tensor[:, 1:].to(device=device, dtype=torch.int64, non_blocking=use_cuda)+ inputs = batch_tensor[:, :-1].to(device=device, dtype=torch.int32, non_blocking=use_cuda).contiguous()+ targets = batch_tensor[:, 1:].to(device=device, dtype=torch.int64, non_blocking=use_cuda).contiguous()# Apply the loss mask from render_conversation (mask=1 for assistant completions,# mask=0 for user prompts, BOS, special tokens, tool outputs). mask[1:] aligns