LLM inference in C/C++
C++ ggml
model-conversion : add --add-bos to run org model script (#29558)
Daniel Bevenius8df332de1b7d
diff --git a/examples/model-conversion/scripts/causal/run-org-model.py b/examples/model-conversion/scripts/causal/run-org-model.pyindex 6f85ee4485..b0f6e886c2 100755--- a/examples/model-conversion/scripts/causal/run-org-model.py+++ b/examples/model-conversion/scripts/causal/run-org-model.py@@ -19,6 +19,8 @@ def parse_arguments():parser.add_argument("--prompt-file", "-f", help="Optional prompt file", required=False)parser.add_argument("--verbose", "-v", action="store_true", help="Enable verbose debug output")parser.add_argument("--device", "-d", help="Device to use (cpu, cuda, mps, auto)", default="auto")+ parser.add_argument("--add-bos", action=argparse.BooleanOptionalAction, default=None,+ help="Override BOS token setting (default: use model's own setting)")return parser.parse_args()def load_model_and_tokenizer(model_path, device="auto"):@@ -119,6 +121,9 @@ def main():model, tokenizer, config = load_model_and_tokenizer(model_path, args.device)+ if args.add_bos is not None and hasattr(tokenizer, "add_bos_token"):+ tokenizer.add_bos_token = args.add_bos+if args.verbose:enable_torch_debugging(model)