Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

27.1 KB
Raw Blame
# Copyright (c) 2025 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import argparse
import importlib.util
import os
import time
from pathlib import Path

import torch
from transformers.utils.versions import require_version

from auto_round.utils import (
    DEVICE_ENVIRON_VARIABLE_MAPPING,
    dispatch_model_block_wise,
    get_device_and_parallelism,
    get_major_device,
    get_model_dtype,
    is_diffusion_model,
    set_cuda_visible_devices,
)


def _has_eval_model_artifacts(model_dir: str) -> bool:
    if not os.path.isdir(model_dir):
        return False
    artifact_names = {"config.json", "model_index.json", "modular_model_index.json"}
    for name in artifact_names:
        if os.path.isfile(os.path.join(model_dir, name)):
            return True
    return any(name.endswith(".gguf") for name in os.listdir(model_dir))


def _resolve_exported_model_path(model_path):
    """Resolve wrapper output directories to the actual exported model folder.

    Some helper scripts pass a parent output directory (for example
    ``outputs/aware_topk_tune_shared``) while the export itself lives in a
    single nested subdirectory derived from model name and quantization scheme.
    When there is exactly one child directory containing loadable model
    artifacts, prefer that child automatically.
    """
    from auto_round.utils import logger

    if not isinstance(model_path, str) or not os.path.isdir(model_path):
        return model_path

    if _has_eval_model_artifacts(model_path):
        return model_path

    candidate_dirs = []
    for entry in os.listdir(model_path):
        candidate = os.path.join(model_path, entry)
        if os.path.isdir(candidate) and _has_eval_model_artifacts(candidate):
            candidate_dirs.append(candidate)

    if len(candidate_dirs) == 1:
        resolved = candidate_dirs[0]
        logger.info("Resolved evaluation model path '%s' to exported artifact '%s'.", model_path, resolved)
        return resolved
    return model_path


class EvalArgumentParser(argparse.ArgumentParser):

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.add_argument(
            "model",
            default=None,
            nargs="?",
            help="Path to the pre-trained model or model identifier from huggingface.co/models. "
            "Examples: 'facebook/opt-125m', 'bert-base-uncased', or local path like '/path/to/model'",
        )
        self.add_argument(
            "--model_name",
            "--model",
            "--model_name_or_path",
            dest="model_name",
            default=None,
            help="Path to the pre-trained model or model identifier from huggingface.co/models. "
            "Examples: 'facebook/opt-125m', 'bert-base-uncased', or local path like '/path/to/model'",
        )
        self.add_argument("--mllm", action="store_true", help="whether to eval multi-modal model.")
        self.add_argument(
            "--device_map",
            "--device",
            "--devices",
            default="0",
            type=str,
            help="the device to be used for evaluation. "
            "The default is set to 0,"
            "allowing for automatic detection and switch to any devices."
            "set --device 0,1,2 to use multiple cards.",
        )

        self.add_argument(
            "--tasks",
            "--task",
            default="lambada_openai,hellaswag,winogrande,piqa,mmlu,wikitext,truthfulqa_mc1,"
            "truthfulqa_mc2,openbookqa,boolq,rte,arc_easy,arc_challenge",
            help="LM-Evaluation-Harness tasks to run. "
            "Specify specific tasks like 'mmlu,wikitext' for custom evaluation.",
        )
        self.add_argument(
            "--disable_trust_remote_code",
            action="store_true",
            help="Disable trusting remote code when loading models. "
            "Use for security if you don't trust the model source.",
        )
        self.add_argument("--seed", default=42, type=int, help="Random seed for reproducibility.")
        self.add_argument(
            "--eval_bs", "--bs", "--batch_size", default=None, type=int, help="The batch size for evaluation"
        )
        self.add_argument(
            "--eval_task_by_task", action="store_true", help="Evaluate tasks sequentially instead of batching. "
        )
        self.add_argument(
            "--eval_model_dtype",
            default=None,
            type=str,
            help="Torch data type for model loading during evaluation. "
            "Options: 'float16', 'bfloat16', 'float32'. "
            "Should match your hardware capabilities for best performance.",
        )
        self.add_argument(
            "--limit",
            type=float,
            default=None,
            metavar="N|0<N<1",
            help="Limit the number of examples per task. "
            "Integer: exact number of examples (e.g., 1000). "
            "Float between 0-1: fraction of total examples.",
        )
        self.add_argument("--num_fewshot", "--num-fewshot", default=None, type=int, help="Number of few-shot examples.")
        self.add_argument(
            "--eval_gen_kwargs", "--eval-gen-kwargs", default=None, type=str, help="Generation kwargs for LM-Eval."
        )
        self.add_argument(
            "--fewshot_as_multiturn",
            "--fewshot-as-multiturn",
            action="store_true",
            help="Use multi-turn format for few-shot examples in LM-Eval.",
        )
        self.add_argument(
            "--eval_backend",
            default="hf",
            type=str,
            choices=["hf", "vllm"],
            help="Backend to use for model evaluation. Use hf backend for evaluation by default.",
        )
        self.add_argument("--add_bos_token", action="store_true", help="add BOS token")
        self.add_argument(
            "--vllm_args",
            default=None,
            type=str,
            help="(for vllm) Custom vllm arguments in format: 'arg1=value1,arg2=value2'. "
            "Example: 'tensor_parallel_size=2,gpu_memory_utilization=0.9'",
        )
        # Diffusion model specific arguments
        diffusion_args = self.add_argument_group("diffusion model arguments")
        diffusion_args.add_argument(
            "--prompt_file",
            default=None,
            type=str,
            help="File containing prompts for evaluation, one per line. "
            "Use this for batch evaluation with multiple prompts.",
        )
        diffusion_args.add_argument(
            "--prompt",
            default=None,
            type=str,
            help="Single prompt for quick testing. " "Overrides prompt_file if both are specified.",
        )
        diffusion_args.add_argument(
            "--metrics",
            "--metric",
            default="clip",
            help="Evaluation metrics for generated images. "
            "'clip': CLIP score measuring text-image alignment. "
            "'clip-iqa': CLIP-based image quality assessment. "
            "'imagereward': Learned metric for image quality.",
        )
        diffusion_args.add_argument(
            "--image_save_dir",
            default="./tmp_image_save",
            type=str,
            help="Directory to save generated images during evaluation. " "Useful for visual inspection of results.",
        )
        diffusion_args.add_argument(
            "--guidance_scale",
            default=7.5,
            type=float,
            help="Classifier-free guidance scale for diffusion models. "
            "Higher values (7-20) make the model follow the prompt more closely. "
            "Lower values give more creative/random results.",
        )
        diffusion_args.add_argument(
            "--num_inference_steps",
            default=50,
            type=int,
            help="Number of denoising steps in the diffusion process. "
            "More steps (50-100) usually give better quality but take longer. "
            "Fewer steps (10-30) are faster but lower quality.",
        )
        diffusion_args.add_argument(
            "--generator_seed",
            default=None,
            type=int,
            help="Random seed for image generation reproducibility. "
            "Using the same seed produces identical results across runs.",
        )


def _eval_init(tasks, model_path, device, disable_trust_remote_code=False, dtype="auto"):
    set_cuda_visible_devices(device)
    device_str, parallelism = get_device_and_parallelism(device)
    if dtype != "auto":
        dtype = get_model_dtype(model_dtype=dtype)
    # ,add_bos_token={True}
    model_args = f"pretrained={model_path},trust_remote_code={not disable_trust_remote_code},dtype={dtype}"
    if parallelism:
        model_args += ",parallelize=True"
    if isinstance(tasks, str):
        tasks = tasks.split(",")
    return tasks, model_args, device_str


def eval(args):
    require_version(
        "lm_eval>=0.4.2", "lm-eval is required for evaluation, please install it with `pip install 'lm-eval>=0.4.2'`"
    )

    args.model_name = _resolve_exported_model_path(args.model_name)

    if is_diffusion_model(args.model_name):
        from auto_round.eval.evaluation import evaluate_diffusion_model
        from auto_round.utils import diffusion_load_model

        pipe, _ = diffusion_load_model(args.model_name)
        evaluate_diffusion_model(args, pipe=pipe)
        return
    if args.eval_backend == "vllm":
        assert isinstance(args.model_name, str), "vllm evaluation only supports model name or path."
        eval_with_vllm(args)
        return
    tasks, model_args, device_str = _eval_init(
        args.tasks, args.model_name, args.device_map, args.disable_trust_remote_code, args.eval_model_dtype
    )

    # load after _eval_int in order to make sure import torch after set CUDA_VISIBLE_DEVICES
    from auto_round.eval.evaluation import simple_evaluate, simple_evaluate_user_model
    from auto_round.utils import logger

    if (batch_size := args.eval_bs) is None:
        batch_size = "auto:8"

    model, tokenizer, is_gguf_file, gguf_file = _load_gguf_model_if_needed(args.model_name, args.eval_model_dtype)

    if is_gguf_file:
        from lm_eval.utils import make_table  # pylint: disable=E0401

        st = time.time()
        res = simple_evaluate_user_model(
            model,
            tokenizer,
            tasks=tasks,
            batch_size=batch_size,
            device=device_str,
            limit=args.limit,
            num_fewshot=getattr(args, "num_fewshot", None),
            gen_kwargs=getattr(args, "eval_gen_kwargs", None),
            fewshot_as_multiturn=getattr(args, "fewshot_as_multiturn", False),
            add_bos_token=args.add_bos_token,
        )
        print(make_table(res))
        print(f"evaluation running time={int(time.time() - st)}s")
    else:
        st = time.time()
        if "auto" in str(batch_size) and args.mllm:
            logger.warning("Batch size 'auto' is not yet supported for hf-multimodal models, reset to 16")
            batch_size = 16
        model_args += f",add_bos_token={args.add_bos_token}"
        res = simple_evaluate(
            model="hf" if not args.mllm else "hf-multimodal",
            model_args=model_args,
            tasks=tasks,
            device=device_str,
            batch_size=batch_size,
            limit=args.limit,
            num_fewshot=getattr(args, "num_fewshot", None),
            gen_kwargs=getattr(args, "eval_gen_kwargs", None),
            fewshot_as_multiturn=getattr(args, "fewshot_as_multiturn", False),
        )
        from lm_eval.utils import make_table  # pylint: disable=E0401

        print(make_table(res))
        print(f"evaluation running time={int(time.time() - st)}s")


def eval_with_vllm(args):
    # set vllm worker method
    os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"

    import time

    from lm_eval import evaluator  # pylint: disable=E0401
    from lm_eval.models.vllm_causallms import VLLM  # pylint: disable=E0401
    from lm_eval.utils import make_table  # pylint: disable=E0401

    st = time.time()
    os.environ["TOKENIZERS_PARALLELISM"] = "false"
    device_str, _ = get_device_and_parallelism(args.device_map)
    eval_model_dtype = get_model_dtype(args.eval_model_dtype, "auto")
    if (batch_size := args.eval_bs) is None:
        batch_size = "auto:8"
    if isinstance(args.tasks, str):
        tasks = args.tasks.split(",")

    # Parse custom vllm_args if provided
    custom_vllm_kwargs = parse_vllm_args(getattr(args, "vllm_args", None))

    # Build vllm kwargs with base parameters
    vllm_kwargs = {
        "pretrained": args.model_name,
        "dtype": eval_model_dtype,
        "trust_remote_code": not args.disable_trust_remote_code,
        "add_bos_token": args.add_bos_token,
        "device": device_str,
        "batch_size": batch_size,
        "allow_deprecated_quantization": True,  # for vLLM==0.14.0
    }

    # Override with custom vllm_args if provided
    if custom_vllm_kwargs:
        from auto_round.logger import logger

        logger.info(f"Overriding VLLM parameters with custom args: {custom_vllm_kwargs}")
        vllm_kwargs.update(custom_vllm_kwargs)

    device = get_major_device()
    if "tensor_parallel_size" not in vllm_kwargs:
        # Parse device_map to determine tensor_parallel_size and set the relevant env var
        # Only accept formats like "0" or "0,1,2". If the environment variable is
        # already set externally, do not overwrite it — but still derive
        # `tensor_parallel_size` from the existing value.
        assert device in DEVICE_ENVIRON_VARIABLE_MAPPING, f"Device {device} not supported for vllm tensor parallelism."
        env_name = DEVICE_ENVIRON_VARIABLE_MAPPING[device]
        device_map = args.device_map
        device_ids = [d.strip() for d in str(device_map).split(",") if d.strip().isdigit()]

        from auto_round.logger import logger

        existing_env = os.environ.get(env_name)
        if existing_env:
            existing_ids = [d.strip() for d in existing_env.split(",") if d.strip()]
            if existing_ids:
                tensor_parallel_size = len(existing_ids)
                vllm_kwargs["tensor_parallel_size"] = tensor_parallel_size
                logger.info(
                    f"Detected existing {env_name}={existing_env}, skipping overwrite; "
                    f"tensor_parallel_size={tensor_parallel_size} derived from {env_name}"
                )
        elif device_ids:
            device_id_str = ",".join(device_ids)
            os.environ[env_name] = device_id_str
            tensor_parallel_size = len(device_ids)
            vllm_kwargs["tensor_parallel_size"] = tensor_parallel_size

            logger.info(f"Set {env_name}={os.environ[env_name]}, tensor_parallel_size={tensor_parallel_size}")

    if args.mllm:
        from lm_eval.models.vllm_vlms import VLLM_VLM  # pylint: disable=E0401

        vllm_lm = VLLM_VLM(**vllm_kwargs)
    else:
        vllm_lm = VLLM(**vllm_kwargs)
    task_manager = _get_lm_eval_task_manager(tasks)
    res = evaluator.simple_evaluate(
        model=vllm_lm,
        tasks=tasks,
        limit=args.limit,
        num_fewshot=getattr(args, "num_fewshot", None),
        gen_kwargs=getattr(args, "eval_gen_kwargs", None),
        fewshot_as_multiturn=getattr(args, "fewshot_as_multiturn", False),
        task_manager=task_manager,
    )

    print(make_table(res))
    print(f"evaluation running time={int(time.time() - st)}s")


def eval_task_by_task(
    model,
    device=None,
    tasks=None,
    tokenizer=None,
    batch_size=None,
    limit=None,
    max_batch_size=64,
    trust_remote_code=True,
    eval_model_dtype=None,
    retry_times=3,
    mllm=False,
    add_bos_token=False,
    num_fewshot=None,
    gen_kwargs=None,
    fewshot_as_multiturn=False,
):
    require_version(
        "lm_eval>=0.4.2", "lm-eval is required for evaluation, please install it with `pip install 'lm-eval>=0.4.2'`"
    )

    set_cuda_visible_devices(device)
    device_str, parallelism = get_device_and_parallelism(device)

    # load after _eval_int in order to make sure import torch after set CUDA_VISIBLE_DEVICES
    from lm_eval.models.huggingface import HFLM  # pylint: disable=E0401

    from auto_round.utils import logger

    if batch_size is None:
        batch_size = "auto:8"

    if not isinstance(model, str) and parallelism:
        from accelerate import dispatch_model, infer_auto_device_map

        device_map = infer_auto_device_map(model)
        model = dispatch_model(model, device_map=device_map)
        parallelism = False
        is_gguf_file = False
        gguf_file = None
    else:
        model, tokenizer, is_gguf_file, gguf_file = _load_gguf_model_if_needed(model, eval_model_dtype)
        if is_gguf_file:
            parallelism = False
    if isinstance(model, torch.nn.Module):
        dispatch_model_block_wise(model, device_map="auto")  # As we set visible device before, so explcits

    eval_model_dtype = get_model_dtype(eval_model_dtype)
    if mllm:
        if batch_size is None or batch_size == "auto":
            logger.warning("hf-multimodal models does not support auto currently, reset eval_bs to 16")
            batch_size = 16
        from lm_eval.models.hf_vlms import HFMultimodalLM  # pylint: disable=E0401

        hflm = HFMultimodalLM(
            pretrained=model,
            tokenizer=tokenizer,
            device=device_str,
            batch_size=batch_size,
            max_batch_size=max_batch_size,
            parallelize=parallelism,
            trust_remote_code=trust_remote_code,
            dtype=eval_model_dtype,
            add_bos_token=add_bos_token,
        )
    else:
        hflm = HFLM(
            pretrained=model,
            tokenizer=tokenizer,
            device=device_str,
            batch_size=batch_size,
            max_batch_size=max_batch_size,
            parallelize=parallelism,
            trust_remote_code=trust_remote_code,
            dtype=eval_model_dtype,
            add_bos_token=add_bos_token,
        )

    _evaluate_tasks_with_retry(
        tasks,
        hflm,
        device_str,
        batch_size,
        limit,
        retry_times,
        num_fewshot=num_fewshot,
        gen_kwargs=gen_kwargs,
        fewshot_as_multiturn=fewshot_as_multiturn,
    )


def _load_gguf_model_if_needed(model_path, eval_model_dtype=None):
    """Detect and load GGUF model if the path points to a GGUF file.

    Args:
        model_path: Path to model or GGUF file
        eval_model_dtype: Data type for model evaluation

    Returns:
        Tuple of (model, tokenizer, is_gguf_file, gguf_file_name)
        If not a GGUF file, returns (model_path, None, False, None)
    """
    from auto_round.utils import logger

    is_gguf_file = False
    gguf_file = None
    tokenizer = None
    model = model_path

    # Check if model_path is a string before processing
    if isinstance(model_path, str):
        if os.path.isfile(model_path) and model_path.endswith(".gguf"):
            is_gguf_file = True
            gguf_file = os.path.basename(model_path)
            model = os.path.dirname(model_path)
        elif os.path.exists(model_path):
            for file in os.listdir(model_path):
                if file.endswith(".gguf"):
                    is_gguf_file = True
                    gguf_file = file
                    break

    if is_gguf_file:
        from transformers import AutoModelForCausalLM, AutoTokenizer

        eval_model_dtype = get_model_dtype(eval_model_dtype)
        tokenizer = AutoTokenizer.from_pretrained(model, gguf_file=gguf_file)

        logger.warning("evaluating gguf model is an experimental feature, the accuracy may be not correct.")
        if eval_model_dtype == "float32" or eval_model_dtype == "auto":
            logger.warning(
                "set '--eval_model_dtype bf16' can significantly speed up evaluation for gguf model,"
                " but may affect accuracy."
            )

        model = AutoModelForCausalLM.from_pretrained(
            model, gguf_file=gguf_file, device_map="auto", torch_dtype=eval_model_dtype
        )
        model.eval()

    return model, tokenizer, is_gguf_file, gguf_file


def _get_lm_eval_task_manager(tasks: list[str]) -> object | None:
    """Use exact installed task dirs when possible to avoid scanning all lm-eval tasks."""
    try:
        import lm_eval  # pylint: disable=E0401
        from lm_eval.tasks import TaskManager  # pylint: disable=E0401
    except ImportError:
        return None

    tasks_root = Path(lm_eval.__file__).parent / "tasks"
    task_paths = []
    for task in tasks:
        task_path = tasks_root / task
        if task_path.is_dir():
            task_paths.append(str(task_path))
            continue

        task_configs = list(tasks_root.rglob(f"{task}.yaml"))
        if not task_configs:
            return None
        task_paths.extend(str(task_config.parent) for task_config in task_configs)

    try:
        return TaskManager(include_defaults=False, include_path=list(dict.fromkeys(task_paths)))
    except TypeError:
        return None


def _evaluate_tasks_with_retry(
    tasks,
    hflm,
    device_str,
    batch_size,
    limit,
    retry_times,
    num_fewshot=None,
    gen_kwargs=None,
    fewshot_as_multiturn=False,
):
    """Evaluate tasks with automatic retry on OOM errors.

    Args:
        tasks: List of task names to evaluate
        hflm: HuggingFace LM model instance
        device_str: Device string for evaluation
        batch_size: Batch size for evaluation
        limit: Limit number of examples per task
        retry_times: Number of retry attempts on failure

    Returns:
        Aggregated results dictionary containing results, versions, n-shot, and higher_is_better
    """
    import time
    import traceback

    import lm_eval  # pylint: disable=E0401
    from lm_eval.utils import make_table  # pylint: disable=E0401

    from auto_round.utils import logger

    if isinstance(tasks, str):
        tasks = tasks.replace(" ", "").split(",")

    res_all = {}
    res_keys = ["results", "versions", "n-shot", "higher_is_better"]
    st = time.time()
    task_manager = _get_lm_eval_task_manager(tasks)

    for task in tasks:
        current_retry_times = retry_times
        res = None
        last_error = None
        while current_retry_times:
            try:
                res = lm_eval.simple_evaluate(
                    model=hflm,
                    model_args=None,
                    device=device_str,
                    tasks=task,
                    batch_size=batch_size,
                    limit=limit,
                    num_fewshot=num_fewshot,
                    gen_kwargs=gen_kwargs,
                    task_manager=task_manager,
                    fewshot_as_multiturn=fewshot_as_multiturn,
                )
                break
            except torch.OutOfMemoryError as e:
                last_error = e
                cuda_error_msg = traceback.format_exc()
                ori_batch_sizes = getattr(hflm, "batch_sizes", None) or {"0": 64}
                if not getattr(hflm, "batch_sizes", None):
                    hflm.batch_sizes = ori_batch_sizes.copy()
                try:
                    for k, v in hflm.batch_sizes.items():
                        hflm.batch_sizes[k] = max(v // 2, 1)
                    logger.warning(f"Out of memory, reset batch_size to {hflm.batch_sizes} and re-try.")
                    res = lm_eval.simple_evaluate(
                        model=hflm,
                        model_args=None,
                        device=device_str,
                        tasks=task,
                        batch_size=1,
                        limit=limit,
                        num_fewshot=num_fewshot,
                        gen_kwargs=gen_kwargs,
                        task_manager=task_manager,
                        fewshot_as_multiturn=fewshot_as_multiturn,
                    )
                except torch.OutOfMemoryError as e:
                    last_error = e
                    traceback.print_exc()
                    res = None
                finally:
                    hflm.batch_sizes = ori_batch_sizes
            current_retry_times -= 1

        if res is None:
            raise RuntimeError(f"Failed to evaluate task '{task}' after {retry_times} attempts") from last_error
        if not res_all:
            res_all = res
        else:
            for key in res_keys:
                if key not in res_all:
                    continue
                else:
                    res_all[key].update(res[key])
        print(make_table(res_all))

    print("total eval time:", time.time() - st)


def parse_vllm_args(vllm_args_str):
    """Parse custom vllm arguments from string format.

    Args:
        vllm_args_str: String containing vllm arguments in format:
                      "--arg1=value1,--arg2=value2" or "arg1=value1,arg2=value2"

    Returns:
        Dictionary of parsed arguments with appropriate types (int, float, bool, or string)

    Example:
        >>> parse_vllm_args("--tensor_parallel_size=2,--gpu_memory_utilization=0.9")
        {'tensor_parallel_size': 2, 'gpu_memory_utilization': 0.9}
    """
    from auto_round.logger import logger

    custom_vllm_kwargs = {}

    if not vllm_args_str:
        return custom_vllm_kwargs

    logger.info(f"Parsing custom vllm arguments: {vllm_args_str}")

    for arg_pair in vllm_args_str.split(","):
        arg_pair = arg_pair.strip()
        # Normalize: replace space separator with '=' (e.g., "--arg value" -> "--arg=value")
        if "=" not in arg_pair and " " in arg_pair:
            parts = arg_pair.split(None, 1)  # Split on whitespace, max 2 parts
            if len(parts) == 2:
                arg_pair = f"{parts[0]}={parts[1]}"
        if "=" in arg_pair:
            # Remove leading '--' if present
            arg_pair = arg_pair.removeprefix("--")
            key, value = arg_pair.split("=", 1)
            key = key.strip()
            value = value.strip()

            # Try to convert value to appropriate type
            try:
                # Try int first
                if value.isdigit() or (value.startswith("-") and value[1:].isdigit()):
                    custom_vllm_kwargs[key] = int(value)
                # Try float
                elif "." in value:
                    custom_vllm_kwargs[key] = float(value)
                # Try boolean
                elif value.lower() in ("true", "false"):
                    custom_vllm_kwargs[key] = value.lower() == "true"
                # Keep as string
                else:
                    custom_vllm_kwargs[key] = value
                logger.info(
                    f"  Parsed vllm arg: {key}={custom_vllm_kwargs[key]}"
                    + f" (type: {type(custom_vllm_kwargs[key]).__name__})"
                )
            except Exception as e:
                logger.warning(f"  Failed to parse vllm arg '{key}={value}': {e}, keeping as string")
                custom_vllm_kwargs[key] = value

    return custom_vllm_kwargs