PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
[vllm-ext] allows setting flashinfer workspace (#1259)
Yi Liu6e1ea9cbe87f
diff --git a/auto_round_extension/vllm_ext/envs_ext.py b/auto_round_extension/vllm_ext/envs_ext.pyindex 70fbb3238..eb251ccef 100644--- a/auto_round_extension/vllm_ext/envs_ext.py+++ b/auto_round_extension/vllm_ext/envs_ext.py@@ -26,6 +26,7 @@ extra_environment_variables: dict[str, Callable[[], Any]] = {"VLLM_ENABLE_STATIC_MOE": lambda: os.getenv("VLLM_ENABLE_STATIC_MOE", "0") in ("1", "true", "True"),"VLLM_AR_MXFP4_MODULAR_MOE": lambda: os.getenv("VLLM_AR_MXFP4_MODULAR_MOE", "1") in ("1", "true", "True"),"VLLM_AR_POST_PROCESS_GPTOSS": lambda: os.getenv("VLLM_AR_POST_PROCESS_GPTOSS", "0") in ("1", "true", "True"),+ "VLLM_AR_FLASHINFER_WORKSPACE_BUFFER_SIZE": lambda: int(os.getenv("VLLM_AR_FLASHINFER_WORKSPACE_BUFFER_SIZE", "0")),}# Add the extra environment variables to vllm.envsimport vllm.envs as envsdiff --git a/auto_round_extension/vllm_ext/vllm_oot_patches.py b/auto_round_extension/vllm_ext/vllm_oot_patches.pyindex af05cfc0c..cf229e596 100644--- a/auto_round_extension/vllm_ext/vllm_oot_patches.py+++ b/auto_round_extension/vllm_ext/vllm_oot_patches.py@@ -13,11 +13,23 @@# limitations under the License.import vllm+import vllm.envs as envsfrom vllm.logger import init_loggerlogger = init_logger(__name__)+def update_flashinfer_workspace_buffer_size(new_size: int):+ import vllm.v1.attention.backends.flashinfer as flashinfer_backend++ flashinfer_backend.WORKSPACE_BUFFER_SIZE = new_size+ logger.info(f"Updated FlashInfer WORKSPACE_BUFFER_SIZE to {new_size}")+++if envs.VLLM_AR_FLASHINFER_WORKSPACE_BUFFER_SIZE > 0:+ update_flashinfer_workspace_buffer_size(envs.VLLM_AR_FLASHINFER_WORKSPACE_BUFFER_SIZE)++def oot_maybe_remap_kv_scale_name(name: str, params_dict: dict) -> str | None:"""Remap the name of FP8 k/v_scale parameters.