Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

[vllm-ext] allows setting flashinfer workspace (#1259)

Yi Liu6e1ea9cbe87f
diff --git a/auto_round_extension/vllm_ext/envs_ext.py b/auto_round_extension/vllm_ext/envs_ext.py
index 70fbb3238..eb251ccef 100644
--- a/auto_round_extension/vllm_ext/envs_ext.py
+++ b/auto_round_extension/vllm_ext/envs_ext.py
@@ -26,6 +26,7 @@ extra_environment_variables: dict[str, Callable[[], Any]] = {
"VLLM_ENABLE_STATIC_MOE": lambda: os.getenv("VLLM_ENABLE_STATIC_MOE", "0") in ("1", "true", "True"),
"VLLM_AR_MXFP4_MODULAR_MOE": lambda: os.getenv("VLLM_AR_MXFP4_MODULAR_MOE", "1") in ("1", "true", "True"),
"VLLM_AR_POST_PROCESS_GPTOSS": lambda: os.getenv("VLLM_AR_POST_PROCESS_GPTOSS", "0") in ("1", "true", "True"),
+ "VLLM_AR_FLASHINFER_WORKSPACE_BUFFER_SIZE": lambda: int(os.getenv("VLLM_AR_FLASHINFER_WORKSPACE_BUFFER_SIZE", "0")),
}
# Add the extra environment variables to vllm.envs
import vllm.envs as envs
diff --git a/auto_round_extension/vllm_ext/vllm_oot_patches.py b/auto_round_extension/vllm_ext/vllm_oot_patches.py
index af05cfc0c..cf229e596 100644
--- a/auto_round_extension/vllm_ext/vllm_oot_patches.py
+++ b/auto_round_extension/vllm_ext/vllm_oot_patches.py
@@ -13,11 +13,23 @@
# limitations under the License.
import vllm
+import vllm.envs as envs
from vllm.logger import init_logger
logger = init_logger(__name__)
+def update_flashinfer_workspace_buffer_size(new_size: int):
+ import vllm.v1.attention.backends.flashinfer as flashinfer_backend
+
+ flashinfer_backend.WORKSPACE_BUFFER_SIZE = new_size
+ logger.info(f"Updated FlashInfer WORKSPACE_BUFFER_SIZE to {new_size}")
+
+
+if envs.VLLM_AR_FLASHINFER_WORKSPACE_BUFFER_SIZE > 0:
+ update_flashinfer_workspace_buffer_size(envs.VLLM_AR_FLASHINFER_WORKSPACE_BUFFER_SIZE)
+
+
def oot_maybe_remap_kv_scale_name(name: str, params_dict: dict) -> str | None:
"""Remap the name of FP8 k/v_scale parameters.