PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
docs(examples): align recipes with recommended W2 and batch size settings (#2499) (#2511)
Vishva Ra0134a4ca807
diff --git a/examples/Cloudflare-clef/clef-flash_W4A16.ipynb b/examples/Cloudflare-clef/clef-flash_W4A16.ipynbindex 8b9705543..2a8905bdb 100644--- a/examples/Cloudflare-clef/clef-flash_W4A16.ipynb+++ b/examples/Cloudflare-clef/clef-flash_W4A16.ipynb@@ -163,6 +163,7 @@" alg_configs=alg_config, # Applies the algorithm-specific optimization settings defined above\n"," nsamples=512, # Total calibration dataset samples to maintain quality and avoid overfitting\n"," batch_size=2, # Batch size for calibration; kept < 8 to avoid slow execution/high memory overhead\n",+ " gradient_accumulate_steps=4, # Set to 8 / batch_size to maintain an effective global batch size of 8; smaller values risk accuracy drop \n"," seqlen=2048, # Calibration context length; captures long-range dependency activations\n"," low_gpu_mem_usage=False, # Keeps modules on GPU to maximize speed if sufficient VRAM is available\n"," quant_nontext_module=False, # Leaves non-text modules (e.g., vision encoders) unquantized to protect precision\n",diff --git a/examples/Cloudflare-clef/clef_W4A16.ipynb b/examples/Cloudflare-clef/clef_W4A16.ipynbindex fcfa9e6a4..e355413ed 100644--- a/examples/Cloudflare-clef/clef_W4A16.ipynb+++ b/examples/Cloudflare-clef/clef_W4A16.ipynb@@ -163,6 +163,7 @@" alg_configs=alg_config, # Applies the algorithm-specific optimization settings defined above\n"," nsamples=512, # Total calibration dataset samples to maintain quality and avoid overfitting\n"," batch_size=2, # Batch size for calibration; kept < 8 to avoid slow execution/high memory overhead\n",+ " gradient_accumulate_steps=4, # Set to 8 / batch_size to maintain an effective global batch size of 8; smaller values risk accuracy drop \n"," seqlen=2048, # Calibration context length; captures long-range dependency activations\n"," low_gpu_mem_usage=False, # Keeps modules on GPU to maximize speed if sufficient VRAM is available\n"," quant_nontext_module=False, # Leaves non-text modules (e.g., vision encoders) unquantized to protect precision\n",diff --git a/examples/DotsOCR/DotsmOCR_W4A16.ipynb b/examples/DotsOCR/DotsmOCR_W4A16.ipynbindex 275b0cd60..dcc7ba9e8 100644--- a/examples/DotsOCR/DotsmOCR_W4A16.ipynb+++ b/examples/DotsOCR/DotsmOCR_W4A16.ipynb@@ -143,6 +143,7 @@" alg_configs=alg_config, # Applies the algorithm-specific optimization settings defined above\n"," nsamples=512, # Total calibration dataset samples to maintain quality and avoid overfitting\n"," batch_size=4, # Batch size for calibration; kept < 8 to avoid slow execution/high memory overhead\n",+ " gradient_accumulate_steps=2, # Set to 8 / batch_size to maintain an effective global batch size of 8; smaller values risk accuracy drop \n"," seqlen=4096, # Calibration context length; captures long-range dependency activations\n"," low_gpu_mem_usage=False, # Keeps modules on GPU to maximize speed if sufficient VRAM is available\n"," enable_torch_compile=True, # Compiles forward passes using torch.compile to accelerate calibration\n",diff --git a/examples/LightOnOCR/LightOnOCR-3-0.8B_W4A16.ipynb b/examples/LightOnOCR/LightOnOCR-3-0.8B_W4A16.ipynbindex 70c786d8b..9f944f6b9 100644--- a/examples/LightOnOCR/LightOnOCR-3-0.8B_W4A16.ipynb+++ b/examples/LightOnOCR/LightOnOCR-3-0.8B_W4A16.ipynb@@ -163,6 +163,7 @@" alg_configs=alg_config, # Applies the algorithm-specific optimization settings defined above\n"," nsamples=512, # Total calibration dataset samples to maintain quality and avoid overfitting\n"," batch_size=4, # Batch size for calibration; kept < 8 to avoid slow execution/high memory overhead\n",+ " gradient_accumulate_steps=2, # Set to 8 / batch_size to maintain an effective global batch size of 8; smaller values risk accuracy drop \n"," seqlen=4096, # Calibration context length; captures long-range dependency activations\n"," low_gpu_mem_usage=False, # Keeps modules on GPU to maximize speed if sufficient VRAM is available\n"," enable_torch_compile=True, # Compiles forward passes using torch.compile to accelerate calibration\n",diff --git a/examples/LightOnOCR/LightOnOCR-3-4B_W4A16.ipynb b/examples/LightOnOCR/LightOnOCR-3-4B_W4A16.ipynbindex 5ab1885a8..b5863c09e 100644--- a/examples/LightOnOCR/LightOnOCR-3-4B_W4A16.ipynb+++ b/examples/LightOnOCR/LightOnOCR-3-4B_W4A16.ipynb@@ -163,6 +163,7 @@" alg_configs=alg_config, # Applies the algorithm-specific optimization settings defined above\n"," nsamples=512, # Total calibration dataset samples to maintain quality and avoid overfitting\n"," batch_size=4, # Batch size for calibration; kept < 8 to avoid slow execution/high memory overhead\n",+ " gradient_accumulate_steps=2, # Set to 8 / batch_size to maintain an effective global batch size of 8; smaller values risk accuracy drop \n"," seqlen=4096, # Calibration context length; captures long-range dependency activations\n"," low_gpu_mem_usage=False, # Keeps modules on GPU to maximize speed if sufficient VRAM is available\n"," enable_torch_compile=True, # Compiles forward passes using torch.compile to accelerate calibration\n",diff --git a/examples/LiquidAI/LiquidAId3B_W4A16.ipynb b/examples/LiquidAI/LiquidAId3B_W4A16.ipynbindex cabfa2e15..85e44c4b2 100644--- a/examples/LiquidAI/LiquidAId3B_W4A16.ipynb+++ b/examples/LiquidAI/LiquidAId3B_W4A16.ipynb@@ -144,6 +144,7 @@" alg_configs=alg_config, # Applies the algorithm-specific optimization settings defined above\n"," nsamples=512, # Total calibration dataset samples to maintain quality and avoid overfitting\n"," batch_size=2, # Batch size for calibration; kept < 8 to avoid slow execution/high memory overhead\n",+ " gradient_accumulate_steps=4, # Set to 8 / batch_size to maintain an effective global batch size of 8; smaller values risk accuracy drop \n"," seqlen=2048, # Calibration context length; captures long-range dependency activations\n"," low_gpu_mem_usage=False, # Keeps modules on GPU to maximize speed if sufficient VRAM is available\n"," enable_torch_compile=False, # Compiles forward passes using torch.compile to accelerate calibration\n",diff --git a/examples/Muse-Glimmer/Muse-Glimmer_W2A16.ipynb b/examples/Muse-Glimmer/Muse-Glimmer_W2A16.ipynbindex 5600fb801..d389ec8c9 100644--- a/examples/Muse-Glimmer/Muse-Glimmer_W2A16.ipynb+++ b/examples/Muse-Glimmer/Muse-Glimmer_W2A16.ipynb@@ -118,9 +118,11 @@"source": ["# Configure optimization parameters for the AutoRound / SignRound algorithm\n","alg_config = SignRoundConfig(\n",- " iters=800, # Optimization iterations per block; high accuracy for production grade\n",+ " iters=1000, # Optimization iterations per block (steps ~= 1000 recommended for W2)\n"," group_size=64, # Number of weights sharing one scale. Cannot go below 32 (vLLM's GPTQ kernel won't run it). Only use 16 if targeting NVFP4.\n"," sym=True, # Symmetric quantization; enables faster compute and broader engine compatibility\n",+ " enable_alg_ext=True, # Algorithm extension for extreme low-bit (W2A16) precision boost\n",+ " lr=2 / 1000, # lr = 2 / steps (~0.002); prevents gradient divergence during 2-bit optimization\n",")\n","\n","# Initialize the AutoRound quantization pipeline\n",@@ -132,6 +134,7 @@" alg_configs=alg_config, # Applies the algorithm-specific optimization settings defined above\n"," nsamples=512, # Total calibration dataset samples to maintain quality and avoid overfitting\n"," batch_size=4, # Batch size for calibration; reduced to manage GPU memory usage on 48GB VRAM\n",+ " gradient_accumulate_steps=2, # Set to 8 / batch_size to maintain an effective global batch size of 8; smaller values risk accuracy drop \n"," seqlen=2048, # Calibration context length; captures long-range dependency activations\n"," low_gpu_mem_usage=False, # Keeps modules on GPU to maximize speed if sufficient VRAM is available\n"," enable_torch_compile=True, # Compiles forward passes using torch.compile to accelerate calibration\n",diff --git a/examples/Muse-Glimmer/Muse-Glimmer_W4A16.ipynb b/examples/Muse-Glimmer/Muse-Glimmer_W4A16.ipynbindex fd28d1176..d9dd3e1fd 100644--- a/examples/Muse-Glimmer/Muse-Glimmer_W4A16.ipynb+++ b/examples/Muse-Glimmer/Muse-Glimmer_W4A16.ipynb@@ -132,6 +132,7 @@" alg_configs=alg_config, # Applies the algorithm-specific optimization settings defined above\n"," nsamples=512, # Total calibration dataset samples to maintain quality and avoid overfitting\n"," batch_size=4, # Batch size for calibration; reduced to manage GPU memory usage on 48GB VRAM\n",+ " gradient_accumulate_steps=2, # Set to 8 / batch_size to maintain an effective global batch size of 8; smaller values risk accuracy drop \n"," seqlen=2048, # Calibration context length; captures long-range dependency activations\n"," low_gpu_mem_usage=False, # Keeps modules on GPU to maximize speed if sufficient VRAM is available\n"," enable_torch_compile=True, # Compiles forward passes using torch.compile to accelerate calibration\n",diff --git a/examples/NuExtract3/NuExtract3_W4A16.ipynb b/examples/NuExtract3/NuExtract3_W4A16.ipynbindex 032a43106..3c4d112af 100644--- a/examples/NuExtract3/NuExtract3_W4A16.ipynb+++ b/examples/NuExtract3/NuExtract3_W4A16.ipynb@@ -162,6 +162,7 @@" alg_configs=alg_config, # Applies the algorithm-specific optimization settings defined above\n"," nsamples=512, # Total calibration dataset samples to maintain quality and avoid overfitting\n"," batch_size=4, # Batch size for calibration; reduced to manage GPU memory usage on 48GB VRAM\n",+ " gradient_accumulate_steps=2, # Set to 8 / batch_size to maintain an effective global batch size of 8; smaller values risk accuracy drop \n"," seqlen=4096, # Calibration context length; captures long-range dependency activations\n"," low_gpu_mem_usage=False, # Keeps modules on GPU to maximize speed if sufficient VRAM is available\n"," enable_torch_compile=True, # Compiles forward passes using torch.compile to accelerate calibration\n",diff --git a/examples/Qwen3.5/Qwen3.5-0.8B_NVFP4.ipynb b/examples/Qwen3.5/Qwen3.5-0.8B_NVFP4.ipynbindex 12baac191..c5107793a 100644--- a/examples/Qwen3.5/Qwen3.5-0.8B_NVFP4.ipynb+++ b/examples/Qwen3.5/Qwen3.5-0.8B_NVFP4.ipynb@@ -164,6 +164,7 @@" alg_configs=alg_config, # Applies the algorithm-specific optimization settings defined above\n"," nsamples=512, # Total calibration dataset samples to maintain quality and avoid overfitting\n"," batch_size=4, # Calibration batch size; balances processing speed and GPU memory usage\n",+ " gradient_accumulate_steps=2, # Set to 8 / batch_size to maintain an effective global batch size of 8; smaller values risk accuracy drop \n"," seqlen=2048, # Calibration context length; captures long-range dependency activations\n"," low_gpu_mem_usage=False, # Keeps modules on GPU to maximize speed if sufficient VRAM is available\n"," quant_nontext_module=False, # Leaves non-text modules (e.g., vision encoders) unquantized to protect precision\n",diff --git a/examples/Qwen3.5/Qwen3.5-2B_NVFP4.ipynb b/examples/Qwen3.5/Qwen3.5-2B_NVFP4.ipynbindex 84b796c8c..4886741d5 100644--- a/examples/Qwen3.5/Qwen3.5-2B_NVFP4.ipynb+++ b/examples/Qwen3.5/Qwen3.5-2B_NVFP4.ipynb@@ -164,6 +164,7 @@" alg_configs=alg_config, # Applies the algorithm-specific optimization settings defined above\n"," nsamples=512, # Total calibration dataset samples to maintain quality and avoid overfitting\n"," batch_size=4, # Calibration batch size; balances processing speed and GPU memory usage\n",+ " gradient_accumulate_steps=2, # Set to 8 / batch_size to maintain an effective global batch size of 8; smaller values risk accuracy drop \n"," seqlen=2048, # Calibration context length; captures long-range dependency activations\n"," low_gpu_mem_usage=False, # Keeps modules on GPU to maximize speed if sufficient VRAM is available\n"," quant_nontext_module=False, # Leaves non-text modules (e.g., vision encoders) unquantized to protect precision\n",diff --git a/examples/Qwen3.5/Qwen3.5-4B_NVFP4.ipynb b/examples/Qwen3.5/Qwen3.5-4B_NVFP4.ipynbindex ca2123e9b..aa3c4870d 100644--- a/examples/Qwen3.5/Qwen3.5-4B_NVFP4.ipynb+++ b/examples/Qwen3.5/Qwen3.5-4B_NVFP4.ipynb@@ -164,6 +164,7 @@" alg_configs=alg_config, # Applies the algorithm-specific optimization settings defined above\n"," nsamples=512, # Total calibration dataset samples to maintain quality and avoid overfitting\n"," batch_size=4, # Calibration batch size; balances processing speed and GPU memory usage\n",+ " gradient_accumulate_steps=2, # Set to 8 / batch_size to maintain an effective global batch size of 8; smaller values risk accuracy drop \n"," seqlen=2048, # Calibration context length; captures long-range dependency activations\n"," low_gpu_mem_usage=False, # Keeps modules on GPU to maximize speed if sufficient VRAM is available\n"," quant_nontext_module=False, # Leaves non-text modules (e.g., vision encoders) unquantized to protect precision\n",diff --git a/examples/Qwen3.5/Qwen3.5-9B_NVFP4.ipynb b/examples/Qwen3.5/Qwen3.5-9B_NVFP4.ipynbindex ef9dcdd1b..a5241cdaf 100644--- a/examples/Qwen3.5/Qwen3.5-9B_NVFP4.ipynb+++ b/examples/Qwen3.5/Qwen3.5-9B_NVFP4.ipynb@@ -164,6 +164,7 @@" alg_configs=alg_config, # Applies the algorithm-specific optimization settings defined above\n"," nsamples=512, # Total calibration dataset samples to maintain quality and avoid overfitting\n"," batch_size=4, # Calibration batch size; balances processing speed and GPU memory usage\n",+ " gradient_accumulate_steps=2, # Set to 8 / batch_size to maintain an effective global batch size of 8; smaller values risk accuracy drop \n"," seqlen=2048, # Calibration context length; captures long-range dependency activations\n"," low_gpu_mem_usage=False, # Keeps modules on GPU to maximize speed if sufficient VRAM is available\n"," quant_nontext_module=False, # Leaves non-text modules (e.g., vision encoders) unquantized to protect precision\n",diff --git a/examples/Qwen3.5/Qwen3_5_0.8B_W4A16.ipynb b/examples/Qwen3.5/Qwen3_5_0.8B_W4A16.ipynbindex c7d54d82c..9cf29a703 100644--- a/examples/Qwen3.5/Qwen3_5_0.8B_W4A16.ipynb+++ b/examples/Qwen3.5/Qwen3_5_0.8B_W4A16.ipynb@@ -153,6 +153,7 @@" alg_configs=alg_config, # Applies the algorithm-specific optimization settings defined above\n"," nsamples=512, # Total calibration dataset samples to maintain quality and avoid overfitting\n"," batch_size=2, # Batch size for calibration; reduced to manage GPU memory usage on 48GB VRAM\n",+ " gradient_accumulate_steps=4, # Set to 8 / batch_size to maintain an effective global batch size of 8; smaller values risk accuracy drop \n"," seqlen=2048, # Calibration context length; captures long-range dependency activations\n"," low_gpu_mem_usage=False, # Keeps modules on GPU to maximize speed if sufficient VRAM is available\n"," enable_torch_compile=True, # Compiles forward passes using torch.compile to accelerate calibration\n",diff --git a/examples/Qwen3.5/Qwen3_5_4B_W2A16.ipynb b/examples/Qwen3.5/Qwen3_5_4B_W2A16.ipynbindex f22b28de0..c179b8714 100644--- a/examples/Qwen3.5/Qwen3_5_4B_W2A16.ipynb+++ b/examples/Qwen3.5/Qwen3_5_4B_W2A16.ipynb@@ -138,9 +138,11 @@"\n","# Configure optimization parameters for the AutoRound / SignRound algorithm\n","alg_config = SignRoundConfig(\n",- " iters=1000, # Optimization iterations per block; balances calibration time and accuracy\n",- " group_size=32, # Number of weights sharing one scale. Cannot go below 32 (vLLM's GPTQ kernel won't run it). Only use 16 if targeting NVFP4.\n",- " sym=True, # Symmetric quantization; enables faster compute and broader engine compatibility\n",+ " iters=1000, # Optimization iterations per block (steps ~= 1000 recommended for W2)\n",+ " group_size=64, # Number of weights sharing one scale. Cannot go below 32 (vLLM's GPTQ kernel won't run it). Only use 16 if targeting NVFP4.\n",+ " sym=True, # Symmetric quantization; enables faster compute and broader engine compatibility\n",+ " enable_alg_ext=True, # Algorithm extension for extreme low-bit (W2A16) precision boost\n",+ " lr=2 / 1000, # lr = 2 / steps (~0.002); prevents gradient divergence during 2-bit optimization\n",")\n","\n","# Initialize the AutoRound quantization pipeline\n",@@ -152,6 +154,7 @@" alg_configs=alg_config, # Applies the algorithm-specific optimization settings defined above\n"," nsamples=512, # Total calibration dataset samples to maintain quality and avoid overfitting\n"," batch_size=2, # Batch size for calibration; balances speed and GPU memory usage\n",+ " gradient_accumulate_steps=4, # Set to 8 / batch_size to maintain an effective global batch size of 8; smaller values risk accuracy drop \n"," seqlen=2048, # Calibration context length; captures long-range dependency activations\n"," low_gpu_mem_usage=False, # Keeps modules on GPU to maximize speed if sufficient VRAM is available\n"," enable_torch_compile=True, # Compiles forward passes using torch.compile to accelerate calibration\n",diff --git a/examples/Qwen3.5/Qwen3_5_4B_W4A16.ipynb b/examples/Qwen3.5/Qwen3_5_4B_W4A16.ipynbindex 7358fee3e..e51e9e85f 100644--- a/examples/Qwen3.5/Qwen3_5_4B_W4A16.ipynb+++ b/examples/Qwen3.5/Qwen3_5_4B_W4A16.ipynb@@ -162,6 +162,7 @@" alg_configs=alg_config, # Applies the algorithm-specific optimization settings defined above\n"," nsamples=512, # Total calibration dataset samples to maintain quality and avoid overfitting\n"," batch_size=2, # Batch size for calibration; reduced to manage GPU memory usage on 48GB VRAM\n",+ " gradient_accumulate_steps=4, # Set to 8 / batch_size to maintain an effective global batch size of 8; smaller values risk accuracy drop \n"," seqlen=2048, # Calibration context length; captures long-range dependency activations\n"," low_gpu_mem_usage=False, # Keeps modules on GPU to maximize speed if sufficient VRAM is available\n"," enable_torch_compile=True, # Compiles forward passes using torch.compile to accelerate calibration\n",diff --git a/examples/Qwen3.5/Qwen3_5_9B_W4A16.ipynb b/examples/Qwen3.5/Qwen3_5_9B_W4A16.ipynbindex 6f0f33765..6b9ca991c 100644--- a/examples/Qwen3.5/Qwen3_5_9B_W4A16.ipynb+++ b/examples/Qwen3.5/Qwen3_5_9B_W4A16.ipynb@@ -139,30 +139,6 @@"tokenizer = processor.tokenizer\n"]},- {- "cell_type": "code",- "execution_count": null,- "id": "7c1063ff",- "metadata": {},- "outputs": [],- "source": [- "TUNING_CONFIG = {\n",- " \"group_size\": 32,\n",- " \"sym\": True,\n",- " \"iters\": 1200, # High accuracy (Production grade)\n",- " \"nsamples\": 512, # More calibration data\n",- " \"batch_size\": 4,\n",- " \"seqlen\": 4096,\n",- " \"low_gpu_mem_usage\": False, # Keep on GPU for speed\n",- " \"enable_torch_compile\": True, # JIT acceleration\n",- " \"quant_nontext_module\": False, # Keep Vision Tower in FP16 (Crucial for VLM accuracy)\n",- " \"layer_config\": {\n",- " \"mtp\": {\"data_type\": \"bfloat16\"},\n",- " \"mtp.fc\": {\"data_type\": \"bfloat16\"},\n",- " },\n",- "}"- ]- },{"cell_type": "code","execution_count": null,@@ -186,6 +162,7 @@" alg_configs=alg_config, # Applies the algorithm-specific optimization settings defined above\n"," nsamples=512, # Total calibration dataset samples to maintain quality and avoid overfitting\n"," batch_size=4, # Batch size for calibration; keep < 8 to avoid slow execution/high memory overhead\n",+ " gradient_accumulate_steps=2, # Set to 8 / batch_size to maintain an effective global batch size of 8; smaller values risk accuracy drop \n"," seqlen=4096, # Calibration context length; captures long-range dependency activations\n"," low_gpu_mem_usage=False, # Keeps modules on GPU to maximize speed if sufficient VRAM is available\n"," enable_torch_compile=True, # Compiles forward passes using torch.compile to accelerate calibration (JIT acceleration)\n",diff --git a/examples/Qwen3.8/Qwen3_8_27B_W4A16.ipynb b/examples/Qwen3.8/Qwen3_8_27B_W4A16.ipynbindex a53d6e7c8..202ef646c 100644--- a/examples/Qwen3.8/Qwen3_8_27B_W4A16.ipynb+++ b/examples/Qwen3.8/Qwen3_8_27B_W4A16.ipynb@@ -163,6 +163,7 @@" alg_configs=alg_config, # Applies the algorithm-specific optimization settings defined above\n"," nsamples=512, # Total calibration dataset samples to maintain quality and avoid overfitting\n"," batch_size=2, # Batch size for calibration; reduced to manage GPU memory usage on 48GB VRAM\n",+ " gradient_accumulate_steps=4, # Set to 8 / batch_size to maintain an effective global batch size of 8; smaller values risk accuracy drop \n"," seqlen=2048, # Calibration context length; captures long-range dependency activations\n"," low_gpu_mem_usage=False, # Keeps modules on GPU to maximize speed if sufficient VRAM is available\n"," enable_torch_compile=True, # Compiles forward passes using torch.compile to accelerate calibration\n",diff --git a/examples/vLLM-sr/d3-edge_W4A16.ipynb b/examples/vLLM-sr/d3-edge_W4A16.ipynbindex ff51b7364..f573b8581 100644--- a/examples/vLLM-sr/d3-edge_W4A16.ipynb+++ b/examples/vLLM-sr/d3-edge_W4A16.ipynb@@ -163,6 +163,7 @@" alg_configs=alg_config, # Applies the algorithm-specific optimization settings defined above\n"," nsamples=512, # Total calibration dataset samples to maintain quality and avoid overfitting\n"," batch_size=4, # Batch size for calibration; kept < 8 to avoid slow execution/high memory overhead\n",+ " gradient_accumulate_steps=2, # Set to 8 / batch_size to maintain an effective global batch size of 8; smaller values risk accuracy drop \n"," seqlen=4096, # Calibration context length; captures long-range dependency activations\n"," low_gpu_mem_usage=False, # Keeps modules on GPU to maximize speed if sufficient VRAM is available\n"," quant_nontext_module=False, # Leaves non-text modules (e.g., vision encoders) unquantized to protect precision\n",diff --git a/examples/vLLM-sr/d3-lite_W4A16.ipynb b/examples/vLLM-sr/d3-lite_W4A16.ipynbindex 399cebc9b..7fcae6a27 100644--- a/examples/vLLM-sr/d3-lite_W4A16.ipynb+++ b/examples/vLLM-sr/d3-lite_W4A16.ipynb@@ -163,6 +163,7 @@" alg_configs=alg_config, # Applies the algorithm-specific optimization settings defined above\n"," nsamples=512, # Total calibration dataset samples to maintain quality and avoid overfitting\n"," batch_size=4, # Batch size for calibration; kept < 8 to avoid slow execution/high memory overhead\n",+ " gradient_accumulate_steps=2, # Set to 8 / batch_size to maintain an effective global batch size of 8; smaller values risk accuracy drop \n"," seqlen=4096, # Calibration context length; captures long-range dependency activations\n"," low_gpu_mem_usage=False, # Keeps modules on GPU to maximize speed if sufficient VRAM is available\n"," quant_nontext_module=False, # Leaves non-text modules (e.g., vision encoders) unquantized to protect precision\n",diff --git a/examples/vLLM-sr/d3-mini_W4A16.ipynb b/examples/vLLM-sr/d3-mini_W4A16.ipynbindex 245426f7f..ed61d720d 100644--- a/examples/vLLM-sr/d3-mini_W4A16.ipynb+++ b/examples/vLLM-sr/d3-mini_W4A16.ipynb@@ -163,6 +163,7 @@" alg_configs=alg_config, # Applies the algorithm-specific optimization settings defined above\n"," nsamples=512, # Total calibration dataset samples to maintain quality and avoid overfitting\n"," batch_size=4, # Batch size for calibration; kept < 8 to avoid slow execution/high memory overhead\n",+ " gradient_accumulate_steps=2, # Set to 8 / batch_size to maintain an effective global batch size of 8; smaller values risk accuracy drop \n"," seqlen=4096, # Calibration context length; captures long-range dependency activations\n"," low_gpu_mem_usage=False, # Keeps modules on GPU to maximize speed if sufficient VRAM is available\n"," quant_nontext_module=False, # Leaves non-text modules (e.g., vision encoders) unquantized to protect precision\n",diff --git a/examples/vLLM-sr/d3-nano_W4A16.ipynb b/examples/vLLM-sr/d3-nano_W4A16.ipynbindex f02594aa9..1a511c48a 100644--- a/examples/vLLM-sr/d3-nano_W4A16.ipynb+++ b/examples/vLLM-sr/d3-nano_W4A16.ipynb@@ -163,6 +163,7 @@" alg_configs=alg_config, # Applies the algorithm-specific optimization settings defined above\n"," nsamples=512, # Total calibration dataset samples to maintain quality and avoid overfitting\n"," batch_size=4, # Batch size for calibration; kept < 8 to avoid slow execution/high memory overhead\n",+ " gradient_accumulate_steps=2, # Set to 8 / batch_size to maintain an effective global batch size of 8; smaller values risk accuracy drop \n"," seqlen=4096, # Calibration context length; captures long-range dependency activations\n"," low_gpu_mem_usage=False, # Keeps modules on GPU to maximize speed if sufficient VRAM is available\n"," quant_nontext_module=False, # Leaves non-text modules (e.g., vision encoders) unquantized to protect precision\n",