Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

feat(examples): add batch 2 quantization recipes for Muse-Glimmer, NuExtract3, and Qwen3.5 (#2498)

Vishva Rda52d6da5fa6
diff --git a/examples/Muse-Glimmer/Muse-Glimmer_W2A16.ipynb b/examples/Muse-Glimmer/Muse-Glimmer_W2A16.ipynb
new file mode 100644
index 000000000..5600fb801
--- /dev/null
+++ b/examples/Muse-Glimmer/Muse-Glimmer_W2A16.ipynb
@@ -0,0 +1,243 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "19aa22cb",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# %pip install --upgrade transformers --break-system-packages"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "dc47bad2",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# %pip install auto-round --break-system-packages"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "a8040c46",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import os\n",
+ "\n",
+ "import torch\n",
+ "from auto_round import AutoRound, SignRoundConfig\n",
+ "from huggingface_hub import HfApi, create_repo, get_token, notebook_login\n",
+ "from transformers import AutoModelForImageTextToText, AutoProcessor\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "1cddaebc",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\""
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "6517109c",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "print(f\"PyTorch Version: {torch.__version__}\")\n",
+ "print(f\"CUDA Available: {torch.cuda.is_available()}\")\n",
+ "\n",
+ "if torch.cuda.is_available():\n",
+ " print(f\"CUDA Version: {torch.version.cuda}\")\n",
+ " print(f\"GPU Name: {torch.cuda.get_device_name(0)}\")\n",
+ " print(f\"VRAM: {torch.cuda.get_device_properties(0).total_memory / 1024**3:.1f} GB\")\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "4920035b",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "notebook_login()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "48b05d64",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "MODEL_ID = \"meta-models/Muse-Glimmer-30B\"\n",
+ "HF_USER = \"your_hf_username\" # Replace with your Hugging Face username\n",
+ "OUTPUT_BASE_DIR = \"./AutoRound\"\n",
+ "LOCAL_PATH = \"./local_model\""
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "1b1c02ea",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "!hf download $MODEL_ID --local-dir $LOCAL_PATH"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "f99b1719",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "model = AutoModelForImageTextToText.from_pretrained(\n",
+ " LOCAL_PATH, dtype=torch.bfloat16, device_map=\"auto\"\n",
+ ")\n",
+ "processor = AutoProcessor.from_pretrained(LOCAL_PATH)\n",
+ "\n",
+ "tokenizer = processor.tokenizer\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "f02d144d",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Configure optimization parameters for the AutoRound / SignRound algorithm\n",
+ "alg_config = SignRoundConfig(\n",
+ " iters=800, # Optimization iterations per block; high accuracy for production grade\n",
+ " group_size=64, # Number of weights sharing one scale. Cannot go below 32 (vLLM's GPTQ kernel won't run it). Only use 16 if targeting NVFP4.\n",
+ " sym=True, # Symmetric quantization; enables faster compute and broader engine compatibility\n",
+ ")\n",
+ "\n",
+ "# Initialize the AutoRound quantization pipeline\n",
+ "ar = AutoRound(\n",
+ " model=model, # Target pretrained model to be quantized\n",
+ " tokenizer=tokenizer, # Tokenizer used to process text calibration samples\n",
+ " processor=processor, # Processor for multimodal models (vision/audio), if present\n",
+ " scheme=\"W2A16\", # Quantization precision: 2-bit weights and 16-bit activations\n",
+ " alg_configs=alg_config, # Applies the algorithm-specific optimization settings defined above\n",
+ " nsamples=512, # Total calibration dataset samples to maintain quality and avoid overfitting\n",
+ " batch_size=4, # Batch size for calibration; reduced to manage GPU memory usage on 48GB VRAM\n",
+ " seqlen=2048, # Calibration context length; captures long-range dependency activations\n",
+ " low_gpu_mem_usage=False, # Keeps modules on GPU to maximize speed if sufficient VRAM is available\n",
+ " enable_torch_compile=True, # Compiles forward passes using torch.compile to accelerate calibration\n",
+ " quant_nontext_module=False, # Leaves non-text modules (e.g., vision encoders) unquantized to protect precision\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "dc7a478f",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Quantize once and export to disk across all major loader schemas\n",
+ "ar.quantize_and_save(\n",
+ " OUTPUT_BASE_DIR,\n",
+ " # Serializes weights to 3 formats simultaneously to guarantee compatibility:\n",
+ " # - 'auto_round': Reference AutoRound format (vLLM GPTQ/Marlin, Transformers / native loader)\n",
+ " # - 'auto_gptq': AutoGPTQ tensor layout (vLLM GPTQ/Marlin, TGI)\n",
+ " format=\"auto_round,auto_gptq\",\n",
+ " # Overwrites weights in-place to avoid extra VRAM allocation\n",
+ " inplace=True,\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "97d4d929",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def push_to_hub(local_dir, repo_name, token):\n",
+ " \"\"\"Creates repo and uploads folder to Hugging Face.\"\"\"\n",
+ " full_repo_id = f\"{HF_USER}/{repo_name}\"\n",
+ " print(f\"\\n[Hub] Pushing {local_dir} to {full_repo_id}...\")\n",
+ "\n",
+ " try:\n",
+ " api = HfApi()\n",
+ " create_repo(\n",
+ " full_repo_id, repo_type=\"model\", exist_ok=True, private=False, token=token\n",
+ " )\n",
+ "\n",
+ " api.upload_folder(\n",
+ " folder_path=local_dir, repo_id=full_repo_id, repo_type=\"model\", token=token\n",
+ " )\n",
+ " print(f\"[Hub] ✅ Successfully uploaded: https://huggingface.co/{full_repo_id}\")\n",
+ " except Exception as e:\n",
+ " print(f\"[Hub] ❌ Error uploading: {e}\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "6022046c",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "base_name = MODEL_ID.split(\"/\")[-1]\n",
+ "hf_token = get_token()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "38aba13d",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "if hf_token:\n",
+ " push_to_hub(\n",
+ " os.path.join(OUTPUT_BASE_DIR, \"local_model-w4g64/auto-round-auto-gptq\"),\n",
+ " f\"{base_name}-W4A16-AutoRound\",\n",
+ " hf_token,\n",
+ " )\n",
+ " push_to_hub(\n",
+ " os.path.join(OUTPUT_BASE_DIR, \"local_model-w4g64/auto-gptq\"),\n",
+ " f\"{base_name}-W4A16-AutoRound-GPTQ\",\n",
+ " hf_token,\n",
+ " )\n",
+ "\n",
+ "else:\n",
+ " print(\"No Hugging Face token found. Skipping upload to hub.\")"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.12.3"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/examples/Muse-Glimmer/Muse-Glimmer_W4A16.ipynb b/examples/Muse-Glimmer/Muse-Glimmer_W4A16.ipynb
new file mode 100644
index 000000000..fd28d1176
--- /dev/null
+++ b/examples/Muse-Glimmer/Muse-Glimmer_W4A16.ipynb
@@ -0,0 +1,243 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "19aa22cb",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# %pip install --upgrade transformers --break-system-packages"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "dc47bad2",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# %pip install auto-round --break-system-packages"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "a8040c46",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import os\n",
+ "\n",
+ "import torch\n",
+ "from auto_round import AutoRound, SignRoundConfig\n",
+ "from huggingface_hub import HfApi, create_repo, get_token, notebook_login\n",
+ "from transformers import AutoModelForImageTextToText, AutoProcessor\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "1cddaebc",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\""
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "6517109c",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "print(f\"PyTorch Version: {torch.__version__}\")\n",
+ "print(f\"CUDA Available: {torch.cuda.is_available()}\")\n",
+ "\n",
+ "if torch.cuda.is_available():\n",
+ " print(f\"CUDA Version: {torch.version.cuda}\")\n",
+ " print(f\"GPU Name: {torch.cuda.get_device_name(0)}\")\n",
+ " print(f\"VRAM: {torch.cuda.get_device_properties(0).total_memory / 1024**3:.1f} GB\")\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "4920035b",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "notebook_login()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "48b05d64",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "MODEL_ID = \"meta-models/Muse-Glimmer-30B\"\n",
+ "HF_USER = \"your_hf_username\" # Replace with your Hugging Face username\n",
+ "OUTPUT_BASE_DIR = \"./AutoRound\"\n",
+ "LOCAL_PATH = \"./local_model\""
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "1b1c02ea",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "!hf download $MODEL_ID --local-dir $LOCAL_PATH"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "f99b1719",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "model = AutoModelForImageTextToText.from_pretrained(\n",
+ " LOCAL_PATH, dtype=torch.bfloat16, device_map=\"auto\"\n",
+ ")\n",
+ "processor = AutoProcessor.from_pretrained(LOCAL_PATH)\n",
+ "\n",
+ "tokenizer = processor.tokenizer\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "f02d144d",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Configure optimization parameters for the AutoRound / SignRound algorithm\n",
+ "alg_config = SignRoundConfig(\n",
+ " iters=800, # Optimization iterations per block; high accuracy for production grade\n",
+ " group_size=64, # Number of weights sharing one scale. Cannot go below 32 (vLLM's GPTQ kernel won't run it). Only use 16 if targeting NVFP4.\n",
+ " sym=True, # Symmetric quantization; enables faster compute and broader engine compatibility\n",
+ ")\n",
+ "\n",
+ "# Initialize the AutoRound quantization pipeline\n",
+ "ar = AutoRound(\n",
+ " model=model, # Target pretrained model to be quantized\n",
+ " tokenizer=tokenizer, # Tokenizer used to process text calibration samples\n",
+ " processor=processor, # Processor for multimodal models (vision/audio), if present\n",
+ " scheme=\"W4A16\", # Quantization precision: 4-bit weights and 16-bit activations\n",
+ " alg_configs=alg_config, # Applies the algorithm-specific optimization settings defined above\n",
+ " nsamples=512, # Total calibration dataset samples to maintain quality and avoid overfitting\n",
+ " batch_size=4, # Batch size for calibration; reduced to manage GPU memory usage on 48GB VRAM\n",
+ " seqlen=2048, # Calibration context length; captures long-range dependency activations\n",
+ " low_gpu_mem_usage=False, # Keeps modules on GPU to maximize speed if sufficient VRAM is available\n",
+ " enable_torch_compile=True, # Compiles forward passes using torch.compile to accelerate calibration\n",
+ " quant_nontext_module=False, # Leaves non-text modules (e.g., vision encoders) unquantized to protect precision\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "dc7a478f",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Quantize once and export to disk across all major loader schemas\n",
+ "ar.quantize_and_save(\n",
+ " OUTPUT_BASE_DIR,\n",
+ " # Serializes weights to 3 formats simultaneously to guarantee compatibility:\n",
+ " # - 'auto_round': Reference AutoRound format (vLLM GPTQ/Marlin, Transformers / native loader)\n",
+ " # - 'auto_gptq': AutoGPTQ tensor layout (vLLM GPTQ/Marlin, TGI)\n",
+ " format=\"auto_round,auto_gptq\",\n",
+ " # Overwrites weights in-place to avoid extra VRAM allocation\n",
+ " inplace=True,\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "97d4d929",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def push_to_hub(local_dir, repo_name, token):\n",
+ " \"\"\"Creates repo and uploads folder to Hugging Face.\"\"\"\n",
+ " full_repo_id = f\"{HF_USER}/{repo_name}\"\n",
+ " print(f\"\\n[Hub] Pushing {local_dir} to {full_repo_id}...\")\n",
+ "\n",
+ " try:\n",
+ " api = HfApi()\n",
+ " create_repo(\n",
+ " full_repo_id, repo_type=\"model\", exist_ok=True, private=False, token=token\n",
+ " )\n",
+ "\n",
+ " api.upload_folder(\n",
+ " folder_path=local_dir, repo_id=full_repo_id, repo_type=\"model\", token=token\n",
+ " )\n",
+ " print(f\"[Hub] ✅ Successfully uploaded: https://huggingface.co/{full_repo_id}\")\n",
+ " except Exception as e:\n",
+ " print(f\"[Hub] ❌ Error uploading: {e}\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "6022046c",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "base_name = MODEL_ID.split(\"/\")[-1]\n",
+ "hf_token = get_token()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "38aba13d",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "if hf_token:\n",
+ " push_to_hub(\n",
+ " os.path.join(OUTPUT_BASE_DIR, \"local_model-w4g64/auto-round-auto-gptq\"),\n",
+ " f\"{base_name}-W4A16-AutoRound\",\n",
+ " hf_token,\n",
+ " )\n",
+ " push_to_hub(\n",
+ " os.path.join(OUTPUT_BASE_DIR, \"local_model-w4g64/auto-gptq\"),\n",
+ " f\"{base_name}-W4A16-AutoRound-GPTQ\",\n",
+ " hf_token,\n",
+ " )\n",
+ "\n",
+ "else:\n",
+ " print(\"No Hugging Face token found. Skipping upload to hub.\")"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.12.3"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/examples/NuExtract3/NuExtract3_W4A16.ipynb b/examples/NuExtract3/NuExtract3_W4A16.ipynb
new file mode 100644
index 000000000..032a43106
--- /dev/null
+++ b/examples/NuExtract3/NuExtract3_W4A16.ipynb
@@ -0,0 +1,284 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "19aa22cb",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# %pip install --upgrade transformers --break-system-packages"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "dc47bad2",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# %pip install auto-round --break-system-packages"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "c18b96a7",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# !pip install git+https://github.com/sustcsonglin/flash-linear-attention.git --no-build-isolation --break-system-packages"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "88880cc1",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# %pip install git+https://github.com/Dao-AILab/causal-conv1d.git --no-build-isolation --break-system-packages"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "668521b6",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# %pip install compressed-tensors --break-system-packages"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "a8040c46",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import os\n",
+ "\n",
+ "import torch\n",
+ "from auto_round import AutoRound, SignRoundConfig\n",
+ "from huggingface_hub import HfApi, create_repo, get_token, notebook_login\n",
+ "from transformers import AutoModelForImageTextToText, AutoProcessor\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "1cddaebc",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\""
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "6517109c",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "print(f\"PyTorch Version: {torch.__version__}\")\n",
+ "print(f\"CUDA Available: {torch.cuda.is_available()}\")\n",
+ "\n",
+ "if torch.cuda.is_available():\n",
+ " print(f\"CUDA Version: {torch.version.cuda}\")\n",
+ " print(f\"GPU Name: {torch.cuda.get_device_name(0)}\")\n",
+ " print(f\"VRAM: {torch.cuda.get_device_properties(0).total_memory / 1024**3:.1f} GB\")\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "227d1f33",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "notebook_login()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "48b05d64",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "MODEL_ID = \"numind/NuExtract3\"\n",
+ "HF_USER = \"your_hf_username\" # Replace with your Hugging Face username\n",
+ "OUTPUT_BASE_DIR = \"./NuExtract3\"\n",
+ "LOCAL_PATH = \"./local_model_NuExtract3\""
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "1b1c02ea",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "!hf download $MODEL_ID --local-dir $LOCAL_PATH"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "f99b1719",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "model = AutoModelForImageTextToText.from_pretrained(\n",
+ " LOCAL_PATH, dtype=torch.bfloat16, device_map=\"auto\"\n",
+ ")\n",
+ "processor = AutoProcessor.from_pretrained(LOCAL_PATH)\n",
+ "\n",
+ "tokenizer = processor.tokenizer\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "00d0a924",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Configure optimization parameters for the AutoRound / SignRound algorithm\n",
+ "alg_config = SignRoundConfig(\n",
+ " iters=1000, # Optimization iterations per block; high accuracy for production grade\n",
+ " group_size=32, # Number of weights sharing one scale. Cannot go below 32 (vLLM's GPTQ kernel won't run it). Only use 16 if targeting NVFP4.\n",
+ " sym=True, # Symmetric quantization; enables faster compute and broader engine compatibility\n",
+ ")\n",
+ "\n",
+ "# Initialize the AutoRound quantization pipeline\n",
+ "ar = AutoRound(\n",
+ " model=model, # Target pretrained model to be quantized\n",
+ " tokenizer=tokenizer, # Tokenizer used to process text calibration samples\n",
+ " processor=processor, # Processor for multimodal models (vision/audio), if present\n",
+ " scheme=\"W4A16\", # Quantization precision: 4-bit weights and 16-bit activations\n",
+ " alg_configs=alg_config, # Applies the algorithm-specific optimization settings defined above\n",
+ " nsamples=512, # Total calibration dataset samples to maintain quality and avoid overfitting\n",
+ " batch_size=4, # Batch size for calibration; reduced to manage GPU memory usage on 48GB VRAM\n",
+ " seqlen=4096, # Calibration context length; captures long-range dependency activations\n",
+ " low_gpu_mem_usage=False, # Keeps modules on GPU to maximize speed if sufficient VRAM is available\n",
+ " enable_torch_compile=True, # Compiles forward passes using torch.compile to accelerate calibration\n",
+ " quant_nontext_module=False, # Leaves non-text modules (e.g., vision encoders) unquantized to protect precision\n",
+ " ignore_layers=\"linear_attn,mtp,visual\", # Excludes linear-attention, MTP, and visual modules from quantization\n",
+ " layer_config={\n",
+ " \"mtp\": {\"data_type\": \"bfloat16\"}, # Keeps the MTP module in BF16 precision\n",
+ " \"mtp.fc\": {\"data_type\": \"bfloat16\"}, # Keeps the MTP fully connected layer in BF16 precision\n",
+ " }, # Preserves BF16 precision for the specified MTP layers\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "ec12fa86",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Quantize once and export to disk across all major loader schemas\n",
+ "ar.quantize_and_save(\n",
+ " OUTPUT_BASE_DIR,\n",
+ " # Serializes weights to 3 formats simultaneously to guarantee compatibility:\n",
+ " # - 'auto_round': Reference AutoRound format (vLLM GPTQ/Marlin, Transformers / native loader)\n",
+ " # - 'auto_gptq': AutoGPTQ tensor layout (vLLM GPTQ/Marlin, TGI)\n",
+ " # - 'llm_compressor': Compressed-tensors schema (vLLM / Neural Magic)\n",
+ " format=\"auto_round,auto_gptq,llm_compressor\",\n",
+ " # Overwrites weights in-place to avoid extra VRAM allocation\n",
+ " inplace=True,\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "97d4d929",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def push_to_hub(local_dir, repo_name, token):\n",
+ " \"\"\"Creates repo and uploads folder to Hugging Face.\"\"\"\n",
+ " full_repo_id = f\"{HF_USER}/{repo_name}\"\n",
+ " print(f\"\\n[Hub] Pushing {local_dir} to {full_repo_id}...\")\n",
+ "\n",
+ " try:\n",
+ " api = HfApi()\n",
+ " create_repo(\n",
+ " full_repo_id, repo_type=\"model\", exist_ok=True, private=False, token=token\n",
+ " )\n",
+ "\n",
+ " api.upload_folder(\n",
+ " folder_path=local_dir, repo_id=full_repo_id, repo_type=\"model\", token=token\n",
+ " )\n",
+ " print(f\"[Hub] ✅ Successfully uploaded: https://huggingface.co/{full_repo_id}\")\n",
+ " except Exception as e:\n",
+ " print(f\"[Hub] ❌ Error uploading: {e}\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "6022046c",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "base_name = MODEL_ID.split(\"/\")[-1]\n",
+ "hf_token = get_token()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "38aba13d",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "if hf_token:\n",
+ " push_to_hub(\n",
+ " os.path.join(OUTPUT_BASE_DIR, \"local_model-w4g32/auto-round-auto-gptq\"),\n",
+ " f\"{base_name}-W4A16-AutoRound\",\n",
+ " hf_token,\n",
+ " )\n",
+ " push_to_hub(\n",
+ " os.path.join(OUTPUT_BASE_DIR, \"local_model-w4g32/auto-gptq\"),\n",
+ " f\"{base_name}-W4A16-AutoRound-GPTQ\",\n",
+ " hf_token,\n",
+ " )\n",
+ " push_to_hub(\n",
+ " os.path.join(OUTPUT_BASE_DIR, \"local_model-w4g32/llm-compressor-wint-a16\"),\n",
+ " f\"{base_name}-W4A16-AutoRound-LLM-Compressor\",\n",
+ " hf_token,\n",
+ " )\n",
+ "\n",
+ "else:\n",
+ " print(\"No Hugging Face token found. Skipping upload to hub.\")"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.12.3"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/examples/Qwen3.5/Qwen3_5_0.8B_W4A16.ipynb b/examples/Qwen3.5/Qwen3_5_0.8B_W4A16.ipynb
new file mode 100644
index 000000000..c7d54d82c
--- /dev/null
+++ b/examples/Qwen3.5/Qwen3_5_0.8B_W4A16.ipynb
@@ -0,0 +1,268 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "19aa22cb",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# %pip install --upgrade transformers --break-system-packages"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "dc47bad2",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# %pip install auto-round --break-system-packages"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "7582cf42",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# %pip install git+https://github.com/sustcsonglin/flash-linear-attention.git --no-build-isolation --break-system-packages"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "0458affd",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# %pip install git+https://github.com/Dao-AILab/causal-conv1d.git --no-build-isolation --break-system-packages"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "a8040c46",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import os\n",
+ "\n",
+ "import torch\n",
+ "from auto_round import AutoRound, SignRoundConfig\n",
+ "from huggingface_hub import HfApi, create_repo, get_token, notebook_login\n",
+ "from transformers import AutoModelForImageTextToText, AutoProcessor\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "1cddaebc",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\""
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "6517109c",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "print(f\"PyTorch Version: {torch.__version__}\")\n",
+ "print(f\"CUDA Available: {torch.cuda.is_available()}\")\n",
+ "\n",
+ "if torch.cuda.is_available():\n",
+ " print(f\"CUDA Version: {torch.version.cuda}\")\n",
+ " print(f\"GPU Name: {torch.cuda.get_device_name(0)}\")\n",
+ " print(f\"VRAM: {torch.cuda.get_device_properties(0).total_memory / 1024**3:.1f} GB\")\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "4920035b",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "notebook_login()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "48b05d64",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "MODEL_ID = \"Qwen/Qwen3.5-0.8B\"\n",
+ "HF_USER = \"your_hf_username\" # Replace with your Hugging Face username\n",
+ "OUTPUT_BASE_DIR = \"./AutoRound\"\n",
+ "LOCAL_PATH = \"./local_model\""
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "8f76d7ec",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "!hf download $MODEL_ID --local-dir $LOCAL_PATH"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "f99b1719",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "model = AutoModelForImageTextToText.from_pretrained(\n",
+ " LOCAL_PATH, dtype=torch.bfloat16, device_map=\"auto\"\n",
+ ")\n",
+ "processor = AutoProcessor.from_pretrained(LOCAL_PATH)\n",
+ "\n",
+ "tokenizer = processor.tokenizer\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "93ce95c5",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "\n",
+ "# Configure optimization parameters for the AutoRound / SignRound algorithm\n",
+ "alg_config = SignRoundConfig(\n",
+ " iters=1000, # Optimization iterations per block; high accuracy for production grade\n",
+ " group_size=32, # Number of weights sharing one scale. Cannot go below 32 (vLLM's GPTQ kernel won't run it). Only use 16 if targeting NVFP4.\n",
+ " sym=True, # Symmetric quantization; enables faster compute and broader engine compatibility\n",
+ ")\n",
+ "\n",
+ "# Initialize the AutoRound quantization pipeline\n",
+ "ar = AutoRound(\n",
+ " model=model, # Target pretrained model to be quantized\n",
+ " tokenizer=tokenizer, # Tokenizer used to process text calibration samples\n",
+ " processor=processor, # Processor for multimodal models (vision/audio), if present\n",
+ " scheme=\"W4A16\", # Quantization precision: 4-bit weights and 16-bit activations\n",
+ " alg_configs=alg_config, # Applies the algorithm-specific optimization settings defined above\n",
+ " nsamples=512, # Total calibration dataset samples to maintain quality and avoid overfitting\n",
+ " batch_size=2, # Batch size for calibration; reduced to manage GPU memory usage on 48GB VRAM\n",
+ " seqlen=2048, # Calibration context length; captures long-range dependency activations\n",
+ " low_gpu_mem_usage=False, # Keeps modules on GPU to maximize speed if sufficient VRAM is available\n",
+ " enable_torch_compile=True, # Compiles forward passes using torch.compile to accelerate calibration\n",
+ " quant_nontext_module=False, # Leaves non-text modules (e.g., vision encoders) unquantized to protect precision\n",
+ " ignore_layers=\"linear_attn,mtp,visual\", # Excludes linear-attention, MTP, and visual modules from quantization\n",
+ " layer_config={\n",
+ " \"mtp\": {\"data_type\": \"bfloat16\"}, # Keeps the MTP module in BF16 precision\n",
+ " \"mtp.fc\": {\"data_type\": \"bfloat16\"}, # Keeps the MTP fully connected layer in BF16 precision\n",
+ " }, # Preserves BF16 precision for the specified MTP layers\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "ec12fa86",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Quantize once and export to disk across all major loader schemas\n",
+ "ar.quantize_and_save(\n",
+ " OUTPUT_BASE_DIR,\n",
+ " # Serializes weights to 3 formats simultaneously to guarantee compatibility:\n",
+ " # - 'auto_round': Reference AutoRound format (vLLM GPTQ/Marlin, Transformers / native loader)\n",
+ " # - 'auto_gptq': AutoGPTQ tensor layout (vLLM GPTQ/Marlin, TGI)\n",
+ " format=\"auto_round,auto_gptq\",\n",
+ " # Overwrites weights in-place to avoid extra VRAM allocation\n",
+ " inplace=True,\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "97d4d929",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def push_to_hub(local_dir, repo_name, token):\n",
+ " \"\"\"Creates repo and uploads folder to Hugging Face.\"\"\"\n",
+ " full_repo_id = f\"{HF_USER}/{repo_name}\"\n",
+ " print(f\"\\n[Hub] Pushing {local_dir} to {full_repo_id}...\")\n",
+ "\n",
+ " try:\n",
+ " api = HfApi()\n",
+ " create_repo(\n",
+ " full_repo_id, repo_type=\"model\", exist_ok=True, private=False, token=token\n",
+ " )\n",
+ "\n",
+ " api.upload_folder(\n",
+ " folder_path=local_dir, repo_id=full_repo_id, repo_type=\"model\", token=token\n",
+ " )\n",
+ " print(f\"[Hub] ✅ Successfully uploaded: https://huggingface.co/{full_repo_id}\")\n",
+ " except Exception as e:\n",
+ " print(f\"[Hub] ❌ Error uploading: {e}\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "6022046c",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "base_name = MODEL_ID.split(\"/\")[-1]\n",
+ "hf_token = get_token()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "38aba13d",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "if hf_token:\n",
+ " push_to_hub(\n",
+ " os.path.join(OUTPUT_BASE_DIR, \"local_model-w2g32/auto-round-auto-gptq\"),\n",
+ " f\"{base_name}-W2A16-AutoRound\",\n",
+ " hf_token,\n",
+ " )\n",
+ " push_to_hub(\n",
+ " os.path.join(OUTPUT_BASE_DIR, \"local_model-w2g32/auto-gptq\"),\n",
+ " f\"{base_name}-W2A16-AutoRound-GPTQ\",\n",
+ " hf_token,\n",
+ " )\n",
+ "else:\n",
+ " print(\"No Hugging Face token found. Skipping upload to hub.\")"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.12.3"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/examples/Qwen3.5/Qwen3_5_4B_W2A16.ipynb b/examples/Qwen3.5/Qwen3_5_4B_W2A16.ipynb
new file mode 100644
index 000000000..f22b28de0
--- /dev/null
+++ b/examples/Qwen3.5/Qwen3_5_4B_W2A16.ipynb
@@ -0,0 +1,267 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "19aa22cb",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# %pip install --upgrade transformers --break-system-packages"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "dc47bad2",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# %pip install auto-round --break-system-packages"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "c18b96a7",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# %pip install git+https://github.com/sustcsonglin/flash-linear-attention.git --no-build-isolation --break-system-packages"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "1ecf0f22",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# %pip install git+https://github.com/Dao-AILab/causal-conv1d.git --no-build-isolation --break-system-packages"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "a8040c46",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import os\n",
+ "import torch\n",
+ "from auto_round import AutoRound, SignRoundConfig\n",
+ "from huggingface_hub import HfApi, create_repo, notebook_login, get_token\n",
+ "from transformers import AutoModelForImageTextToText, AutoProcessor"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "1cddaebc",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\""
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "6517109c",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "print(f\"PyTorch Version: {torch.__version__}\")\n",
+ "print(f\"CUDA Available: {torch.cuda.is_available()}\")\n",
+ "\n",
+ "if torch.cuda.is_available():\n",
+ " print(f\"CUDA Version: {torch.version.cuda}\")\n",
+ " print(f\"GPU Name: {torch.cuda.get_device_name(0)}\")\n",
+ " print(f\"VRAM: {torch.cuda.get_device_properties(0).total_memory / 1024**3:.1f} GB\")\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "4920035b",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "notebook_login()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "48b05d64",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "MODEL_ID = \"Qwen/Qwen3.5-4B\"\n",
+ "HF_USER = \"your_hf_username\" # Replace with your Hugging Face username\n",
+ "OUTPUT_BASE_DIR = \"./AutoRound\"\n",
+ "LOCAL_PATH = \"./local_model\""
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "1b1c02ea",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "!hf download $MODEL_ID --local-dir $LOCAL_PATH"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "f99b1719",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "model = AutoModelForImageTextToText.from_pretrained(\n",
+ " LOCAL_PATH, dtype=torch.bfloat16, device_map=\"auto\"\n",
+ ")\n",
+ "processor = AutoProcessor.from_pretrained(LOCAL_PATH)\n",
+ "\n",
+ "tokenizer = processor.tokenizer\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "00d0a924",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "\n",
+ "# Configure optimization parameters for the AutoRound / SignRound algorithm\n",
+ "alg_config = SignRoundConfig(\n",
+ " iters=1000, # Optimization iterations per block; balances calibration time and accuracy\n",
+ " group_size=32, # Number of weights sharing one scale. Cannot go below 32 (vLLM's GPTQ kernel won't run it). Only use 16 if targeting NVFP4.\n",
+ " sym=True, # Symmetric quantization; enables faster compute and broader engine compatibility\n",
+ ")\n",
+ "\n",
+ "# Initialize the AutoRound quantization pipeline\n",
+ "ar = AutoRound(\n",
+ " model=model, # Target pretrained model to be quantized\n",
+ " tokenizer=tokenizer, # Tokenizer used to process text calibration samples\n",
+ " processor=processor, # Processor for multimodal models (vision/audio), if present\n",
+ " scheme=\"W2A16\", # 2-bit weights and 16-bit activations for aggressive weight compression\n",
+ " alg_configs=alg_config, # Applies the algorithm-specific optimization settings defined above\n",
+ " nsamples=512, # Total calibration dataset samples to maintain quality and avoid overfitting\n",
+ " batch_size=2, # Batch size for calibration; balances speed and GPU memory usage\n",
+ " seqlen=2048, # Calibration context length; captures long-range dependency activations\n",
+ " low_gpu_mem_usage=False, # Keeps modules on GPU to maximize speed if sufficient VRAM is available\n",
+ " enable_torch_compile=True, # Compiles forward passes using torch.compile to accelerate calibration\n",
+ " quant_nontext_module=False, # Leaves non-text modules (e.g., vision encoders) unquantized to protect precision\n",
+ " layer_config={ # Preserves BF16 precision for the specified MTP layers\n",
+ " \"mtp\": {\"data_type\": \"bfloat16\"}, # Keeps the MTP module in BF16 precision\n",
+ " \"mtp.fc\": {\"data_type\": \"bfloat16\"}, # Keeps the MTP fully connected layer in BF16 precision\n",
+ " }, \n",
+ " enable_alg_ext=True, # Enables AutoRound algorithm extensions for supported quantization workflows\n",
+ ")\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "ec12fa86",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Quantize once and export to disk across all major loader schemas\n",
+ "ar.quantize_and_save(\n",
+ " OUTPUT_BASE_DIR,\n",
+ " # Serializes weights to 3 formats simultaneously to guarantee compatibility:\n",
+ " # - 'auto_round': Reference AutoRound format (vLLM GPTQ/Marlin, Transformers / native loader)\n",
+ " # - 'auto_gptq': AutoGPTQ tensor layout (vLLM GPTQ/Marlin, TGI)\n",
+ " format=\"auto_round,auto_gptq\",\n",
+ " # Overwrites weights in-place to avoid extra VRAM allocation\n",
+ " inplace=True\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "97d4d929",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def push_to_hub(local_dir, repo_name, token):\n",
+ " \"\"\"Creates repo and uploads folder to Hugging Face.\"\"\"\n",
+ " full_repo_id = f\"{HF_USER}/{repo_name}\"\n",
+ " print(f\"\\n[Hub] Pushing {local_dir} to {full_repo_id}...\")\n",
+ "\n",
+ " try:\n",
+ " api = HfApi()\n",
+ " create_repo(\n",
+ " full_repo_id, repo_type=\"model\", exist_ok=True, private=False, token=token\n",
+ " )\n",
+ "\n",
+ " api.upload_folder(\n",
+ " folder_path=local_dir, repo_id=full_repo_id, repo_type=\"model\", token=token\n",
+ " )\n",
+ " print(f\"[Hub] ✅ Successfully uploaded: https://huggingface.co/{full_repo_id}\")\n",
+ " except Exception as e:\n",
+ " print(f\"[Hub] ❌ Error uploading: {e}\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "6022046c",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "base_name = MODEL_ID.split(\"/\")[-1]\n",
+ "hf_token = get_token()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "38aba13d",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "if hf_token:\n",
+ " push_to_hub(\n",
+ " os.path.join(OUTPUT_BASE_DIR, \"local_model-w2g32/auto-round-auto-gptq\"),\n",
+ " f\"{base_name}-W2A16-AutoRound\",\n",
+ " hf_token,\n",
+ " )\n",
+ " push_to_hub(\n",
+ " os.path.join(OUTPUT_BASE_DIR, \"local_model-w2g32/auto-gptq\"),\n",
+ " f\"{base_name}-W2A16-AutoRound-GPTQ\",\n",
+ " hf_token,\n",
+ " )\n",
+ "else:\n",
+ " print(\"No Hugging Face token found. Skipping upload to hub.\")"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.12.3"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/examples/Qwen3.5/Qwen3_5_4B_W4A16.ipynb b/examples/Qwen3.5/Qwen3_5_4B_W4A16.ipynb
new file mode 100644
index 000000000..7358fee3e
--- /dev/null
+++ b/examples/Qwen3.5/Qwen3_5_4B_W4A16.ipynb
@@ -0,0 +1,284 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "19aa22cb",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# %pip install --upgrade transformers --break-system-packages"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "dc47bad2",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# %pip install auto-round --break-system-packages"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "c18b96a7",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# %pip install git+https://github.com/sustcsonglin/flash-linear-attention.git --no-build-isolation --break-system-packages"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "c50dc7d5",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# %pip install git+https://github.com/Dao-AILab/causal-conv1d.git --no-build-isolation --break-system-packages"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "71ca5b44",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# %pip install compressed-tensors --break-system-packages"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "a8040c46",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import os\n",
+ "\n",
+ "import torch\n",
+ "from auto_round import AutoRound, SignRoundConfig\n",
+ "from huggingface_hub import HfApi, create_repo, get_token, notebook_login\n",
+ "from transformers import AutoModelForImageTextToText, AutoProcessor\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "1cddaebc",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\""
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "6517109c",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "print(f\"PyTorch Version: {torch.__version__}\")\n",
+ "print(f\"CUDA Available: {torch.cuda.is_available()}\")\n",
+ "\n",
+ "if torch.cuda.is_available():\n",
+ " print(f\"CUDA Version: {torch.version.cuda}\")\n",
+ " print(f\"GPU Name: {torch.cuda.get_device_name(0)}\")\n",
+ " print(f\"VRAM: {torch.cuda.get_device_properties(0).total_memory / 1024**3:.1f} GB\")\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "4920035b",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "notebook_login()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "48b05d64",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "MODEL_ID = \"Qwen/Qwen3.5-4B\"\n",
+ "HF_USER = \"your_hf_username\" # Replace with your Hugging Face username\n",
+ "OUTPUT_BASE_DIR = \"./AutoRound\"\n",
+ "LOCAL_PATH = \"./local_model\""
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "1b1c02ea",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "!hf download $MODEL_ID --local-dir $LOCAL_PATH"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "f99b1719",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "model = AutoModelForImageTextToText.from_pretrained(\n",
+ " LOCAL_PATH, dtype=torch.bfloat16, device_map=\"auto\"\n",
+ ")\n",
+ "processor = AutoProcessor.from_pretrained(LOCAL_PATH)\n",
+ "\n",
+ "tokenizer = processor.tokenizer\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "00d0a924",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Configure optimization parameters for the AutoRound / SignRound algorithm\n",
+ "alg_config = SignRoundConfig(\n",
+ " iters=1000, # Optimization iterations per block; high accuracy for production grade\n",
+ " group_size=32, # Number of weights sharing one scale. Cannot go below 32 (vLLM's GPTQ kernel won't run it). Only use 16 if targeting NVFP4.\n",
+ " sym=True, # Symmetric quantization; enables faster compute and broader engine compatibility\n",
+ ")\n",
+ "\n",
+ "# Initialize the AutoRound quantization pipeline\n",
+ "ar = AutoRound(\n",
+ " model=model, # Target pretrained model to be quantized\n",
+ " tokenizer=tokenizer, # Tokenizer used to process text calibration samples\n",
+ " processor=processor, # Processor for multimodal models (vision/audio), if present\n",
+ " scheme=\"W4A16\", # Quantization precision: 4-bit weights and 16-bit activations\n",
+ " alg_configs=alg_config, # Applies the algorithm-specific optimization settings defined above\n",
+ " nsamples=512, # Total calibration dataset samples to maintain quality and avoid overfitting\n",
+ " batch_size=2, # Batch size for calibration; reduced to manage GPU memory usage on 48GB VRAM\n",
+ " seqlen=2048, # Calibration context length; captures long-range dependency activations\n",
+ " low_gpu_mem_usage=False, # Keeps modules on GPU to maximize speed if sufficient VRAM is available\n",
+ " enable_torch_compile=True, # Compiles forward passes using torch.compile to accelerate calibration\n",
+ " quant_nontext_module=False, # Leaves non-text modules (e.g., vision encoders) unquantized to protect precision\n",
+ " ignore_layers=\"linear_attn,mtp,visual\", # Excludes linear-attention, MTP, and visual modules from quantization\n",
+ " layer_config={\n",
+ " \"mtp\": {\"data_type\": \"bfloat16\"}, # Keeps the MTP module in BF16 precision\n",
+ " \"mtp.fc\": {\"data_type\": \"bfloat16\"}, # Keeps the MTP fully connected layer in BF16 precision\n",
+ " }, # Preserves BF16 precision for the specified MTP layers\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "ec12fa86",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Quantize once and export to disk across all major loader schemas\n",
+ "ar.quantize_and_save(\n",
+ " OUTPUT_BASE_DIR,\n",
+ " # Serializes weights to 3 formats simultaneously to guarantee compatibility:\n",
+ " # - 'auto_round': Reference AutoRound format (vLLM GPTQ/Marlin, Transformers / native loader)\n",
+ " # - 'auto_gptq': AutoGPTQ tensor layout (vLLM GPTQ/Marlin, TGI)\n",
+ " # - 'llm_compressor': Compressed-tensors schema (vLLM / Neural Magic)\n",
+ " format=\"auto_round,auto_gptq,llm_compressor\",\n",
+ " # Overwrites weights in-place to avoid extra VRAM allocation\n",
+ " inplace=True,\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "97d4d929",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def push_to_hub(local_dir, repo_name, token):\n",
+ " \"\"\"Creates repo and uploads folder to Hugging Face.\"\"\"\n",
+ " full_repo_id = f\"{HF_USER}/{repo_name}\"\n",
+ " print(f\"\\n[Hub] Pushing {local_dir} to {full_repo_id}...\")\n",
+ "\n",
+ " try:\n",
+ " api = HfApi()\n",
+ " create_repo(\n",
+ " full_repo_id, repo_type=\"model\", exist_ok=True, private=False, token=token\n",
+ " )\n",
+ "\n",
+ " api.upload_folder(\n",
+ " folder_path=local_dir, repo_id=full_repo_id, repo_type=\"model\", token=token\n",
+ " )\n",
+ " print(f\"[Hub] ✅ Successfully uploaded: https://huggingface.co/{full_repo_id}\")\n",
+ " except Exception as e:\n",
+ " print(f\"[Hub] ❌ Error uploading: {e}\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "6022046c",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "base_name = MODEL_ID.split(\"/\")[-1]\n",
+ "hf_token = get_token()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "38aba13d",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "if hf_token:\n",
+ " push_to_hub(\n",
+ " os.path.join(OUTPUT_BASE_DIR, \"local_model-w4g32/auto-round-auto-gptq\"),\n",
+ " f\"{base_name}-W4A16-AutoRound\",\n",
+ " hf_token,\n",
+ " )\n",
+ " push_to_hub(\n",
+ " os.path.join(OUTPUT_BASE_DIR, \"local_model-w4g32/auto-gptq\"),\n",
+ " f\"{base_name}-W4A16-AutoRound-GPTQ\",\n",
+ " hf_token,\n",
+ " )\n",
+ " push_to_hub(\n",
+ " os.path.join(OUTPUT_BASE_DIR, \"local_model-w4g32/llm-compressor-wint-a16\"),\n",
+ " f\"{base_name}-W4A16-AutoRound-LLM-Compressor\",\n",
+ " hf_token,\n",
+ " )\n",
+ "\n",
+ "else:\n",
+ " print(\"No Hugging Face token found. Skipping upload to hub.\")"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.12.3"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/examples/Qwen3.5/Qwen3_5_9B_W4A16.ipynb b/examples/Qwen3.5/Qwen3_5_9B_W4A16.ipynb
new file mode 100644
index 000000000..6f0f33765
--- /dev/null
+++ b/examples/Qwen3.5/Qwen3_5_9B_W4A16.ipynb
@@ -0,0 +1,308 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "19aa22cb",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# %pip install --upgrade transformers --break-system-packages"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "dc47bad2",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# %pip install auto-round --break-system-packages"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "c18b96a7",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# %pip install git+https://github.com/sustcsonglin/flash-linear-attention.git --no-build-isolation --break-system-packages"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "e0071e3e",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# %pip install git+https://github.com/Dao-AILab/causal-conv1d.git --no-build-isolation --break-system-packages"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "df0ee49b",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# %pip install compressed-tensors --break-system-packages"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "a8040c46",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import os\n",
+ "\n",
+ "import torch\n",
+ "from auto_round import AutoRound, SignRoundConfig\n",
+ "from huggingface_hub import HfApi, create_repo, get_token, notebook_login\n",
+ "from transformers import AutoModelForImageTextToText, AutoProcessor\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "1cddaebc",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\""
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "6517109c",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "print(f\"PyTorch Version: {torch.__version__}\")\n",
+ "print(f\"CUDA Available: {torch.cuda.is_available()}\")\n",
+ "\n",
+ "if torch.cuda.is_available():\n",
+ " print(f\"CUDA Version: {torch.version.cuda}\")\n",
+ " print(f\"GPU Name: {torch.cuda.get_device_name(0)}\")\n",
+ " print(f\"VRAM: {torch.cuda.get_device_properties(0).total_memory / 1024**3:.1f} GB\")\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "4920035b",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "notebook_login()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "48b05d64",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "MODEL_ID = \"Qwen/Qwen3.5-9B\"\n",
+ "HF_USER = \"your_hf_username\" # Replace with your Hugging Face username\n",
+ "OUTPUT_BASE_DIR = \"./AutoRound\"\n",
+ "LOCAL_PATH = \"./local_model\""
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "1b1c02ea",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "!hf download $MODEL_ID --local-dir $LOCAL_PATH"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "f99b1719",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "model = AutoModelForImageTextToText.from_pretrained(\n",
+ " LOCAL_PATH, dtype=torch.bfloat16, device_map=\"auto\"\n",
+ ")\n",
+ "processor = AutoProcessor.from_pretrained(LOCAL_PATH)\n",
+ "\n",
+ "tokenizer = processor.tokenizer\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "7c1063ff",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "TUNING_CONFIG = {\n",
+ " \"group_size\": 32,\n",
+ " \"sym\": True,\n",
+ " \"iters\": 1200, # High accuracy (Production grade)\n",
+ " \"nsamples\": 512, # More calibration data\n",
+ " \"batch_size\": 4,\n",
+ " \"seqlen\": 4096,\n",
+ " \"low_gpu_mem_usage\": False, # Keep on GPU for speed\n",
+ " \"enable_torch_compile\": True, # JIT acceleration\n",
+ " \"quant_nontext_module\": False, # Keep Vision Tower in FP16 (Crucial for VLM accuracy)\n",
+ " \"layer_config\": {\n",
+ " \"mtp\": {\"data_type\": \"bfloat16\"},\n",
+ " \"mtp.fc\": {\"data_type\": \"bfloat16\"},\n",
+ " },\n",
+ "}"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "00d0a924",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Configure optimization parameters for the AutoRound / SignRound algorithm\n",
+ "alg_config = SignRoundConfig(\n",
+ " iters=1200, # Optimization iterations per block; high accuracy (Production grade)\n",
+ " group_size=32, # Number of weights sharing one scale. Cannot go below 32 (vLLM's GPTQ kernel won't run it). Only use 16 if targeting NVFP4.\n",
+ " sym=True, # Symmetric quantization; enables faster compute and broader engine compatibility\n",
+ ")\n",
+ "\n",
+ "# Initialize the AutoRound quantization pipeline\n",
+ "ar = AutoRound(\n",
+ " model=model, # Target pretrained model to be quantized\n",
+ " tokenizer=tokenizer, # Tokenizer used to process text calibration samples\n",
+ " processor=processor, # Processor for multimodal models (vision/audio), if present\n",
+ " scheme=\"W4A16\", # Quantization precision: 4-bit weights and 16-bit activations\n",
+ " alg_configs=alg_config, # Applies the algorithm-specific optimization settings defined above\n",
+ " nsamples=512, # Total calibration dataset samples to maintain quality and avoid overfitting\n",
+ " batch_size=4, # Batch size for calibration; keep < 8 to avoid slow execution/high memory overhead\n",
+ " seqlen=4096, # Calibration context length; captures long-range dependency activations\n",
+ " low_gpu_mem_usage=False, # Keeps modules on GPU to maximize speed if sufficient VRAM is available\n",
+ " enable_torch_compile=True, # Compiles forward passes using torch.compile to accelerate calibration (JIT acceleration)\n",
+ " quant_nontext_module=False, # Leaves non-text modules (e.g., vision encoders) unquantized to protect precision (Crucial for VLM accuracy)\n",
+ " ignore_layers=\"linear_attn,mtp,visual\", # Excludes linear-attention, MTP, and visual modules from quantization\n",
+ " layer_config={\n",
+ " \"mtp\": {\"data_type\": \"bfloat16\"}, # Keeps the MTP module in BF16 precision\n",
+ " \"mtp.fc\": {\"data_type\": \"bfloat16\"}, # Keeps the MTP fully connected layer in BF16 precision\n",
+ " }, # Preserves BF16 precision for the specified MTP layers\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "ec12fa86",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Quantize once and export to disk across all major loader schemas\n",
+ "ar.quantize_and_save(\n",
+ " OUTPUT_BASE_DIR,\n",
+ " # Serializes weights to 3 formats simultaneously to guarantee compatibility:\n",
+ " # - 'auto_round': Reference AutoRound format (vLLM GPTQ/Marlin, Transformers / native loader)\n",
+ " # - 'auto_gptq': AutoGPTQ tensor layout (vLLM GPTQ/Marlin, TGI)\n",
+ " # - 'llm_compressor': Compressed-tensors schema (vLLM / Neural Magic)\n",
+ " format=\"auto_round,auto_gptq,llm_compressor\",\n",
+ " # Overwrites weights in-place to avoid extra VRAM allocation\n",
+ " inplace=True,\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "97d4d929",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def push_to_hub(local_dir, repo_name, token):\n",
+ " \"\"\"Creates repo and uploads folder to Hugging Face.\"\"\"\n",
+ " full_repo_id = f\"{HF_USER}/{repo_name}\"\n",
+ " print(f\"\\n[Hub] Pushing {local_dir} to {full_repo_id}...\")\n",
+ "\n",
+ " try:\n",
+ " api = HfApi()\n",
+ " create_repo(\n",
+ " full_repo_id, repo_type=\"model\", exist_ok=True, private=False, token=token\n",
+ " )\n",
+ "\n",
+ " api.upload_folder(\n",
+ " folder_path=local_dir, repo_id=full_repo_id, repo_type=\"model\", token=token\n",
+ " )\n",
+ " print(f\"[Hub] ✅ Successfully uploaded: https://huggingface.co/{full_repo_id}\")\n",
+ " except Exception as e:\n",
+ " print(f\"[Hub] ❌ Error uploading: {e}\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "6022046c",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "base_name = MODEL_ID.split(\"/\")[-1]\n",
+ "hf_token = get_token()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "38aba13d",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "if hf_token:\n",
+ " push_to_hub(\n",
+ " os.path.join(OUTPUT_BASE_DIR, \"local_model-w4g32/auto-round-auto-gptq\"),\n",
+ " f\"{base_name}-W4A16-AutoRound\",\n",
+ " hf_token,\n",
+ " )\n",
+ " push_to_hub(\n",
+ " os.path.join(OUTPUT_BASE_DIR, \"local_model-w4g32/auto-gptq\"),\n",
+ " f\"{base_name}-W4A16-AutoRound-GPTQ\",\n",
+ " hf_token,\n",
+ " )\n",
+ " push_to_hub(\n",
+ " os.path.join(OUTPUT_BASE_DIR, \"local_model-w4g32/llm-compressor-wint-a16\"),\n",
+ " f\"{base_name}-W4A16-AutoRound-LLM-Compressor\",\n",
+ " hf_token,\n",
+ " )\n",
+ "\n",
+ "else:\n",
+ " print(\"No Hugging Face token found. Skipping upload to hub.\")"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.12.3"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}