Skip to content

AirLLM 2.8T LLM inference with single 4GB GPU

update test mixtral ipynb

Yu Lib4229e45347b
diff --git a/air_llm/tests/test_notebooks/test_mixtral.ipynb b/air_llm/tests/test_notebooks/test_mixtral.ipynb
index a3afdc5..d2f71da 100644
--- a/air_llm/tests/test_notebooks/test_mixtral.ipynb
+++ b/air_llm/tests/test_notebooks/test_mixtral.ipynb
@@ -99,507 +99,9 @@
"!pip install -U airllm transformers"
]
},
- {
- "cell_type": "code",
- "execution_count": 2,
- "id": "47de26bf-510a-4f8f-ae99-c4c22a0e12b7",
- "metadata": {},
- "outputs": [
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "d8829a3be8414a3d82aacee5f801ece7",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "config.json: 0%| | 0.00/720 [00:00<?, ?B/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "aed2af7d16004b849a401749c74346c8",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "model.safetensors.index.json: 0%| | 0.00/92.7k [00:00<?, ?B/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "e68dab4287b3496ba453c2664e167d4e",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Downloading shards: 0%| | 0/19 [00:00<?, ?it/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "93c15f55db52460cb5ff5bffd604876e",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "model-00001-of-00019.safetensors: 0%| | 0.00/4.89G [00:00<?, ?B/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "02299832dca2482ba4e84b5a34b6604d",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "model-00002-of-00019.safetensors: 0%| | 0.00/4.98G [00:00<?, ?B/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "83dfc1d781dc4263958767984c60e5f3",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "model-00003-of-00019.safetensors: 0%| | 0.00/4.98G [00:00<?, ?B/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "1838e6535eed4020bbc52404a5f0e40e",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "model-00004-of-00019.safetensors: 0%| | 0.00/4.90G [00:00<?, ?B/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "b0282171d02d43c883e0918139336e7b",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "model-00005-of-00019.safetensors: 0%| | 0.00/4.98G [00:00<?, ?B/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "18a1e681ca7c40d78a2ecb7be9821548",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "model-00006-of-00019.safetensors: 0%| | 0.00/4.98G [00:00<?, ?B/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "da04e3e27395458b87c8320150cb5167",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "model-00007-of-00019.safetensors: 0%| | 0.00/4.90G [00:00<?, ?B/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "d3af708bd5984d4b8c6552afbbbf7767",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "model-00008-of-00019.safetensors: 0%| | 0.00/4.98G [00:00<?, ?B/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "05c6af1b24224cc6b7578e6ea8fbf0e0",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "model-00009-of-00019.safetensors: 0%| | 0.00/4.98G [00:00<?, ?B/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "33b169a15603482281fd45ab12a1c1e7",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "model-00010-of-00019.safetensors: 0%| | 0.00/4.90G [00:00<?, ?B/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "010af99eef2b4955a798bcbac95b3789",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "model-00011-of-00019.safetensors: 0%| | 0.00/4.98G [00:00<?, ?B/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "627af607d99c421688698093888c9da6",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "model-00012-of-00019.safetensors: 0%| | 0.00/4.98G [00:00<?, ?B/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "2cd9e5c547024ffca634e1a97f0b8933",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "model-00013-of-00019.safetensors: 0%| | 0.00/4.98G [00:00<?, ?B/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "e8c7cd63a0794a668820c5f1d7bb90ca",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "model-00014-of-00019.safetensors: 0%| | 0.00/4.90G [00:00<?, ?B/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "d6154839ba9946fbbf27563d8c76ee77",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "model-00015-of-00019.safetensors: 0%| | 0.00/4.98G [00:00<?, ?B/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "ad8792741a54426f8db778ea5203b6cc",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "model-00016-of-00019.safetensors: 0%| | 0.00/4.98G [00:00<?, ?B/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "c72707d9eee54201a36c94e0a1ea0a79",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "model-00017-of-00019.safetensors: 0%| | 0.00/4.90G [00:00<?, ?B/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "e8ed9408e2834081bd4f20af78d16bc0",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "model-00018-of-00019.safetensors: 0%| | 0.00/4.98G [00:00<?, ?B/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "0d3ee31f8667458bb9d23f0cf47f1c9e",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "model-00019-of-00019.safetensors: 0%| | 0.00/4.22G [00:00<?, ?B/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "06963554ce7f459b89728330ac5cd589",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Loading checkpoint shards: 0%| | 0/19 [00:00<?, ?it/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "\n",
- "KeyboardInterrupt\n",
- "\n"
- ]
- }
- ],
- "source": [
- "from transformers import AutoModel\n",
- "\n",
- "model = AutoModel.from_pretrained(\"mistralai/Mixtral-8x7B-v0.1\")"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 3,
- "id": "3d670170-b8fe-4d23-96bd-e9d4c1710e57",
- "metadata": {},
- "outputs": [
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "71376c9f04e042678de531b3e06ef9b7",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "tokenizer_config.json: 0%| | 0.00/967 [00:00<?, ?B/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "b946a8fa81a34ec6af8dfca856ff5788",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "tokenizer.model: 0%| | 0.00/493k [00:00<?, ?B/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "8105390138fb4e12a7ceda5ab4a73dc0",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "tokenizer.json: 0%| | 0.00/1.80M [00:00<?, ?B/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "3e7442f9702e4870b7f49f93be265b32",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "special_tokens_map.json: 0%| | 0.00/72.0 [00:00<?, ?B/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- }
- ],
- "source": [
- "from transformers import AutoTokenizer, AutoConfig\n",
- "\n",
- "t = AutoTokenizer.from_pretrained(\"mistralai/Mixtral-8x7B-v0.1\")\n",
- "c = AutoConfig.from_pretrained(\"mistralai/Mixtral-8x7B-v0.1\")"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 1,
- "id": "072d6f69-aa58-4d00-9fb9-fb397a8231b2",
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Name: airllm\n",
- "Version: 2.6\n",
- "Summary: AirLLM allows single 4GB GPU card to run 70B large language models without quantization, distillation or pruning.\n",
- "Home-page: https://github.com/lyogavin/Anima/tree/main/air_llm\n",
- "Author: Gavin Li\n",
- "Author-email: [email protected]\n",
- "License: \n",
- "Location: /home/ubuntu/miniconda3/envs/ghostaienv/lib/python3.8/site-packages\n",
- "Requires: accelerate, huggingface-hub, optimum, safetensors, scipy, torch, tqdm, transformers\n",
- "Required-by: \n"
- ]
- }
- ],
- "source": [
- "!pip show airllm"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 2,
- "id": "17ba0595-b712-49f1-aa7b-997e01118a92",
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "../../__init__.py ../../airllm_mixtral.py ../../auto_model.py\n"
- ]
- }
- ],
- "source": [
- "!ls ../../*.py"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 3,
- "id": "a096444f-c3d9-457b-86bd-95084bf93d90",
- "metadata": {},
- "outputs": [],
- "source": [
- "!cp ../../*.py /home/ubuntu/miniconda3/envs/ghostaienv/lib/python3.8/site-packages/airllm/"
- ]
- },
{
"cell_type": "code",
"execution_count": 4,
- "id": "a4c321ef-0575-43db-b947-880ae484189d",
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "--2023-12-21 12:17:37-- https://huggingface.co/mistralai/Mixtral-8x7B-v0.1/raw/main/generation_config.json\n",
- "Resolving huggingface.co (huggingface.co)... 108.138.246.71, 108.138.246.85, 108.138.246.79, ...\n",
- "Connecting to huggingface.co (huggingface.co)|108.138.246.71|:443... connected.\n",
- "HTTP request sent, awaiting response... 200 OK\n",
- "Length: 116 [text/plain]\n",
- "Saving to: ‘generation_config.json’\n",
- "\n",
- "generation_config.j 100%[===================>] 116 --.-KB/s in 0s \n",
- "\n",
- "2023-12-21 12:17:37 (27.1 MB/s) - ‘generation_config.json’ saved [116/116]\n",
- "\n"
- ]
- }
- ],
- "source": [
- "!wget https://huggingface.co/mistralai/Mixtral-8x7B-v0.1/raw/main/generation_config.json\n",
- "!cp generation_config.json /home/ubuntu/.cache/huggingface/hub/models--mistralai--Mixtral-8x7B-v0.1/snapshots/58301445dc1378584211722b7ebf8743ec4e192b/"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
"id": "19eb8fee-ab17-4a54-9af2-ca809bd096b5",
"metadata": {},
"outputs": [
@@ -645,8 +147,18 @@
"name": "stderr",
"output_type": "stream",
"text": [
- "cuda:0: 86%|████████▌ | 30/35 [04:03<00:41, 8.36s/it]"
+ "cuda:0: 100%|██████████| 35/35 [04:29<00:00, 7.70s/it]\n"
]
+ },
+ {
+ "data": {
+ "text/plain": [
+ "'<s> I like to think of'"
+ ]
+ },
+ "execution_count": 4,
+ "metadata": {},
+ "output_type": "execute_result"
}
],
"source": [
@@ -654,7 +166,7 @@
"\n",
"MAX_LENGTH = 128\n",
"# could use hugging face model repo id:\n",
- "model = AutoModel.from_pretrained(\"/home/ubuntu/.cache/huggingface/hub/models--mistralai--Mixtral-8x7B-v0.1/snapshots/58301445dc1378584211722b7ebf8743ec4e192b/\")\n",
+ "model = AutoModel.from_pretrained(\"mistralai/Mixtral-8x7B-v0.1\")\n",
"\n",
"input_text = [\n",
" 'I like',\n",