AirLLM 2.8T LLM inference with single 4GB GPU
Python chinese-llmchinese-nlpfinetunegenerative-aiinstruct-gptinstruction-setllamallmloraopen-modelsopen-sourceopen-source-modelsqlora
update test mixtral ipynb
Yu Lib4229e45347b
diff --git a/air_llm/tests/test_notebooks/test_mixtral.ipynb b/air_llm/tests/test_notebooks/test_mixtral.ipynbindex a3afdc5..d2f71da 100644--- a/air_llm/tests/test_notebooks/test_mixtral.ipynb+++ b/air_llm/tests/test_notebooks/test_mixtral.ipynb@@ -99,507 +99,9 @@"!pip install -U airllm transformers"]},- {- "cell_type": "code",- "execution_count": 2,- "id": "47de26bf-510a-4f8f-ae99-c4c22a0e12b7",- "metadata": {},- "outputs": [- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "d8829a3be8414a3d82aacee5f801ece7",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "config.json: 0%| | 0.00/720 [00:00<?, ?B/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "aed2af7d16004b849a401749c74346c8",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "model.safetensors.index.json: 0%| | 0.00/92.7k [00:00<?, ?B/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "e68dab4287b3496ba453c2664e167d4e",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "Downloading shards: 0%| | 0/19 [00:00<?, ?it/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "93c15f55db52460cb5ff5bffd604876e",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "model-00001-of-00019.safetensors: 0%| | 0.00/4.89G [00:00<?, ?B/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "02299832dca2482ba4e84b5a34b6604d",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "model-00002-of-00019.safetensors: 0%| | 0.00/4.98G [00:00<?, ?B/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "83dfc1d781dc4263958767984c60e5f3",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "model-00003-of-00019.safetensors: 0%| | 0.00/4.98G [00:00<?, ?B/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "1838e6535eed4020bbc52404a5f0e40e",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "model-00004-of-00019.safetensors: 0%| | 0.00/4.90G [00:00<?, ?B/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "b0282171d02d43c883e0918139336e7b",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "model-00005-of-00019.safetensors: 0%| | 0.00/4.98G [00:00<?, ?B/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "18a1e681ca7c40d78a2ecb7be9821548",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "model-00006-of-00019.safetensors: 0%| | 0.00/4.98G [00:00<?, ?B/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "da04e3e27395458b87c8320150cb5167",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "model-00007-of-00019.safetensors: 0%| | 0.00/4.90G [00:00<?, ?B/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "d3af708bd5984d4b8c6552afbbbf7767",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "model-00008-of-00019.safetensors: 0%| | 0.00/4.98G [00:00<?, ?B/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "05c6af1b24224cc6b7578e6ea8fbf0e0",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "model-00009-of-00019.safetensors: 0%| | 0.00/4.98G [00:00<?, ?B/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "33b169a15603482281fd45ab12a1c1e7",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "model-00010-of-00019.safetensors: 0%| | 0.00/4.90G [00:00<?, ?B/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "010af99eef2b4955a798bcbac95b3789",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "model-00011-of-00019.safetensors: 0%| | 0.00/4.98G [00:00<?, ?B/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "627af607d99c421688698093888c9da6",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "model-00012-of-00019.safetensors: 0%| | 0.00/4.98G [00:00<?, ?B/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "2cd9e5c547024ffca634e1a97f0b8933",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "model-00013-of-00019.safetensors: 0%| | 0.00/4.98G [00:00<?, ?B/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "e8c7cd63a0794a668820c5f1d7bb90ca",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "model-00014-of-00019.safetensors: 0%| | 0.00/4.90G [00:00<?, ?B/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "d6154839ba9946fbbf27563d8c76ee77",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "model-00015-of-00019.safetensors: 0%| | 0.00/4.98G [00:00<?, ?B/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "ad8792741a54426f8db778ea5203b6cc",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "model-00016-of-00019.safetensors: 0%| | 0.00/4.98G [00:00<?, ?B/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "c72707d9eee54201a36c94e0a1ea0a79",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "model-00017-of-00019.safetensors: 0%| | 0.00/4.90G [00:00<?, ?B/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "e8ed9408e2834081bd4f20af78d16bc0",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "model-00018-of-00019.safetensors: 0%| | 0.00/4.98G [00:00<?, ?B/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "0d3ee31f8667458bb9d23f0cf47f1c9e",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "model-00019-of-00019.safetensors: 0%| | 0.00/4.22G [00:00<?, ?B/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "06963554ce7f459b89728330ac5cd589",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "Loading checkpoint shards: 0%| | 0/19 [00:00<?, ?it/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "\n",- "KeyboardInterrupt\n",- "\n"- ]- }- ],- "source": [- "from transformers import AutoModel\n",- "\n",- "model = AutoModel.from_pretrained(\"mistralai/Mixtral-8x7B-v0.1\")"- ]- },- {- "cell_type": "code",- "execution_count": 3,- "id": "3d670170-b8fe-4d23-96bd-e9d4c1710e57",- "metadata": {},- "outputs": [- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "71376c9f04e042678de531b3e06ef9b7",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "tokenizer_config.json: 0%| | 0.00/967 [00:00<?, ?B/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "b946a8fa81a34ec6af8dfca856ff5788",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "tokenizer.model: 0%| | 0.00/493k [00:00<?, ?B/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "8105390138fb4e12a7ceda5ab4a73dc0",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "tokenizer.json: 0%| | 0.00/1.80M [00:00<?, ?B/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "3e7442f9702e4870b7f49f93be265b32",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "special_tokens_map.json: 0%| | 0.00/72.0 [00:00<?, ?B/s]"- ]- },- "metadata": {},- "output_type": "display_data"- }- ],- "source": [- "from transformers import AutoTokenizer, AutoConfig\n",- "\n",- "t = AutoTokenizer.from_pretrained(\"mistralai/Mixtral-8x7B-v0.1\")\n",- "c = AutoConfig.from_pretrained(\"mistralai/Mixtral-8x7B-v0.1\")"- ]- },- {- "cell_type": "code",- "execution_count": 1,- "id": "072d6f69-aa58-4d00-9fb9-fb397a8231b2",- "metadata": {},- "outputs": [- {- "name": "stdout",- "output_type": "stream",- "text": [- "Name: airllm\n",- "Version: 2.6\n",- "Summary: AirLLM allows single 4GB GPU card to run 70B large language models without quantization, distillation or pruning.\n",- "Home-page: https://github.com/lyogavin/Anima/tree/main/air_llm\n",- "Author: Gavin Li\n",- "Author-email: [email protected]\n",- "License: \n",- "Location: /home/ubuntu/miniconda3/envs/ghostaienv/lib/python3.8/site-packages\n",- "Requires: accelerate, huggingface-hub, optimum, safetensors, scipy, torch, tqdm, transformers\n",- "Required-by: \n"- ]- }- ],- "source": [- "!pip show airllm"- ]- },- {- "cell_type": "code",- "execution_count": 2,- "id": "17ba0595-b712-49f1-aa7b-997e01118a92",- "metadata": {},- "outputs": [- {- "name": "stdout",- "output_type": "stream",- "text": [- "../../__init__.py ../../airllm_mixtral.py ../../auto_model.py\n"- ]- }- ],- "source": [- "!ls ../../*.py"- ]- },- {- "cell_type": "code",- "execution_count": 3,- "id": "a096444f-c3d9-457b-86bd-95084bf93d90",- "metadata": {},- "outputs": [],- "source": [- "!cp ../../*.py /home/ubuntu/miniconda3/envs/ghostaienv/lib/python3.8/site-packages/airllm/"- ]- },{"cell_type": "code","execution_count": 4,- "id": "a4c321ef-0575-43db-b947-880ae484189d",- "metadata": {},- "outputs": [- {- "name": "stdout",- "output_type": "stream",- "text": [- "--2023-12-21 12:17:37-- https://huggingface.co/mistralai/Mixtral-8x7B-v0.1/raw/main/generation_config.json\n",- "Resolving huggingface.co (huggingface.co)... 108.138.246.71, 108.138.246.85, 108.138.246.79, ...\n",- "Connecting to huggingface.co (huggingface.co)|108.138.246.71|:443... connected.\n",- "HTTP request sent, awaiting response... 200 OK\n",- "Length: 116 [text/plain]\n",- "Saving to: ‘generation_config.json’\n",- "\n",- "generation_config.j 100%[===================>] 116 --.-KB/s in 0s \n",- "\n",- "2023-12-21 12:17:37 (27.1 MB/s) - ‘generation_config.json’ saved [116/116]\n",- "\n"- ]- }- ],- "source": [- "!wget https://huggingface.co/mistralai/Mixtral-8x7B-v0.1/raw/main/generation_config.json\n",- "!cp generation_config.json /home/ubuntu/.cache/huggingface/hub/models--mistralai--Mixtral-8x7B-v0.1/snapshots/58301445dc1378584211722b7ebf8743ec4e192b/"- ]- },- {- "cell_type": "code",- "execution_count": null,"id": "19eb8fee-ab17-4a54-9af2-ca809bd096b5","metadata": {},"outputs": [@@ -645,8 +147,18 @@"name": "stderr","output_type": "stream","text": [- "cuda:0: 86%|████████▌ | 30/35 [04:03<00:41, 8.36s/it]"+ "cuda:0: 100%|██████████| 35/35 [04:29<00:00, 7.70s/it]\n"]+ },+ {+ "data": {+ "text/plain": [+ "'<s> I like to think of'"+ ]+ },+ "execution_count": 4,+ "metadata": {},+ "output_type": "execute_result"}],"source": [@@ -654,7 +166,7 @@"\n","MAX_LENGTH = 128\n","# could use hugging face model repo id:\n",- "model = AutoModel.from_pretrained(\"/home/ubuntu/.cache/huggingface/hub/models--mistralai--Mixtral-8x7B-v0.1/snapshots/58301445dc1378584211722b7ebf8743ec4e192b/\")\n",+ "model = AutoModel.from_pretrained(\"mistralai/Mixtral-8x7B-v0.1\")\n","\n","input_text = [\n"," 'I like',\n",