AirLLM 2.8T LLM inference with single 4GB GPU
Python chinese-llmchinese-nlpfinetunegenerative-aiinstruct-gptinstruction-setllamallmloraopen-modelsopen-sourceopen-source-modelsqlora
Rearchitect core to hook-based layer streaming (Phase 2)
Yu Lie42a0fecee1c
diff --git a/air_llm/airllm/airllm_base.py b/air_llm/airllm/airllm_base.pyindex f7449b2..411373b 100644--- a/air_llm/airllm/airllm_base.py+++ b/air_llm/airllm/airllm_base.py@@ -6,12 +6,10 @@ import timefrom concurrent.futures import ThreadPoolExecutorimport torch-from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer, AutoModel, GenerationMixin, LlamaForCausalLM, GenerationConfig-from transformers.modeling_outputs import CausalLMOutputWithPast+from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer, GenerationConfigfrom accelerate import init_empty_weights-from accelerate.utils.modeling import set_module_tensor_to_device-from transformers.quantizers import AutoHfQuantizer, HfQuantizer+from transformers.quantizers import AutoHfQuantizerfrom .profiler import LayeredProfiler@@ -27,39 +25,32 @@ except ImportError:bitsandbytes_installed = False+class AirLLMBaseModel:+ """+ Memory-frugal wrapper around a Hugging Face ``*ForCausalLM`` model.-try:- from transformers.cache_utils import Cache, DynamicCache-- cache_utils_installed = True- print('>>>> cache_utils installed')-except ImportError:- cache_utils_installed = False---+ The checkpoint is split into per-layer shards on disk. The real transformers model is+ instantiated on the ``meta`` device (no memory used) and owns the full forward / generation+ logic. AirLLM only attaches forward hooks to each big module (embeddings, every decoder+ layer, the final norm and the lm_head) to stream that module's weights disk -> GPU right+ before it runs and free them right after, prefetching the next module on a worker thread.+ Because transformers drives the forward pass, AirLLM no longer needs to track per-architecture+ attention/rotary/cache details: new model architectures work as soon as transformers supports+ them.+ """--class AirLLMBaseModel(GenerationMixin):-- # customize layer names here+ # Subclasses override this to point at non-standard module names.def set_layer_names_dict(self):self.layer_names_dict = {'embed': 'model.embed_tokens',- 'layer_prefix': 'model.layers',- 'norm': 'model.norm',- 'lm_head': 'lm_head',}--+ 'layer_prefix': 'model.layers',+ 'norm': 'model.norm',+ 'lm_head': 'lm_head'}def __init__(self, model_local_path_or_repo_id, device="cuda:0", dtype=torch.float16, max_seq_len=512,layer_shards_saving_path=None, profiling_mode=False, compression=None,hf_token=None, prefetching=True, delete_original=False):"""- Sharded version of LlamaForCausalLM : the model is splitted into layer shards to reduce GPU memory usage.- During the forward pass, the inputs are processed layer by layer, and the GPU memory is freed after each layer.- To avoid loading the layers multiple times, we could save all the intermediate activations in RAM.-Parameters----------model_local_path_or_repo_id : str or Path@@ -69,158 +60,155 @@ class AirLLMBaseModel(GenerationMixin):dtype : torch.dtype, optionaldtype, by default torch.float16max_seq_len : int, optional- max seq lenght, by default 512+ max seq length, by default 512layer_shards_saving_path : str, optional- optional path to save layered shards model file, by default just save to the local cache of model, subdir named splitted_model will be saved- profiling_mode : book, optional- if to profile the model loading time, default to False- compression: str, optinal- setting to '4bit' or '8bit' to enable compression from 16 bits to 4 bits/8 bits which speeed up 4x or 2x inference time with a tiny accuracy loss.+ optional path to save the splitted shards, by default next to the model cache+ profiling_mode : bool, optional+ whether to profile the model loading time, default False+ compression: str, optional+ '4bit' or '8bit' to enable block-wise quantization of the on-disk shardshf_token: str, optional- huggingface api token could be provided, by default None+ huggingface api token+ prefetching: bool, optional+ overlap the next layer's disk load with the current layer's compute+ delete_original: bool, optional+ delete the original downloaded checkpoint after splitting to save disk space"""-self.profiling_mode = profiling_modeself.profiler = LayeredProfiler()self.total_disk_loading_time = Noneself.total_gpu_loading_time = Noneself.total_compression_overhead_time = None- self._supports_cache_class = Falseself.hf_quantizer = None- if compression is not None:- if not bitsandbytes_installed:- raise ImportError('WARNING: bitsandbytes not found. Compression needs bitsandbytes. To use compression, please install bitsandbytes: `pip install bitsandbytes`')-+ if compression is not None and not bitsandbytes_installed:+ raise ImportError('WARNING: bitsandbytes not found. Compression needs bitsandbytes. '+ 'To use compression, please install bitsandbytes: `pip install bitsandbytes`')self.compression = compressionself.hf_token = hf_token- # Save parameters-self.set_layer_names_dict()+ self.model_local_path, self.checkpoint_path = find_or_create_local_splitted_path(+ model_local_path_or_repo_id,+ layer_shards_saving_path,+ compression=compression,+ layer_names=self.layer_names_dict,+ hf_token=hf_token,+ delete_original=delete_original)- self.model_local_path, self.checkpoint_path = find_or_create_local_splitted_path(model_local_path_or_repo_id,- layer_shards_saving_path,- compression=compression,- layer_names=self.layer_names_dict,- hf_token=hf_token,- delete_original=delete_original)self.running_device = deviceself.device = torch.device(self.running_device)self.running_dtype = dtypeself.dtype = self.running_dtype- # Create modelif hf_token is not None:self.config = AutoConfig.from_pretrained(self.model_local_path, token=hf_token, trust_remote_code=True)else:self.config = AutoConfig.from_pretrained(self.model_local_path, trust_remote_code=True)self.generation_config = self.get_generation_config()- #print(f"using generation_config: {self.generation_config}")-self.tokenizer = self.get_tokenizer(hf_token=hf_token)+ # prefetch executor / state+ self.prefetching = prefetching+ if self.compression is not None and self.prefetching:+ print("prefetching is not supported together with compression for now; disabling prefetching.")+ self.prefetching = False+ self._executor = ThreadPoolExecutor(max_workers=1) if self.prefetching else None+ self._prefetch_future = None+ self._prefetched_idx = Noneself.init_model()- # get layer count:+ # compute layer count from the instantiated modelmodel_attr = self.modelfor attr_name in self.layer_names_dict["layer_prefix"].split("."):model_attr = getattr(model_attr, attr_name)-layers_count = len(model_attr)-- self.layer_names = [self.layer_names_dict['embed']] + [f'{self.layer_names_dict["layer_prefix"]}.{i}' for i in- range(layers_count)] + \+ self.layer_names = [self.layer_names_dict['embed']] + \+ [f'{self.layer_names_dict["layer_prefix"]}.{i}' for i in range(layers_count)] + \[self.layer_names_dict['norm'], self.layer_names_dict['lm_head']]self.max_seq_len = max_seq_len- self.main_input_name = "input_ids"-- # model weights prefetch cuda stream- self.prefetching = prefetching+ self.set_layers_from_layer_names()+ self._install_streaming_hooks()- if self.compression is not None:- self.prefetching = False- print(f"not support prefetching for compression for now. loading with no prepetching mode.")+ # ---- customization hooks for subclasses -------------------------------------------------- # this operation should run only if gpu is available- if prefetching and device.startswith("cuda"):- self.stream = torch.cuda.Stream()- else:- self.stream = None-- # if derived class needs to create generation config differently, like Mistrial, this function can be overriddendef get_generation_config(self):- # protective on generation config-try:return GenerationConfig.from_pretrained(self.model_local_path)- except Exception as e:+ except Exception:return GenerationConfig()- # a chance to customize tokenizerdef get_tokenizer(self, hf_token=None):if hf_token is not None:return AutoTokenizer.from_pretrained(self.model_local_path, token=hf_token, trust_remote_code=True)else:return AutoTokenizer.from_pretrained(self.model_local_path, trust_remote_code=True)- def get_use_better_transformer(self):- return True+ # ---- model construction -----------------------------------------------------------------def init_model(self):-- # Load meta model (no real memory used). Prefer PyTorch's SDPA (flash / memory-efficient- # attention), which is built into transformers now. BetterTransformer is deprecated and- # has been removed from optimum, so we no longer depend on it.+ # Build the real model on meta (no memory). include_buffers=False so non-persistent+ # buffers such as rotary inv_freq are actually computed (they aren't in the checkpoint).self.model = None-try:- with init_empty_weights():+ with init_empty_weights(include_buffers=False):self.model = AutoModelForCausalLM.from_config(self.config, attn_implementation="sdpa", trust_remote_code=True)except (ValueError, TypeError) as e:- # This architecture may not implement an SDPA path; fall back to the default attention.- print(f"attn_implementation='sdpa' not available ({e}), creating model with default attn implementation")- del self.model- clean_memory()+ print(f"attn_implementation='sdpa' not available ({e}), using default attn implementation")self.model = None-if self.model is None:- with init_empty_weights():+ with init_empty_weights(include_buffers=False):self.model = AutoModelForCausalLM.from_config(self.config, trust_remote_code=True)quantization_config = getattr(self.config, "quantization_config", None)-if quantization_config is not None:self.hf_quantizer = AutoHfQuantizer.from_config(quantization_config, pre_quantized=True)device_map = self.hf_quantizer.update_device_map(None)- self.hf_quantizer.preprocess_model(model = self.model, device_map = device_map)+ self.hf_quantizer.preprocess_model(model=self.model, device_map=device_map)self.model.eval()self.model.tie_weights()+ self.model.generation_config = self.generation_config- self.set_layers_from_layer_names()-- # Move buffers to device (not that much GPU memory used)+ # Move all (already-materialized) buffers to the running device, preserving their dtype.+ # This includes rotary inv_freq, which transformers computes once at the model level and+ # passes down to every decoder layer.for buffer_name, buffer in self.model.named_buffers():- set_module_tensor_to_device(self.model, buffer_name, self.running_device, value=buffer,- dtype=self.running_dtype)+ if buffer is not None and buffer.device.type != 'meta':+ set_module_tensor_to_device(self.model, buffer_name, self.running_device, value=buffer)- if 'rotary_pos_emb' in self.layer_names_dict:- # for glm keep rotary_pos_emb in gpu- self.load_rotary_pos_emb_to_device()+ # Force the model to report the running (cuda) device even though its parameters live on+ # meta between layer executions, so transformers' generation utilities place inputs/cache+ # tensors on the right device.+ self._patch_device_property()- def set_layers_from_layer_names(self):+ def _patch_device_property(self):+ running_device = torch.device(self.running_device)+ running_dtype = self.running_dtype+ base_cls = type(self.model)++ class _AirLLMRuntimeModel(base_cls):+ @property+ def device(self):+ return running_device+ @property+ def dtype(self):+ return running_dtype++ self.model.__class__ = _AirLLMRuntimeModel++ def set_layers_from_layer_names(self):self.layers = []model_attr = self.model@@ -231,7 +219,6 @@ class AirLLMBaseModel(GenerationMixin):model_attr = self.modelfor attr_name in self.layer_names_dict["layer_prefix"].split("."):model_attr = getattr(model_attr, attr_name)-self.layers.extend(list(model_attr))model_attr = self.model@@ -244,382 +231,92 @@ class AirLLMBaseModel(GenerationMixin):model_attr = getattr(model_attr, attr_name)self.layers.append(model_attr)- def load_rotary_pos_emb_to_device(self):- state_dict = load_layer(self.checkpoint_path, self.layer_names_dict['rotary_pos_emb'])- self.move_layer_to_device(state_dict)+ # ---- weight streaming -------------------------------------------------------------------def load_layer_to_cpu(self, layer_name):-t = time.time()-load_layer_output = load_layer(self.checkpoint_path, layer_name, self.profiling_mode)elapsed_time = time.time() - tif self.profiling_mode:state_dict, compression_time = load_layer_outputdisk_loading_time = elapsed_time - compression_time-self.profiler.add_profiling_time('load_safe_tensor', disk_loading_time)-self.profiler.add_profiling_time('compression_time', compression_time)else:state_dict = load_layer_output- # pin memory:- if self.prefetching:- t = time.time()- if torch.cuda.is_available(): # Check if CUDA is available- for k in state_dict.keys():- state_dict[k].pin_memory()- else:- # For CPU, no action is needed, but you could optionally add a log or message- print("Prefetching is enabled, but no pin_memory operation is needed for CPU.")-- elapsed_time = time.time() - t- if self.profiling_mode:- self.profiler.add_profiling_time('pin_memory_to_trigger_load', elapsed_time)+ if self.prefetching and torch.cuda.is_available():+ for k in state_dict.keys():+ state_dict[k].pin_memory()return state_dictdef move_layer_to_device(self, state_dict):- layers = []- for param_name, param in state_dict.items():- if self.hf_quantizer is None:- layers.append(param_name)- else:- if '.weight' in param_name:- layer_name = param_name[:param_name.index(".weight") + len(".weight")]- if layer_name not in layers:- layers.append(layer_name)-- for param_name in layers:+ moved = []+ for param_name in self._param_names_from_state_dict(state_dict):if (self.hf_quantizer is None or- not self.hf_quantizer.check_quantized_param(self.model, param_value=None, param_name=param_name, state_dict={})- ):- set_module_tensor_to_device(self.model, param_name, self.running_device, value=state_dict[param_name],- dtype=self.running_dtype,- )+ not self.hf_quantizer.check_quantized_param(self.model, param_value=None,+ param_name=param_name, state_dict={})):+ set_module_tensor_to_device(self.model, param_name, self.running_device,+ value=state_dict[param_name], dtype=self.running_dtype)else:- torch_dtype = self.hf_quantizer.update_torch_dtype(None)- self.hf_quantizer.create_quantized_param(self.model, state_dict[param_name], param_name, self.running_device, state_dict)- return layers-- # make GenerationMixin happy- def can_generate(self):- return True-- def prepare_inputs_for_generation(- self, input_ids, past_key_values=None, attention_mask=None, inputs_embeds=None, **kwargs- ):- if past_key_values is not None:- past_length = self.get_past_key_values_cache_seq_len(past_key_values) #[0][0].shape[2]-- # Some generation methods already pass only the last input ID- if input_ids.shape[1] > past_length:- remove_prefix_length = past_length+ self.hf_quantizer.create_quantized_param(self.model, state_dict[param_name], param_name,+ self.running_device, state_dict)+ moved.append(param_name)+ return moved++ def _param_names_from_state_dict(self, state_dict):+ if self.hf_quantizer is None:+ return list(state_dict.keys())+ names = []+ for param_name in state_dict.keys():+ if '.weight' in param_name:+ layer_name = param_name[:param_name.index(".weight") + len(".weight")]+ if layer_name not in names:+ names.append(layer_name)else:- # Default to old behavior: keep only final ID- remove_prefix_length = input_ids.shape[1] - 1-- input_ids = input_ids[:, remove_prefix_length:]-- position_ids = kwargs.get("position_ids", None)- if attention_mask is not None and position_ids is None:- # create position_ids on the fly for batch generation- position_ids = attention_mask.long().cumsum(-1) - 1- position_ids.masked_fill_(attention_mask == 0, 1)- if past_key_values:- position_ids = position_ids[:, -input_ids.shape[1]:]-- # if `inputs_embeds` are passed, we only want to use them in the 1st generation step- if inputs_embeds is not None and past_key_values is None:- model_inputs = {"inputs_embeds": inputs_embeds}+ names.append(param_name)+ return names++ def _install_streaming_hooks(self):+ # All modules execute in this order during a forward pass: embed -> layers -> norm -> lm_head.+ for idx, (name, module) in enumerate(zip(self.layer_names, self.layers)):+ module._airllm_idx = idx+ module.register_forward_pre_hook(self._pre_hook)+ module.register_forward_hook(self._post_hook)++ def _pre_hook(self, module, args):+ idx = module._airllm_idx++ if self.prefetching and self._prefetch_future is not None and self._prefetched_idx == idx:+ state_dict = self._prefetch_future.result()+ self._prefetch_future = Noneelse:- model_inputs = {"input_ids": input_ids}-- model_inputs.update(- {- "position_ids": position_ids,- "past_key_values": past_key_values,- "use_cache": kwargs.get("use_cache"),- "attention_mask": attention_mask,- }- )- return model_inputs-- def __call__(self, *args, **kwargs):- return self.forward(*args, **kwargs)-- def get_past_key_values_cache_seq_len(self, past_key_values):- return past_key_values[0][0].shape[2]- def get_sequence_len(self, seq):- return seq.shape[1]-- def get_pos_emb_args(self, len_p, len_s):- return {}-- def get_past_key_value_args(self, k_cache, v_cache):- return {'past_key_value': (k_cache, v_cache)}-- def get_attention_mask_args(self, full_attention_mask, len_p, len_s):- return {'attention_mask': full_attention_mask[:, :, -len_s:, -len_p - len_s:]}-- def get_position_ids_args(self, full_position_ids, len_p, len_s):-- return {'position_ids': full_position_ids[:, len_p:len_p + len_s]}-+ state_dict = self.load_layer_to_cpu(self.layer_names[idx])- def run_lm_head(self, layer, seq):- return layer(seq).float()+ module._airllm_moved = self.move_layer_to_device(state_dict)- def run_norm(self, layer, seq):- return layer(seq)+ if self.prefetching and (idx + 1) < len(self.layer_names):+ self._prefetch_future = self._executor.submit(self.load_layer_to_cpu, self.layer_names[idx + 1])+ self._prefetched_idx = idx + 1- def forward(- self,- input_ids: torch.LongTensor = None,- attention_mask: Optional[torch.Tensor] = None,- position_ids: Optional[torch.LongTensor] = None,- past_key_values: Optional[List[torch.FloatTensor]] = None,- inputs_embeds: Optional[torch.FloatTensor] = None,- labels: Optional[torch.LongTensor] = None,- use_cache: Optional[bool] = None,- output_attentions: Optional[bool] = None,- output_hidden_states: Optional[bool] = None,- return_dict: Optional[bool] = None,- ) -> Union[Tuple, CausalLMOutputWithPast]:-- if cache_utils_installed:- # we don't support kv cache for new version yet- use_cache = False-- if self.profiling_mode:- self.profiler.clear_profiling_time()-- forward_start = time.process_time()- forward_start_wall = time.time()-- # Reboot the model to make sure buffers are loaded and memory is clean- del self.model+ def _post_hook(self, module, args, output):+ if self.hf_quantizer is not None:+ for param_name in getattr(module, '_airllm_moved', []):+ set_module_tensor_to_device(self.model, param_name, 'meta')+ else:+ module.to('meta')clean_memory()- self.init_model()-- batch = [input_ids_unit.to(self.running_device).unsqueeze(0) for input_ids_unit in input_ids]- n_seq = len(batch[0])-- # Create attention mask for the largest input, and position ids to use KV cache- attention_mask = torch.ones(self.max_seq_len, self.max_seq_len)- attention_mask = attention_mask.triu(diagonal=1)[None, None, ...] == 0- attention_mask = attention_mask.to(self.running_device)- position_ids = torch.arange(self.max_seq_len, dtype=torch.long, device=self.running_device)[None, :]-- kv_cache_list = [] if use_cache else None- if use_cache:- for x in self.layers:- kv_cache_list.append(([], []))- all_hidden_states = [] * len(self.layers) if output_hidden_states else None- all_self_attns = [] * len(self.layers) if output_attentions else None-- with torch.inference_mode(), ThreadPoolExecutor() as executor:-- # Load first layer- if self.prefetching:- #with torch.cuda.stream(self.stream):- #state_dict = self.load_layer_to_cpu(self.layer_names[0])- future = executor.submit(self.load_layer_to_cpu, self.layer_names[0])--- for i, (layer_name, layer) in tqdm(enumerate(zip(self.layer_names, self.layers)),- desc=f'running layers({self.running_device})',- total=len(self.layers)):-- if self.prefetching:- if self.profiling_mode:- t = time.time()- # Load current layer and prepare next layer- state_dict = future.result()- #torch.cuda.current_stream().wait_stream(self.stream)- if self.profiling_mode:- elapsed_time = time.time() - t- self.profiler.add_profiling_time('load_safe_tensor_cpu_wait', elapsed_time)-- #for param_name, param in state_dict.items():- # state_dict[param_name] = param.to('cuda', non_blocking=True)-- if self.profiling_mode:- t = time.time()- moved_layers = self.move_layer_to_device(state_dict)- if self.profiling_mode:- elapsed_time = time.time() - t- self.profiler.add_profiling_time('create_layer_from_state_dict', elapsed_time)-- # kick off next layer loading-- if (i + 1) < len(self.layer_names):- #with torch.cuda.stream(self.stream):- #state_dict = self.load_layer_to_cpu(self.layer_names[i + 1])- if self.profiling_mode:- t = time.time()- future = executor.submit(self.load_layer_to_cpu, self.layer_names[i+1])- #for param_name, param in state_dict.items():- # state_dict[param_name] = param.to('cuda', non_blocking=True)-- if self.profiling_mode:- elapsed_time = time.time() - t- self.profiler.add_profiling_time('kick_off_load_cpu', elapsed_time)-- else:- state_dict = self.load_layer_to_cpu(layer_name)- if self.profiling_mode:- t = time.time()- moved_layers = self.move_layer_to_device(state_dict)- if self.profiling_mode:- elapsed_time = time.time() - t- self.profiler.add_profiling_time('create_layer_from_safe_tensor', elapsed_time)-- # Run layer-- for j, seq in enumerate(batch):-- if layer_name == self.layer_names_dict['embed']:- batch[j] = layer(seq)- elif layer_name == self.layer_names_dict['norm']:- #batch[j] = layer(seq[torch.arange(n_seq), batch_eos[j]][:, None])- batch[j] = self.run_norm(layer, seq)-- if output_attentions:- all_hidden_states[i].append(batch[j])- elif layer_name == self.layer_names_dict['lm_head']:- batch[j] = self.run_lm_head(layer, seq)- else:-- if output_attentions:- all_hidden_states[i].append(new_seq)-- if past_key_values is not None:- # join past kv- k_cache, v_cache = past_key_values[i - 1]- len_p = self.get_past_key_values_cache_seq_len(past_key_values)- len_s = self.get_sequence_len(seq)-- position_ids_args = self.get_position_ids_args(position_ids, len_p, len_s)- attention_mask_args = self.get_attention_mask_args(attention_mask, len_p, len_s)- past_key_value_args = self.get_past_key_value_args(k_cache, v_cache)-- kwargs = {'use_cache':True,- }-- pos_embed_args = self.get_pos_emb_args(len_p, len_s)- kwargs = {**kwargs, **past_key_value_args, **pos_embed_args, **attention_mask_args,- **position_ids_args}--- layer_outputs = layer(seq,- **kwargs- )- new_seq = layer_outputs[0]-- if output_attentions:- all_self_attns[i].append(layer_outputs[1])-- if use_cache:- (k_cache, v_cache) = layer_outputs[2 if output_attentions else 1]- kv_cache_list[i][0].append(k_cache)- kv_cache_list[i][1].append(v_cache)+ return output+ # ---- delegation to the underlying transformers model ------------------------------------- else:- len_seq = self.get_sequence_len(seq)+ def generate(self, *args, **kwargs):+ return self.model.generate(*args, **kwargs)+ def forward(self, *args, **kwargs):+ return self.model(*args, **kwargs)-- pos_embed_args = self.get_pos_emb_args(0, len_seq)- attention_mask_args = self.get_attention_mask_args(attention_mask, 0, len_seq)- position_ids_args = self.get_position_ids_args(position_ids, 0, len_seq)----- if not use_cache:-- kwargs = {'use_cache': False,- 'attention_mask': attention_mask[:, :, -len_seq:, -len_seq:],- }- kwargs = {**kwargs, **pos_embed_args, **attention_mask_args, **position_ids_args}--- new_seq = layer(seq, **kwargs)[0]- else:-- kwargs = {'use_cache': True,- 'attention_mask': attention_mask[:, :, -len_seq:, -len_seq:],- }- kwargs = {**kwargs, **pos_embed_args, **attention_mask_args, **position_ids_args}-- layer_out = layer(seq, **kwargs)-- # TODO: adopt Cache mechanism in 4.36- new_seq, (k_cache, v_cache) = layer_out- kv_cache_list[i][0].append(k_cache)- kv_cache_list[i][1].append(v_cache)-- # print(f"k_cache sizes: {[len(x[1]) for x in kv_cache_list]}")-- batch[j] = new_seq-- if output_hidden_states:- all_hidden_states += (torch.cat(batch, 0),)-- # Remove previous layer from memory (including buffers)-- if self.hf_quantizer is not None:- for param_name in moved_layers:#param_name, param in state_dict.items():- set_module_tensor_to_device(self.model, param_name,'meta')- else:- layer.to("meta")-- layer.to("meta")- clean_memory() # proposed by CPMP-- logits = torch.cat(batch, 0)- if use_cache:- kv_cache_list = kv_cache_list[1:-2]- for i in range(len(kv_cache_list)):- # print(f"{i} - {kv_cache_list[i][0].shape}")- kv_cache_list[i] = (torch.cat(kv_cache_list[i][0], 0), torch.cat(kv_cache_list[i][1], 0))- #print(f"returning kvcache size: {kv_cache_list[0][0].shape}")-- if output_attentions:- all_self_attns = all_self_attns[0:-2]- for i in range(len(all_self_attns)):- all_self_attns[i] = torch.cat(all_self_attns[i], 0)-- if output_hidden_states:- all_hidden_states = all_hidden_states[0:-2]- for i in range(len(all_hidden_states)):- all_hidden_states[i] = torch.cat(all_hidden_states[i], 0)-- if not return_dict:- return tuple(v for v in [logits,- tuple(kv_cache_list) if kv_cache_list is not None else None,- tuple(all_hidden_states) if all_hidden_states is not None else None,- tuple(all_self_attns) if all_self_attns is not None else None] if v is not None)- if self.profiling_mode:- forward_elapsed_time = time.process_time() - forward_start- forward_elapsed_time_wall = time.time() - forward_start_wall- self.profiler.print_profiling_time()--- print(f"total infer process time(including all above plus gpu compute): {forward_elapsed_time:.04f}")- print(f"total infer wall time(including all above plus gpu compute): {forward_elapsed_time_wall:.04f}")-- self.profiler.clear_profiling_time()--- return CausalLMOutputWithPast(- loss=None,- logits=logits,- past_key_values=tuple(kv_cache_list) if kv_cache_list is not None else None,- hidden_states=tuple(all_hidden_states) if all_hidden_states is not None else None,- attentions=tuple(all_self_attns) if all_hidden_states is not None else None,- )\ No newline at end of file+ def __call__(self, *args, **kwargs):+ return self.model(*args, **kwargs)diff --git a/air_llm/tests/test_streaming_gpu.py b/air_llm/tests/test_streaming_gpu.pynew file mode 100644index 0000000..24b4601--- /dev/null+++ b/air_llm/tests/test_streaming_gpu.py@@ -0,0 +1,106 @@+"""+Manual GPU test harness for AirLLM layer-streaming inference.++Runs a model through AirLLM (one layer on the GPU at a time) and reports peak VRAM.+Optionally caps the visible VRAM to emulate a small card, and/or compares the output+against a normal full-load run of the same model for correctness.++Examples+--------+# Tiny model, verify output matches a normal full-load run, report peak VRAM:+python test_streaming_gpu.py --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 --compare++# Emulate a 4GB card running a 7B model:+python test_streaming_gpu.py --model Qwen/Qwen2.5-7B-Instruct --max-vram-gb 4++# A real 70B on whatever card you have, capped to 8GB:+python test_streaming_gpu.py --model meta-llama/Llama-3.3-70B-Instruct --max-vram-gb 8 \+ --prompt "The capital of France is" --max-new-tokens 8+"""+import argparse+import time++import torch+++def cap_vram(max_vram_gb):+ """Limit how much of the GPU this process may allocate, to emulate a smaller card."""+ if max_vram_gb is None:+ return+ total = torch.cuda.get_device_properties(0).total_memory+ frac = (max_vram_gb * (1024 ** 3)) / total+ if frac >= 1.0:+ print(f"requested cap {max_vram_gb}GB >= device memory; not capping")+ return+ torch.cuda.set_per_process_memory_fraction(frac, 0)+ print(f"capped process VRAM to ~{max_vram_gb}GB ({frac*100:.1f}% of device)")+++def run_airllm(args):+ from airllm import AutoModel++ model = AutoModel.from_pretrained(args.model, compression=args.compression)+ ids = model.tokenizer([args.prompt], return_tensors="pt",+ return_attention_mask=False)["input_ids"].cuda()++ torch.cuda.reset_peak_memory_stats()+ t = time.time()+ out = model.generate(ids, max_new_tokens=args.max_new_tokens, do_sample=False,+ return_dict_in_generate=True)+ elapsed = time.time() - t++ text = model.tokenizer.decode(out.sequences[0])+ peak = torch.cuda.max_memory_allocated() / 1e6+ print(f"\n=== AirLLM ===")+ print(f"output : {text!r}")+ print(f"peak VRAM: {peak:.1f} MB")+ print(f"time : {elapsed:.1f}s ({args.max_new_tokens} new tokens)")+ return out.sequences[0].tolist()+++def run_reference(args):+ """Full-load run for correctness comparison (only feasible for small models)."""+ from transformers import AutoModelForCausalLM, AutoTokenizer++ tok = AutoTokenizer.from_pretrained(args.model)+ ref = AutoModelForCausalLM.from_pretrained(args.model, dtype=torch.float16).cuda().eval()+ ids = tok([args.prompt], return_tensors="pt", return_attention_mask=False)["input_ids"].cuda()+ out = ref.generate(ids, max_new_tokens=args.max_new_tokens, do_sample=False)+ text = tok.decode(out[0])+ print(f"\n=== Reference (full load) ===")+ print(f"output : {text!r}")+ del ref+ torch.cuda.empty_cache()+ return out[0].tolist()+++def main():+ p = argparse.ArgumentParser()+ p.add_argument("--model", required=True)+ p.add_argument("--prompt", default="The capital of France is")+ p.add_argument("--max-new-tokens", type=int, default=12)+ p.add_argument("--max-vram-gb", type=float, default=None)+ p.add_argument("--compression", default=None, choices=[None, "4bit", "8bit"])+ p.add_argument("--compare", action="store_true",+ help="also run a full-load reference and assert outputs match")+ args = p.parse_args()++ ref_seq = None+ if args.compare:+ # run reference first (before capping) so the full model fits+ ref_seq = run_reference(args)++ cap_vram(args.max_vram_gb)+ air_seq = run_airllm(args)++ if ref_seq is not None:+ match = ref_seq == air_seq+ print(f"\n=== correctness: {'MATCH' if match else 'MISMATCH'} ===")+ if not match:+ print(f"ref : {ref_seq}")+ print(f"air : {air_seq}")+ raise SystemExit(1)+++if __name__ == "__main__":+ main()