# Not able to scale Trainer code to single node multi GPU **URL:** \ **Category:** 🤗Transformers\ **Created:** [September 14, 2023, 2:53pm UTC](https://discuss.huggingface.co/t/not-able-to-scale-trainer-code-to-single-node-multi-gpu/55025 "2023-09-14T14:53:22Z")\ **Posts on this page:** 1\ **Page:** 1
**Author:** ![ashish-ram](https://sea2.discourse-cdn.com/hellohellohello/user_avatar/discuss.huggingface.co/ashish-ram/32/19497_2.png) [@ashish-ram](https://discuss.huggingface.co/u/ashish-ram)\ **Post date:** [September 14, 2023, 2:53pm UTC](https://discuss.huggingface.co/t/not-able-to-scale-trainer-code-to-single-node-multi-gpu/55025/1 "2023-09-14T14:53:22Z")
I am using the code provided in this [blog](https://huggingface.co/blog/speecht5). Here is the link to google colab notebook [here](https://colab.research.google.com/drive/1i7I5pzBcU3WDFarDnzweIj4-sVVoIUFJ) The notebook runs perfectly fine in a machine with single GPU. However, when I run it on machine with Mutiple GPUs (n=4, Nvidia Tesla T4), I am getting the following error (at the end) To debug, I set `use_cpu=True` and the training loop runs ok as expected. So, I am pretty sure it is about multi-GPU. Trainer code (full code in the colab notebook) ```auto from transformers import Seq2SeqTrainingArguments training_args = Seq2SeqTrainingArguments( output_dir="./speecht5_tts_voxpopuli_nl", # change to a repo name of your choice per_device_train_batch_size=16, gradient_accumulation_steps=2, learning_rate=1e-5, warmup_steps=500, max_steps=4000, gradient_checkpointing=True, fp16=True, evaluation_strategy="steps", per_device_eval_batch_size=8, save_steps=1000, eval_steps=1000, logging_steps=25, report_to=["tensorboard"], load_best_model_at_end=True, greater_is_better=False, label_names=["labels"], push_to_hub=True, ) from transformers import Seq2SeqTrainer trainer = Seq2SeqTrainer( args=training_args, model=model, train_dataset=dataset["train"], eval_dataset=dataset["test"], data_collator=data_collator, tokenizer=processor.tokenizer, ) trainer.train() ``` ```auto /opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. warnings.warn('Was asked to gather along dimension 0, but all ' --------------------------------------------------------------------------- AttributeError Traceback (most recent call last) File :1 File /opt/conda/lib/python3.10/site-packages/transformers/trainer.py:1575, in Trainer.train(self, resume_from_checkpoint, trial, ignore_keys_for_eval, **kwargs) 1573 hf_hub_utils.enable_progress_bars() 1574 else: -> 1575 return inner_training_loop( 1576 args=args, 1577 resume_from_checkpoint=resume_from_checkpoint, 1578 trial=trial, 1579 ignore_keys_for_eval=ignore_keys_for_eval, 1580 ) File /opt/conda/lib/python3.10/site-packages/transformers/trainer.py:1875, in Trainer._inner_training_loop(self, batch_size, args, resume_from_checkpoint, trial, ignore_keys_for_eval) 1872 self.control = self.callback_handler.on_step_begin(args, self.state, self.control) 1874 with self.accelerator.accumulate(model): -> 1875 tr_loss_step = self.training_step(model, inputs) 1877 if ( 1878 args.logging_nan_inf_filter 1879 and not is_torch_tpu_available() 1880 and (torch.isnan(tr_loss_step) or torch.isinf(tr_loss_step)) 1881 ): 1882 # if loss is nan or inf simply add the average of previous logged losses 1883 tr_loss += tr_loss / (1 + self.state.global_step - self._globalstep_last_logged) File /opt/conda/lib/python3.10/site-packages/transformers/trainer.py:2740, in Trainer.training_step(self, model, inputs) 2737 return loss_mb.reduce_mean().detach().to(self.args.device) 2739 with self.compute_loss_context_manager(): -> 2740 loss = self.compute_loss(model, inputs) 2742 if self.args.n_gpu > 1: 2743 loss = loss.mean() # mean() to average on multi-gpu parallel training File /opt/conda/lib/python3.10/site-packages/transformers/trainer.py:2765, in Trainer.compute_loss(self, model, inputs, return_outputs) 2763 else: 2764 labels = None -> 2765 outputs = model(**inputs) 2766 # Save past state if it exists 2767 # TODO: this needs to be fixed and made cleaner later. 2768 if self.args.past_index >= 0: File /opt/conda/lib/python3.10/site-packages/torch/nn/modules/module.py:1501, in Module._call_impl(self, *args, **kwargs) 1496 # If we don't have any hooks, we want to skip the rest of the logic in 1497 # this function, and just call forward. 1498 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks 1499 or _global_backward_pre_hooks or _global_backward_hooks 1500 or _global_forward_hooks or _global_forward_pre_hooks): -> 1501 return forward_call(*args, **kwargs) 1502 # Do not call functions when jit is used 1503 full_backward_hooks, non_full_backward_hooks = [], [] File /opt/conda/lib/python3.10/site-packages/torch/nn/parallel/data_parallel.py:172, in DataParallel.forward(self, *inputs, **kwargs) 170 replicas = self.replicate(self.module, self.device_ids[:len(inputs)]) 171 outputs = self.parallel_apply(replicas, inputs, kwargs) --> 172 return self.gather(outputs, self.output_device) File /opt/conda/lib/python3.10/site-packages/torch/nn/parallel/data_parallel.py:184, in DataParallel.gather(self, outputs, output_device) 183 def gather(self, outputs, output_device): --> 184 return gather(outputs, output_device, dim=self.dim) File /opt/conda/lib/python3.10/site-packages/torch/nn/parallel/scatter_gather.py:86, in gather(outputs, target_device, dim) 83 # Recursive function calls like this create reference cycles. 84 # Setting the function to None clears the refcycle. 85 try: ---> 86 res = gather_map(outputs) 87 finally: 88 gather_map = None File /opt/conda/lib/python3.10/site-packages/torch/nn/parallel/scatter_gather.py:77, in gather..gather_map(outputs) 75 if not all(len(out) == len(d) for d in outputs): 76 raise ValueError('All dicts must have the same number of keys') ---> 77 return type(out)((k, gather_map([d[k] for d in outputs])) 78 for k in out) 79 if _is_namedtuple(out): 80 return type(out)._make(map(gather_map, zip(*outputs))) File :12, in __init__ (self, loss, spectrogram, past_key_values, decoder_hidden_states, decoder_attentions, cross_attentions, encoder_last_hidden_state, encoder_hidden_states, encoder_attentions) File /opt/conda/lib/python3.10/site-packages/transformers/utils/generic.py:343, in ModelOutput. __post_init__ (self) 340 # if we provided an iterator as first field and the iterator is a (key, value) iterator 341 # set the associated fields 342 if first_field_iterator: --> 343 for idx, element in enumerate(iterator): 344 if ( 345 not isinstance(element, (list, tuple)) 346 or not len(element) == 2 347 or not isinstance(element[0], str) 348 ): 349 if idx == 0: 350 # If we do not have an iterator of key/values, set it as attribute File /opt/conda/lib/python3.10/site-packages/torch/nn/parallel/scatter_gather.py:77, in (.0) 75 if not all(len(out) == len(d) for d in outputs): 76 raise ValueError('All dicts must have the same number of keys') ---> 77 return type(out)((k, gather_map([d[k] for d in outputs])) 78 for k in out) 79 if _is_namedtuple(out): 80 return type(out)._make(map(gather_map, zip(*outputs))) File /opt/conda/lib/python3.10/site-packages/torch/nn/parallel/scatter_gather.py:81, in gather..gather_map(outputs) 79 if _is_namedtuple(out): 80 return type(out)._make(map(gather_map, zip(*outputs))) ---> 81 return type(out)(map(gather_map, zip(*outputs))) File /opt/conda/lib/python3.10/site-packages/torch/nn/parallel/scatter_gather.py:71, in gather..gather_map(outputs) 69 out = outputs[0] 70 if isinstance(out, torch.Tensor): ---> 71 return Gather.apply(target_device, dim, *outputs) 72 if out is None: 73 return None File /opt/conda/lib/python3.10/site-packages/torch/autograd/function.py:506, in Function.apply(cls, *args, **kwargs) 503 if not torch._C._are_functorch_transforms_active(): 504 # See NOTE: [functorch vjp and autograd interaction] 505 args = _functorch.utils.unwrap_dead_wrappers(args) --> 506 return super().apply(*args, **kwargs) # type: ignore[misc] 508 if cls.setup_context == _SingleLevelFunction.setup_context: 509 raise RuntimeError( 510 'In order to use an autograd.Function with functorch transforms ' 511 '(vmap, grad, jvp, jacrev, ...), it must override the setup_context ' 512 'staticmethod. For more details, please see ' 513 'https://pytorch.org/docs/master/notes/extending.func.html') File /opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:56, in Gather.forward(ctx, target_device, dim, *inputs) 54 @staticmethod 55 def forward(ctx, target_device, dim, *inputs): ---> 56 assert all(i.device.type != 'cpu' for i in inputs), ( 57 'Gather function not implemented for CPU tensors' 58 ) 59 if (target_device == 'cpu'): 60 ctx.target_device = 'cpu' File /opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:56, in (.0) 54 @staticmethod 55 def forward(ctx, target_device, dim, *inputs): ---> 56 assert all(i.device.type != 'cpu' for i in inputs), ( 57 'Gather function not implemented for CPU tensors' 58 ) 59 if (target_device == 'cpu'): 60 ctx.target_device = 'cpu' AttributeError: 'NoneType' object has no attribute 'device' ```