Fix gradient clipping when --clip-norm=0
@@ -265,7 +265,10 @@ class MultiprocessingTrainer(MultiprocessingEventLoop):
self._all_reduce_and_rescale_grads(grad_denom)
# clip grads
+ if self.args.clip_norm > 0:
grad_norm = torch.nn.utils.clip_grad_norm(self.model.parameters(), self.args.clip_norm)
+ else:
+ grad_norm = math.sqrt(sum([p.grad.data.norm()**2 for p in self.model.parameters()]))
# take an optimization step
self.optimizer.step()
