Directly decay weight instead of L2 penalty
See
@@ -35,15 +35,14 @@ class NAG(Optimizer):
continue
d_p = p.grad.data
- if weight_decay != 0:
- d_p.add_(weight_decay, p.data)
-
param_state = self.state[p]
if 'momentum_buffer' not in param_state:
param_state['momentum_buffer'] = d_p.clone().zero_()
buf = param_state['momentum_buffer']
+ if weight_decay != 0:
+ p.data.mul_(1 - weight_decay)
p.data.add_(momentum * momentum, buf)
p.data.add_(-(1 + momentum) * lr, d_p)
