No gradients stats
@@ -157,19 +157,11 @@ if __name__ == "__main__":
prob_v = F.softmax(logits_v)
entropy_loss_v = ENTROPY_BETA * (prob_v * log_prob_v).sum(dim=1).mean()
- # calculate policy gradients only
- loss_policy_v.backward(retain_graph=True)
- grads = np.concatenate([p.grad.data.cpu().numpy().flatten()
- for p in net.parameters()
- if p.grad is not None])
-
# apply entropy and value gradients
- loss_v = entropy_loss_v + loss_value_v
+ loss_v = entropy_loss_v + loss_value_v + loss_policy_v
loss_v.backward()
nn_utils.clip_grad_norm(net.parameters(), CLIP_GRAD)
optimizer.step()
- # get full loss
- loss_v += loss_policy_v
tb_tracker.track("advantage", adv_v, step_idx)
tb_tracker.track("values", value_v, step_idx)
@@ -178,9 +170,3 @@ if __name__ == "__main__":
tb_tracker.track("loss_policy", loss_policy_v, step_idx)
tb_tracker.track("loss_value", loss_value_v, step_idx)
tb_tracker.track("loss_total", loss_v, step_idx)
-
- tb_tracker.track("grad_l2", np.sqrt(np.mean(np.square(grads))), step_idx)
- tb_tracker.track("grad_max", np.max(np.abs(grads)), step_idx)
- tb_tracker.track("grad_var", np.var(grads), step_idx)
-
-pass
