Return entropy to a2c
@@ -22,7 +22,7 @@ REWARD_STEPS = 5
BATCH_SIZE = 32
LEARNING_RATE_ACTOR = 1e-5
LEARNING_RATE_CRITIC = 1e-4
-ENTROPY_BETA = 1e-4
+ENTROPY_BETA = 1e-3
ENVS_COUNT = 16
TEST_ITERS = 1000
@@ -124,19 +124,16 @@ if __name__ == "__main__":
adv_v = vals_ref_v.unsqueeze(dim=-1) - value_v.detach()
log_prob_v = adv_v * calc_logprob(mu_v, var_v, actions_v)
loss_policy_v = -log_prob_v.mean()
- loss_policy_v.backward()
+ entropy_loss_v = ENTROPY_BETA * (-(torch.log(2*math.pi*var_v) + 1)/2).mean()
+ loss_v = loss_policy_v + entropy_loss_v
+ loss_v.backward()
opt_act.step()
-# entropy_loss_v = ENTROPY_BETA * (-(torch.log(2*math.pi*var_v) + 1)/2).mean()
-
-# loss_v = loss_policy_v + entropy_loss_v + loss_value_v
-# loss_v.backward()
-# optimizer.step()
tb_tracker.track("advantage", adv_v, step_idx)
tb_tracker.track("values", value_v, step_idx)
tb_tracker.track("batch_rewards", vals_ref_v, step_idx)
-# tb_tracker.track("loss_entropy", entropy_loss_v, step_idx)
+ tb_tracker.track("loss_entropy", entropy_loss_v, step_idx)
tb_tracker.track("loss_policy", loss_policy_v, step_idx)
tb_tracker.track("loss_value", loss_value_v, step_idx)
-# tb_tracker.track("loss_total", loss_v, step_idx)
+ tb_tracker.track("loss_total", loss_v, step_idx)
