a65ee92b69c9dfa05defbee17abd7af09103f88e,rllib/agents/marwil/marwil_torch_policy.py,,marwil_loss,#,44

Before Change

    adv = advantages - state_values
    // Update averaged advantage norm.
    policy.ma_adv_norm.add_(
        1e-6 * (torch.mean(torch.pow(adv, 2.0)) - policy.ma_adv_norm))
    // //xponentially weighted advantages.
    exp_advs = torch.exp(policy.config["beta"] *
                         (adv / (1e-8 + torch.pow(policy.ma_adv_norm, 0.5))))

After Change

    // Advantage estimation.
    adv = advantages - state_values
    adv_squared = torch.mean(torch.pow(adv, 2.0))

    // Value loss.
    policy.v_loss = 0.5 * adv_squared

Instances