f9f4587661731a334408d94a6bb79593a088c444,transformer/Optim.py,ScheduledOptim,_get_lr_scale,#ScheduledOptim#,22

Before Change

        self._optimizer.zero_grad()

    def _get_lr_scale(self):
        return np.min([
            np.power(self.n_current_steps, -0.5),
            np.power(self.n_warmup_steps, -1.5) * self.n_current_steps])

    def _update_learning_rate(self):
        """ Learning rate scheduling per step """

After Change

    def _get_lr_scale(self):
        n_steps, n_warmup_steps = self.n_steps, self.n_warmup_steps
        return min(n_steps ** (-0.5), n_steps * n_warmup_steps ** (-1.5))


    def _update_learning_rate(self):
        """ Learning rate scheduling per step """

Instances