f98bd2ec9d4289939ff6661d5a9c43ee7e8996f7,models/shared_rnn.py,RNN,forward,#RNN#,195

Before Change

            if hidden_norms.data.max() > max_norm:
                logger.info(f"clipping {hidden_norms.max()} to {max_norm}")
                norm = hidden[hidden_norms > max_norm].norm(dim=-1)
                norm = norm.unsqueeze(-1)
                detached_norm = torch.autograd.Variable(norm.data,
                                                        requires_grad=False)
                hidden[hidden_norms > max_norm] *= max_norm/detached_norm

After Change

                clip_select = hidden_norms > max_norm
                clip_norms = hidden_norms[clip_select]

                mask = np.ones(hidden.size())
                normalizer = max_norm/clip_norms
                normalizer = normalizer[:, np.newaxis]

                mask[clip_select] = normalizer

Instances