33b341d1f8c6397b547f12ae2d5937bf5e2fd1ec,transformer/SubLayers.py,PositionwiseFeedForward,forward,#PositionwiseFeedForward#,75

Before Change

    def forward(self, x):
        residual = x
        output = F.relu(self.w_1(x.transpose(1, 2)))
        output = self.w_2(output).transpose(2, 1)
        output = self.dropout(output)
        return self.layer_norm(output + residual)

After Change

    def forward(self, x):
        residual = x
        output = x.transpose(1, 2)
        output = self.w_2(F.relu(self.w_1(output)))
        output = output.transpose(1, 2)
        output = self.dropout(output)
        output = self.layer_norm(output + residual)
        return output

Instances