144b5afdd5d9262a5a9a9fdd9afa56a14b629ba1,texar/modules/encoders/transformer_encoders.py,TransformerEncoder,_build,#TransformerEncoder#Any#Any#,114

Before Change

                        num_units=self._hparams.embedding.dim,
                        causality=False,
                        scope="multihead_attention")
                    enc = tf.layers.dropout(
                        enc,
                        rate=self._hparams.dropout,
                        training=context.is_train())
                    self.enc += enc
                poswise_network = FeedForwardNetwork(hparams=self._hparams["poswise_feedforward"])
                with tf.variable_scope(poswise_network.variable_scope):
                    enc = layers.layer_normalize(self.enc)
                    enc = poswise_network(enc)
                    enc = tf.layers.dropout(
                        enc,
                        rate=self._hparams.dropout,
                        training=context.is_train())
                    self.enc += enc

        if not self._built:
            self._add_internal_trainable_variables()
            self._built = True

After Change

                        causality=False,
                        scope="multihead_attention"
                    )
                    self.enc = self.enc + tf.layers.dropout(
                        selfatt_output,
                        rate=self._hparams.dropout,
                        training=context.is_train()
                    )
                poswise_network = FeedForwardNetwork(hparams=self._hparams["poswise_feedforward"])
                with tf.variable_scope(poswise_network.variable_scope):
                    sub_output = tf.layers.dropout(
                        poswise_network(layers.layer_normalize(self.enc)),
                        rate=self._hparams.dropout,
                        training=context.is_train())
                    self.enc = self.enc + sub_output

        self.enc = layers.layer_normalize(self.enc)
        if not self._built:
            self._add_internal_trainable_variables()

Instances