44ae6129731ee1d225db8a5f1bafea5fd4b9b542,open_seq2seq/parts/transformer/common.py,LayerNormalization,call,#LayerNormalization#Any#,70

Before Change

      variance = tf.reduce_mean(tf.abs(x), axis=[-1], keepdims=True)
      norm_x = tf.div(x , variance + self.epsilon)
    else:
      print("WARNING: Layer norm: type ", self.norm_type, "not supported")
      norm_x = x

    y = norm_x * self.scale + self.bias
    if dtype == tf.float16:

After Change

    if self.norm_type=="layernorm_L2":
      epsilon = self.epsilon
      dtype = x.dtype
      x = tf.cast(x=x, dtype=tf.float32)
      mean = tf.reduce_mean(x, axis=[-1], keepdims=True)
      variance = tf.reduce_mean(tf.square(x - mean), axis=[-1], keepdims=True)
      norm_x = (x - mean) * tf.rsqrt(variance + epsilon)
      result = norm_x * self.scale + self.bias
      return tf.cast(x=result, dtype=dtype)

    else: //if self.norm_type=="layernorm_L1":
      dtype = x.dtype

Instances