f1d2a776dfa29bf08a8a13745be82e7d6788e365,keras/layers/dense_attention.py,BaseDenseAttention,_apply_scores,#BaseDenseAttention#Any#Any#Any#Any#,93

Before Change

    if scores_mask is not None:
      padding_mask = tf.logical_not(scores_mask)
      // Bias so padding positions do not contribute to attention distribution.
      scores -= 1.e9 * tf.cast(padding_mask, dtype=K.floatx())
    if training is None:
      training = K.learning_phase()
    weights = tf.compat.v1.math.softmax(scores)

After Change

      padding_mask = tf.logical_not(scores_mask)
      // Bias so padding positions do not contribute to attention distribution.
      // Note 65504. is the max float16 value.
      if scores.dtype is tf.float16:
        scores -= 65504. * tf.cast(padding_mask, dtype=scores.dtype)
      else:
        scores -= 1.e9 * tf.cast(padding_mask, dtype=scores.dtype)
    if training is None:
      training = K.learning_phase()
    weights = tf.compat.v1.math.softmax(scores)

Instances