24d8fe8af00c6e0e47985d8b609f396e85bda8e3,matchzoo/models/drmmtks.py,DRMMTKS,attention_layer,#Any#Any#Any#,95

Before Change

        
        // shape = [B, L, 1]
        dense_input = keras.layers.Dense(1, use_bias=False)(attention_input)
        if attention_mask:
            // Since attention_mask is 1.0 for positions we want to attend and
            // 0.0 for masked positions, this operation will create a tensor
            // which is 0.0 for positions we want to attend and -10000.0 for
            // masked positions.

            // shape = [B, L, 1]
            adder = (1.0 - K.tf.cast(attention_mask, K.tf.float32)) * -1000.0
            // shape = [B, L, 1]
            dense_input += adder
        // shape = [B, L, 1]
        attention_probs = keras.layers.Lambda(
            lambda x: keras.layers.activations.softmax(x, axis=1),
            output_shape=lambda s: (s[0], s[1], s[2])
        )(dense_input)

After Change

        
        // shape = [B, L, 1]
        dense_input = keras.layers.Dense(1, use_bias=False)(attention_input)
        if attention_mask is not None:
            // Since attention_mask is 1.0 for positions we want to attend and
            // 0.0 for masked positions, this operation will create a tensor
            // which is 0.0 for positions we want to attend and -10000.0 for
            // masked positions.

            // shape = [B, L, 1]
            dense_input = keras.layers.Lambda(
                lambda x: x + (1.0 - attention_mask) * -1000.0,
                name="attention_mask"
            )(dense_input)
        // shape = [B, L, 1]
        attention_probs = keras.layers.Lambda(
            lambda x: keras.layers.activations.softmax(x, axis=1),
            output_shape=lambda s: (s[0], s[1], s[2]),
            name="attention_probs"

Instances