e3931c33e2e5f4b148a918d798cc27224ed090b7,examples/bert/utils/tokenization.py,,convert_to_unicode,#Any#,29

Before Change

    
    Converts `text` to Unicode (if it"s not already), assuming utf-8 input.
    
    if six.PY3:
        if isinstance(text, str):
            return text
        elif isinstance(text, bytes):
            return text.decode("utf-8", "ignore")
        else:
            raise ValueError("Unsupported string type: %s" % (type(text)))
    elif six.PY2:
        if isinstance(text, str):
            return text.decode("utf-8", "ignore")
        elif isinstance(text, unicode):
            return text
        else:
            raise ValueError("Unsupported string type: %s" % (type(text)))
    else:
        raise ValueError("Not running on Python2 or Python 3?")


def printable_text(text):
    Returns text encoded in a way suitable for print or `tf.logging`.

    // These functions want `str` for both Python2 and Python3, but in one case

After Change

def convert_to_unicode(text):
    Returns the given argument as a unicode string.
    return tf.compat.as_text(text)

def printable_text(text):
    Returns text encoded in a way suitable for print or `tf.logging`.
    return tf.compat.as_str_any(text)

Instances