cea292769af4ac688649573a11b20f4d69024e3d,tests/unit_test/processor_units/test_processor_units.py,,test_bert_tokenizer_unit,#,145

Before Change

def test_bert_tokenizer_unit():
    vocab_tokens = [
        "[UNK]", "[CLS]", "[SEP]", "want", "////want", "////ed", "wa", "un", "runn",
        "////ing", ","
    ]
    with tempfile.NamedTemporaryFile(delete=False) as vocab_writer:
        vocab_writer.write("".join(
            [x + "\n" for x in vocab_tokens]).encode("utf-8"))

    vocab_file = vocab_writer.name

After Change

                     "[UNK]", "glands", ".", "此", "外", ",", "在", "显", "微", "镜", "下"]

    vocab_dict = {}
    for idx, token in enumerate(vocab_tokens):
        vocab_dict[token] = idx

    clean_unit = units.BertClean()
    cleaned_text = clean_unit.transform(raw_text)
    chinese_tokenize_unit = units.ChineseTokenize()
    chinese_tokenized_text = chinese_tokenize_unit.transform(cleaned_text)

Instances