1e50a6606c5215e0dc28d016bb5f57bd668f47af,src/gluonnlp/vocab/bert.py,BERTVocab,from_json,#Any#Any#,177

Before Change

        bert_vocab._reserved_tokens = vocab_dict.get("reserved_tokens")
        bert_vocab._padding_token = vocab_dict.get("padding_token")
        bert_vocab._bos_token = vocab_dict.get("bos_token")
        bert_vocab._eos_token = vocab_dict.get("eos_token")
        bert_vocab._mask_token = vocab_dict.get("mask_token")
        bert_vocab._sep_token = vocab_dict.get("sep_token")
        bert_vocab._cls_token = vocab_dict.get("cls_token")

After Change

        token_to_idx = vocab_dict.get("token_to_idx")
        unknown_token = vocab_dict.get("unknown_token")
        reserved_tokens = vocab_dict.get("reserved_tokens")
        identifiers_to_tokens = vocab_dict.get("identifiers_to_tokens", dict())

        special_tokens = {unknown_token}

        // Backward compatibility for explicit serialization of padding_token,

Instances