1e50a6606c5215e0dc28d016bb5f57bd668f47af,src/gluonnlp/vocab/bert.py,BERTVocab,from_json,#Any#Any#,177

Before Change

        bert_vocab._sep_token = vocab_dict.get("sep_token")
        bert_vocab._cls_token = vocab_dict.get("cls_token")

        return bert_vocab

    @classmethod
    def from_sentencepiece(cls,
                           path,

After Change

                t for t in reserved_tokens if t not in special_tokens
            ]

        return cls(counter=count_tokens(token_to_idx.keys()),
                   unknown_token=unknown_token,
                   reserved_tokens=reserved_tokens,
                   token_to_idx=token_to_idx,
                   **identifiers_to_tokens)

    @classmethod
    def from_sentencepiece(cls,
                           path,

Instances