dda76fd82e99ce21cba2e1345732fa859a20fc72,allennlp/data/token_indexers/pretrained_transformer_indexer.py,PretrainedTransformerIndexer,_add_encoding_to_vocabulary_if_needed,#PretrainedTransformerIndexer#Any#,65

Before Change

            return

        pretrained_vocab = self._tokenizer.get_vocab()
        for word, idx in pretrained_vocab.items():
            vocab._token_to_index[self._namespace][word] = idx
            vocab._index_to_token[self._namespace][idx] = word

After Change

        try:
            vocab_items = self._tokenizer.get_vocab().items()
        except NotImplementedError:
            vocab_items = (
                (self._tokenizer.convert_ids_to_tokens(idx), idx)
                for idx in range(self._tokenizer.vocab_size)
            )
        for word, idx in vocab_items:
            vocab._token_to_index[self._namespace][word] = idx
            vocab._index_to_token[self._namespace][idx] = word

Instances