adeb1b1278619ff2d74d4fd82825e50a36f95ff4,allennlp/data/token_indexers/pretrained_transformer_indexer.py,PretrainedTransformerIndexer,_add_encoding_to_vocabulary_if_needed,#PretrainedTransformerIndexer#Any#,65

Before Change

        if hasattr(self._tokenizer, "vocab"):
            vocab_field_name = "vocab"
        elif hasattr(self._tokenizer, "encoder"):
            vocab_field_name = "encoder"
        else:
            logger.warning(
                Wasn"t able to fetch vocabulary from pretrained transformers lib.
                Neither <vocab> nor <encoder> are the valid fields for vocab.
                Your tokens will still be correctly indexed, but vocabulary file will not be saved.
            )
        if vocab_field_name is not None:
            pretrained_vocab = getattr(self._tokenizer, vocab_field_name)
            for word, idx in pretrained_vocab.items():
                vocab._token_to_index[self._namespace][word] = idx
                vocab._index_to_token[self._namespace][idx] = word

        self._added_to_vocabulary = True

    @overrides
    def count_vocab_items(self, token: Token, counter: Dict[str, Dict[str, int]]):

After Change

        if vocab_field_name is not None:
            pretrained_vocab = getattr(self._tokenizer, vocab_field_name)
            if vocab_field_name == "sp_model":
                for idx in range(len(pretrained_vocab)):
                    word = pretrained_vocab.id_to_piece(idx)
                    vocab._token_to_index[self._namespace][word] = idx
                    vocab._index_to_token[self._namespace][idx] = word
            else:
                for word, idx in pretrained_vocab.items():
                    vocab._token_to_index[self._namespace][word] = idx
                    vocab._index_to_token[self._namespace][idx] = word

Instances