f4275c0b80197e0f1bbd3a2a1a31cf07d85013b1,scripts/word_embeddings/evaluate_pretrained.py,,,#,173

Before Change

    // Load pre-trained embeddings
    if not args_.embedding_path:
        if args_.embedding_name.lower() == "fasttext":
            token_embedding_ = nlp.embedding.create(
                args_.embedding_name,
                source=args_.embedding_source,
                load_ngrams=args_.fasttext_load_ngrams,
                allow_extend=True,
                unknown_autoextend=True)
        else:
            token_embedding_ = nlp.embedding.create(
                args_.embedding_name, source=args_.embedding_source)
        name = "-" + args_.embedding_name + "-" + args_.embedding_source
    else:
        token_embedding_ = load_embedding_from_path(args_)
        name = ""

After Change

    known_tokens = set(token_embedding_.idx_to_token)

    if args_.similarity_datasets:
        with utils.print_time("find relevant tokens for similarity"):
            tokens = evaluation.get_similarity_task_tokens(args_)
        vocab = nlp.Vocab(nlp.data.count_tokens(tokens))
        with utils.print_time("set {} embeddings".format(len(tokens))):
            vocab.set_embedding(token_embedding_)
        evaluation.evaluate_similarity(

Instances